diff --git a/PKGBUILD b/PKGBUILD
index f522bb0..3d3b485 100644
--- a/PKGBUILD
+++ b/PKGBUILD
@@ -7,9 +7,9 @@ pkgname=$pkgbase
 # comment the following line to build a single package containing the kernel and the headers
 (( 1 )) && pkgname=("$pkgbase" "$pkgbase-headers" "$pkgbase-docs")
 pkgdesc="The Linux Kernel and modules from Linus' git tree"
-depends=('coreutils' 'linux-firmware-git' 'mkinitcpio')
+depends=('coreutils' 'linux-firmware' 'module-init-tools' 'mkinitcpio')
 
-pkgver=4.8.rc8
+pkgver=4.11.rc5
 pkgrel=1
 url="http://www.kernel.org/"
 arch=(i686 x86_64)
@@ -24,7 +24,7 @@ sha256sums=('SKIP')
 # set _gitrev to a git revision (man gitrevisions) like a tag, a commit sha1
 # hash or a branch name to build from this tree instead of master
 
-_gitrev="v4.7.5"
+_gitrev="v4.10.8"
 
 ####################################################################
 # KERNEL CONFIG FILES
@@ -32,9 +32,9 @@ _gitrev="v4.7.5"
 # This PKGBUILD searches for config files in the current directory
 # and will use the first one it finds from the following
 # list as base configuration:
-# 	config.local
-# 	config.saved.$CARCH
-# 	config.$CARCH
+#       config.local
+#       config.saved.$CARCH
+#       config.$CARCH
 #
 ####################################################################
 
@@ -61,7 +61,15 @@ _gitrev="v4.7.5"
 #
 # Uncomment desired options
 #############################
-#_make_modules=0
+#_make_modules=1
+
+
+#######
+# Skip the merge of Linus's kernel tree
+#
+# _skip_merge=1
+
+
 
 MAKEFLAGS="-j $(expr $(cat /proc/cpuinfo |grep processor |wc -l) \* 2)"
 #######
@@ -87,12 +95,29 @@ _config_cmd="${_config_cmd:-menuconfig}"
 # _configure_only=1
 
 
+#######
+# The directory where the kernel should be built
+#
+# Can be useful, for example, if you want to compile on a
+# tmpfs mount, which can speed up the compilation process
+#
+#_build_dir="${_build_dir:-$srcdir}"
+
+
 #######
 # Append the date to the localversion
 #
-#	e.g. -ARCH -> -ARCH-20090422
+#       e.g. -ARCH -> -ARCH-20090422
 #
-# _date_localversion=1
+#_date_localversion=0
+
+
+
+#######
+# Set the pkgver to the kernel version
+# rather than the build date
+#
+# _kernel_pkgver=1
 
 
 #######
@@ -105,7 +130,7 @@ _save_config=1
 #######
 # Do not compress kernel modules
 #
-# _no_modules_compression=1
+_no_modules_compression=0
 
 
 #######
@@ -117,245 +142,250 @@ _save_config=1
 # internal variables
 (( 1 )) && _kernel_src="$pkgname"
 #(( 1 )) && _kernel_src="$BUILDDIR/$(find . -maxdepth 1 -type d -name "linux-*" -printf "%f\n" | head -1)"
+#(( 1 )) && _kernel_src="$_build_dir/$pkgname_$"
+
 
 #######
 # define required functions
 
 pkgver() {
-	cd "$_kernel_src"
-	git describe --always | sed 's/^v//;s/-/./g'
+        cd "$_kernel_src"
+        git describe --always | sed 's/^v//;s/-/./g'
 }
 
 # single package
 package() {
-	eval package_$pkgbase-headers
-	eval package_$pkgbase
+        eval package_$pkgbase-headers
+        eval package_$pkgbase
 }
 
 # split package functions
-eval "package_$pkgbase() { _generic_package_linux; }"
-eval "package_$pkgbase-headers() { _generic_package_linux-headers; }"
-eval "package_$pkgbase-docs() { _generic_package_linux-docs; }"
+eval "package_$pkgbase() { _generic_package_kernel; }"
+eval "package_$pkgbase-headers() { _generic_package_kernel-headers; }"
+eval "package_$pkgbase-docs() { _generic_package_kernel-docs; }"
 
 
 ##############################
 # where the magic happens...
 ##############################
 build() {
-	cd "$_kernel_src"
-	msg "Sanitizing source tree.."
-	[[ -n $_gitrev ]] && git reset --hard "$_gitrev" 
-	# cleaning source trees
-	git clean -f
+        cd "$_kernel_src"
+        msg "Sanitizing source tree.."
+        [[ -n $_gitrev ]] && git reset --hard "$_gitrev" 
+        # cleaning source trees
+        git clean -f
 
-	#################
-	# Apply patches
-	#################
-	msg "Applying patches..."
-	local i patches
-	for i in "${source[@]}"; do
-		i=${i##*/}
-		[[ $i =~ .*\.patch$ ]] && patches=("${patches[@]}" "$srcdir/$i")
-		[[ ${i%.*} =~ .*\.patch$ ]] && patches=("${patches[@]}" "$srcdir/${i%.*}")
-	done
+        #################
+        # Apply patches
+        #################
+        msg "Applying patches..."
+        local i patches
+        for i in "${source[@]}"; do
+                i=${i##*/}
+                [[ $i =~ .*\.patch$ ]] && patches=("${patches[@]}" "$srcdir/$i")
+                [[ ${i%.*} =~ .*\.patch$ ]] && patches=("${patches[@]}" "$srcdir/${i%.*}")
+        done
 
-	shopt -s nullglob
-	for i in "${patches[@]}" "$startdir/patches/"*; do
-		msg2 "Applying ${i##*/}..."
-		patch -Np1 -i "$i" || (error "Applying ${i##*/} failed" && return 1)
-	done
-	shopt -u nullglob
+        shopt -s nullglob
+        for i in "${patches[@]}" "$startdir/patches/"*; do
+                msg2 "Applying ${i##*/}..."
+                patch -Np1 -i "$i" || (error "Applying ${i##*/} failed" && return 1)
+        done
+        shopt -u nullglob
 
 
-	#################
-	# CONFIGURATION
-	#################
+        #################
+        # CONFIGURATION
+        #################
 
-	#########################
-	# Loading configuration
-	#########################
-	msg "Loading configuration..."
-	for i in local "saved.$CARCH" "$CARCH"; do
-		if [[ -e $startdir/config.$i ]]; then
-			msg2 "Using kernel config file config.$i..."
-			cp -f "$startdir/config.$i" .config
-			break
-		fi
-	done
+        #########################
+        # Loading configuration
+        #########################
+        msg "Loading configuration..."
+        for i in local "saved.$CARCH" "$CARCH"; do
+                if [[ -e $startdir/config.$i ]]; then
+                        msg2 "Using kernel config file config.$i..."
+                        cp -f "$startdir/config.$i" .config
+                        break
+                fi
+        done
 
-	[[ ! -e .config ]] &&
-		warning "No suitable kernel config file was found. You'll have to configure the kernel from scratch."
+        [[ ! -e .config ]] &&
+                warning "No suitable kernel config file was found. You'll have to configure the kernel from scratch."
 
 
-	###########################
-	# Start the configuration
-	###########################
-	msg "Updating configuration..."
-	yes "" | make config > /dev/null
+        ###########################
+        # Start the configuration
+        ###########################
+        msg "Updating configuration..."
+        yes "" | make config > /dev/null
 
-	if [[ -f "$startdir/config.saved.$CARCH" ]]; then
-		msg2 "migrating previous config..."
-		cp "$startdir/config.saved.$CARCH" .config
-		make oldconfig
-	else
-		msg2 "migrating default config..."
-		cp "$startdir/config.$CARCH" .config
-		make oldconfig
-	fi
-	if [[ -n ${_config_cmd// /} ]]; then
-		msg2 "Running make $_config_cmd..."
-		make $_config_cmd
-	else
-		warning "Unknown config command: $_config_cmd"
-	fi
+        if [[ -f "$startdir/config.saved.$CARCH" ]]; then
+                msg2 "migrating previous config..."
+                cp "$startdir/config.saved.$CARCH" .config
+                make oldconfig
+        else
+                msg2 "migrating default config..."
+                cp "$startdir/config.$CARCH" .config
+                make oldconfig
+        fi
+        if [[ -n ${_config_cmd// /} ]]; then
+                msg2 "Running make $_config_cmd..."
+                make $_config_cmd
+        else
+                warning "Unknown config command: $_config_cmd"
+        fi
 
-	##############################################
-	# Save the config file the package directory
-	##############################################
-	if [[ -n $_save_config || -n $_configure_only ]]; then
-		msg "Saving configuration..."
-		msg2 "Saving $_kernel_src/.config as $startdir/config.saved.$CARCH"
-		cp .config "$startdir/config.saved.$CARCH"
-	fi
+        ##############################################
+        # Save the config file the package directory
+        ##############################################
+        if [[ -n $_save_config || -n $_configure_only ]]; then
+                msg "Saving configuration..."
+                msg2 "Saving $_kernel_src/.config as $startdir/config.saved.$CARCH"
+                cp .config "$startdir/config.saved.$CARCH"
+        fi
 
 
-	#######################################
-	# Stop after configuration if desired
-	#######################################
-	if [[ -n $_configure_only ]]; then
-		rm -rf "$srcdir" "$pkgdir"
-		return 1
-	fi
+        #######################################
+        # Stop after configuration if desired
+        #######################################
+        if [[ -n $_configure_only ]]; then
+                rm -rf "$_kernel_src" "$srcdir" "$pkgdir"
+                return 1
+        fi
 
 
-	###############################
-	# Append date to localversion
-	###############################
-	if [[ -n $_date_localversion ]]; then
-		local _localversion="$(sed -rn 's/^CONFIG_LOCALVERSION="([^"]*)"$/\1/p' .config)"
-		[[ -n $_localversion ]] && msg2 "CONFIG_LOCALVERSION is set to: $_localversion"
+        ###############################
+        # Append date to localversion
+        ###############################
+        if [[ -n $_date_localversion ]]; then
+                local _localversion="$(sed -rn 's/^CONFIG_LOCALVERSION="([^"]*)"$/\1/p' .config)"
+                [[ -n $_localversion ]] && msg2 "CONFIG_LOCALVERSION is set to: $_localversion"
 
-		# since this is a git package, the $pkgver is equal to $(date +%Y%m%d)
-		msg2 "Appending $pkgver to CONFIG_LOCALVERSION..."
-		sed -ri "s/^(CONFIG_LOCALVERSION=).*$/\1\"$_localversion-$pkgver\"/" .config
-	fi
+                # since this is a git package, the $pkgver is equal to $(date +%Y%m%d)
+                msg2 "Appending $pkgver to CONFIG_LOCALVERSION..."
+                sed -ri "s/^(CONFIG_LOCALVERSION=).*$/\1\"$_localversion-$pkgver\"/" .config
+        fi
 
 
-	####################################
-	# Append pkgrel to kernel version
-	####################################
-	sed -ri "s/^(EXTRAVERSION =).*$/\1 -$pkgrel/" Makefile
 
+        #################
+        # BUILD PROCESS
+        #################
 
-	####################################
-	# don't run depmod on 'make install'
-	####################################
-	sed -i '2iexit 0' scripts/depmod.sh
-	git update-index --assume-unchanged scripts/depmod.sh
+        ################################
+        # Build the kernel and modules
+        ################################
+        msg "Building kernel and modules..."
+        if [[ -n $_make_modules ]]; then
+                make $MAKEFLAGS V="$_verbose" bzImage modules
+        else
+                make $MAKEFLAGS V="$_verbose" bzImage
+        fi
 
+       ############
+       # CLEANUP
+       ############
 
-	#################
-	# BUILD PROCESS
-	#################
-
-	################################
-	# Build the kernel and modules
-	################################
-	msg "Building kernel and modules..."
-if [[ -n $_make_modules ]]; then
-	make $MAKEFLAGS V="$_verbose" bzImage modules
-else
-	make $MAKEFLAGS V="$_verbose" bzImage
-fi
+       ###################################
+       # Copy files from build directory
+       ####################################
+#       if (( ! CLEANUP )) && [[ $_build_dir != $srcdir ]]; then
+#               msg "Saving $_kernel_src to $srcdir/${_kernel_src##*/}..."
+#               mv "$_kernel_src" "$srcdir"
+#               rm -rf "$_kernel_src"
+#       fi
 }
 
 
 _generic_package_initialization() {
-	cd "$srcdir/${_kernel_src##*/}"
+        cd "$_kernel_src"
 
-	_karch="x86"
+        _karch="x86"
+
+        ######################
+        # Get kernel version
+        ######################
+        _kernver=$(make kernelrelease)
+        _basekernel=${_kernver%%-*}
+
+       ############################################################
+       # Use kernel version instead of the current date as pkgver
+       ############################################################
+       if [[ -n $_kernel_pkgver ]]; then
+               pkgver=${_kernver//-/_}
+               msg "Setting pkgver to kernel version: $pkgver"
+       fi
 
-	######################
-	# Get kernel version
-	######################
-	_kernver=$(make kernelrelease)
-	_basekernver=${_kernver%%-*}
 }
 
-_generic_package_linux() {
-	pkgdesc="The Linux Kernel and modules from Linus' git tree"
-	backup=(etc/mkinitcpio.d/$pkgname.preset)
-	install=$pkgname.install
-	changelog=$pkgname.changelog
+_generic_package_kernel() {
+        pkgdesc="The Linux Kernel and modules from Linus' git tree"
+        depends=('coreutils' 'linux-firmware' 'module-init-tools' 'mkinitcpio')
+        backup=(etc/mkinitcpio.d/$pkgname.preset)
+        install=$pkgname.install
+        changelog=$pkgname.changelog
 
-	# set required variables
-	_generic_package_initialization
+        # set required variables
+        _generic_package_initialization
 
 
-	#############################################################
-	# Provide linux
-	# (probably someone wants to use this kernel exclusively?)
-	#############################################################
-	provides=("${provides[@]}" "linux=${_kernver//-/_}")
+        #############################################################
+        # Provide linux
+        # (probably someone wants to use this kernel exclusively?)
+        #############################################################
+        provides=("${provides[@]}" "linux=${_kernver//-/_}")
 
-	################
-	# INSTALLATION
-	################
+        ################
+        # INSTALLATION
+        ################
 
-	#####################
-	# Install the image
-	#####################
-	msg "Installing kernel image..."
-	install -Dm644 arch/$_karch/boot/bzImage "$pkgdir/boot/vmlinuz-$pkgname"
+        #####################
+        # Install the image
+        #####################
+        msg "Installing kernel image..."
+        install -Dm644 arch/$_karch/boot/bzImage "$pkgdir/boot/vmlinuz-$pkgname"
 
 
-	##########################
-	# Install kernel modules
-	##########################
-	msg "Installing kernel modules..."
-	if [[ -n $_make_modules ]]; then
-	# force -j1 to work around make 3.82 bug
-		make -j1 INSTALL_MOD_PATH="$pkgdir/usr" modules_install
-	[[ -z $_no_modules_compression ]] && find "$pkgdir" -name "*.ko" -exec gzip -9 {} +
+        ##########################
+        # Install kernel modules
+        ##########################
+        if [[ -n $_make_modules ]]; then
+	        msg "Installing kernel modules..."
+                make INSTALL_MOD_PATH="$pkgdir" modules_install
+	        [[ -z $_no_modules_compression ]] && find "$pkgdir" -name "*.ko" 
+-exec gzip -9 {} +
 
-	#########################################################
-	# Set up extramodules directory (for external modules)
-	#########################################################
-	local extramodules="$pkgdir/usr/lib/modules/extramodules-$(cut -d. -f1,2 <<<$_basekernver)"
-	local modversion=$(grep '^CONFIG_LOCALVERSION=' .config | cut -d'"' -f2)
-	[[ -n $modversion ]] && extramodules+=$modversion
-	install -dm755 "${extramodules}${_pkgext}"
-	echo $_kernver > "${extramodules}${_pkgext}/version"
-	ln -s "../${extramodules##*/}${_pkgext}" "$pkgdir/usr/lib/modules/$_kernver/extramodules"
 
-	
-	##################################
-	# Create important symlinks
-	##################################
-	msg "Creating important symlinks..."
+        ##################################
+        # Create important symlinks
+        ##################################
+        msg "Creating important symlinks..."
 
-	# Create generic modules symlink
-	if [[ $_kernver != ${_basekernver}${_pkgext} ]]; then
-		cd "$pkgdir/usr/lib/modules"
-		ln -s "$_kernver" "${_basekernver}${_pkgext}"
-		cd "$OLDPWD"
+        # Create generic modules symlink
+
+   	    if [[ $_kernver != ${_basekernel}${_pkgext} ]]; then
+        	       cd "$pkgdir/lib/modules"
+	               ln -s "$_kernver" "${_basekernel}${_pkgext}"
+        	       cd "$OLDPWD"
+
+
+        	# remove header symlinks
+	        cd "$pkgdir/lib/modules/$_kernver"
+                rm -rf source build
+                cd "$OLDPWD"
+	    fi
 	fi
 
-	# remove header symlinks
-	cd "$pkgdir/usr/lib/modules/$_kernver"
-		rm -rf source build
-		cd "$OLDPWD"
-	fi
+        ############################
+        # Install mkinitcpio files
+        ############################
+        install -d "$pkgdir/etc/mkinitcpio.d"
 
+        msg "Generating $pkgname.preset..."
+        cat > "$pkgdir/etc/mkinitcpio.d/$pkgname.preset" <<EOF
+# mkinitcpio preset file for $pkgname
 
-	############################
-	# Install mkinitcpio files
-	############################
-	install -d "$pkgdir/etc/mkinitcpio.d"
-
-	msg "Generating $pkgname.preset..."
-	cat > "$pkgdir/etc/mkinitcpio.d/$pkgname.preset" <<EOF
-# mkinitcpio preset file for the '$pkgname' package
 
 ALL_config="/etc/mkinitcpio.conf"
 ALL_kver="/boot/vmlinuz-$pkgname"
@@ -368,149 +398,159 @@ COMPRESSION="lz4"   # since kernel 2.6.34
 
 EOF
 
-
-	#######################
-	# Update install file
-	#######################
-	msg "Updating install file..."
-	sed -ri "s/^(pkgname=).*$/\1$pkgname/" "$startdir/$pkgname.install"
-	sed -ri "s/^(kernver=).*$/\1$_kernver/" "$startdir/$pkgname.install"
+       msg "Generating $pkgname.kver..."
+       echo -e "# DO NOT EDIT THIS FILE\nALL_kver='$_kernver'" \
+               > "$pkgdir/etc/mkinitcpio.d/$pkgname.kver"
 
 
-	#######################
-	# Remove the firmware
-	#######################
-	rm -rf "$pkgdir/usr/lib/firmware"
+
+        #######################
+        # Update install file
+        #######################
+        msg "Updating install file..."
+        sed -ri "s/^(pkgname=).*$/\1$pkgname/" "$startdir/$pkgname.install"
+        sed -ri "s/^(kernver=).*$/\1$_kernver/" "$startdir/$pkgname.install"
 
 
-	#######################
-	# Run depmod
-	#######################
-	if [[ -n $_make_modules ]]; then
-		depmod -a "$pkgdir/usr"
-		depmod -b "$pkgdir/usr" -F System.map "$_kernver"
+
+        #######################
+        # Remove the firmware
+        #######################
+
+	# remove the firmware
+	rm -rf "${pkgdir}/lib/firmware"
+        if [[ -n $_make_modules ]]; then
+		# Now we call depmod...
+		depmod -b "${pkgdir}" -F System.map "${_kernver}"
+
+		# move module tree /lib -> /usr/lib
+		mkdir -p "${pkgdir}/usr"
+		mv "${pkgdir}/lib" "${pkgdir}/usr/"
 	fi
+
 }
 
 
+_generic_package_kernel-headers() {
+        pkgdesc="Header files and scripts for building modules for $pkgbase"
+        depends=("$pkgbase")
 
-_generic_package_linux-headers() {
-	pkgdesc="Header files and scripts for building modules for $pkgbase"
-	depends=("$pkgbase")
+        # set required variables
+        _generic_package_initialization
 
-	# set required variables
-	_generic_package_initialization
-
-	#############################################################
-	# Provide linux-headers
-	# (probably someone wants to use this kernel exclusively?)
-	#############################################################
-	provides=("${provides[@]}" "linux-headers=${_kernver//-/_}")
+        #############################################################
+        # Provide linux-headers
+        # (probably someone wants to use this kernel exclusively?)
+        #############################################################
+        provides=("${provides[@]}" "linux-headers=${_kernver//-/_}")
 
 
-	##############################
-	# Install fake kernel source
-	##############################
-	install -Dm644 Module.symvers  "$pkgdir/usr/src/linux-$_kernver/Module.symvers"
-	install -Dm644 Makefile        "$pkgdir/usr/src/linux-$_kernver/Makefile"
-	install -Dm644 kernel/Makefile "$pkgdir/usr/src/linux-$_kernver/kernel/Makefile"
-	install -Dm644 .config         "$pkgdir/usr/src/linux-$_kernver/.config"
-	install -Dm644 .config         "$pkgdir/usr/lib/modules/$_kernver/.config"
+        ##############################
+        # Install fake kernel source
+        ##############################
+        install -Dm644 Module.symvers  "$pkgdir/usr/src/linux-$_kernver/Module.symvers"
+        install -Dm644 Makefile        "$pkgdir/usr/src/linux-$_kernver/Makefile"
+        install -Dm644 kernel/Makefile "$pkgdir/usr/src/linux-$_kernver/kernel/Makefile"
+        install -Dm644 .config         "$pkgdir/usr/lib/modules/$_kernver/.config"
 
 
-	#######################################################
-	# Install scripts directory and fix permissions on it
-	#######################################################
-	cp -a scripts "$pkgdir/usr/src/linux-$_kernver"
+
+        #######################################################
+        # Install scripts directory and fix permissions on it
+        #######################################################
+        cp -a scripts "$pkgdir/usr/src/linux-$_kernver"
 
 
-	##########################
-	# Install header files
-	##########################
-	msg "Installing header files..."
+        ##########################
+        # Install header files
+        ##########################
+        msg "Installing header files..."
 
-	for i in net/ipv4/netfilter/ipt_CLUSTERIP.c \
-		$(find include/ net/mac80211/ drivers/md -iname "*.h") \
-		$(find include/config/ -type f) \
-		$(find . -name "Kconfig*")
-	do
-		mkdir -p "$pkgdir/usr/src/linux-$_kernver/${i%/*}"
-		cp -af "$i" "$pkgdir/usr/src/linux-$_kernver/$i"
-	done
+        for i in net/ipv4/netfilter/ipt_CLUSTERIP.c \
+                $(find include/ net/mac80211/ drivers/{md,media/video/} -iname "*.h") \
+                $(find include/config/ -type f) \
+                $(find . -name "Kconfig*")
+        do
+                mkdir -p "$pkgdir/usr/src/linux-$_kernver/${i%/*}"
+                cp -af "$i" "$pkgdir/usr/src/linux-$_kernver/$i"
+        done
 
-	# required by virtualbox and probably others
-	ln -s "../generated/autoconf.h" "$pkgdir/usr/src/linux-$_kernver/include/linux/"
+        # required by virtualbox and probably others
+        ln -s "../generated/autoconf.h" "$pkgdir/usr/src/linux-$_kernver/include/linux/"
 
 
-	########################################
-	# Install architecture dependent files
-	########################################
-	msg "Installing architecture files..."
-	mkdir -p "$pkgdir/usr/src/linux-$_kernver/arch/$_karch/kernel"
-	cp -a arch/$_karch/kernel/asm-offsets.s "$pkgdir/usr/src/linux-$_kernver/arch/$_karch/kernel/"
+        ########################################
+        # Install architecture dependent files
+        ########################################
+        msg "Installing architecture files..."
+        mkdir -p "$pkgdir/usr/src/linux-$_kernver/arch/$_karch/kernel"
+        cp -a arch/$_karch/kernel/asm-offsets.s "$pkgdir/usr/src/linux-$_kernver/arch/$_karch/kernel/"
 
-	cp -a arch/$_karch/Makefile* "$pkgdir/usr/src/linux-$_kernver/arch/$_karch/"
-	cp -a arch/$_karch/configs "$pkgdir/usr/src/linux-$_kernver/arch/$_karch/"
+        cp -a arch/$_karch/Makefile* "$pkgdir/usr/src/linux-$_kernver/arch/$_karch/"
+        cp -a arch/$_karch/configs "$pkgdir/usr/src/linux-$_kernver/arch/$_karch/"
 
-	# copy arch includes for external modules and fix the nVidia issue
-	mkdir -p "$pkgdir/usr/src/linux-$_kernver/arch/$_karch"
-	cp -a "arch/$_karch/include" "$pkgdir/usr/src/linux-$_kernver/arch/$_karch/"
+        # copy arch includes for external modules and fix the nVidia issue
+        mkdir -p "$pkgdir/usr/src/linux-$_kernver/arch/$_karch"
+        cp -a "arch/$_karch/include" "$pkgdir/usr/src/linux-$_kernver/arch/$_karch/"
 
-	# create a necessary symlink to the arch folder
-	cd "$pkgdir/usr/src/linux-$_kernver/arch"
+        # create a necessary symlink to the arch folder
+        cd "$pkgdir/usr/src/linux-$_kernver/arch"
 
-	if [[ $CARCH = "x86_64" ]]; then
-		ln -s $_karch x86_64
-	else
-		ln -s $_karch i386
+        if [[ $CARCH = "x86_64" ]]; then
+                ln -s $_karch x86_64
+        else
+                ln -s $_karch i386
+        fi
+
+        cd "$OLDPWD"
+
+
+        ################################
+        # Remove unneeded architecures
+        ################################
+        msg "Removing unneeded architectures..."
+        for i in "$pkgdir/usr/src/linux-$_kernver/arch/"*; do
+                [[ ${i##*/} =~ ($_karch|Kconfig) ]] || rm -rf "$i"
+        done
+
+
+        ############################
+        # Remove .gitignore files
+        ############################
+        msg "Removing .gitignore files from kernel source..."
+        find "$pkgdir/usr/src/linux-$_kernver/" -name ".gitignore" -delete
+
+
+        ##################################
+        # Create important symlinks
+        ##################################
+        msg "Creating important symlinks..."
+
+        # the build symlink needs to be relative
+        if [[ -n $_make_modules ]]; then
+	        cd "$pkgdir/usr/lib/modules/$_kernver"
+                rm -rf source build
+                ln -s "/usr/src/linux-$_kernver" build
+                cd "$OLDPWD"
 	fi
 
-	cd "$OLDPWD"
-
-
-	################################
-	# Remove unneeded architecures
-	################################
-	msg "Removing unneeded architectures..."
-	for i in "$pkgdir/usr/src/linux-$_kernver/arch/"*; do
-		[[ ${i##*/} =~ ($_karch|Kconfig) ]] || rm -rf "$i"
-	done
-
-
-	############################
-	# Remove .gitignore files
-	############################
-	msg "Removing .gitignore files from kernel source..."
-	find "$pkgdir/usr/src/linux-$_kernver/" -name ".gitignore" -delete
-
-
-	##################################
-	# Create important symlinks
-	##################################
-	msg "Creating important symlinks..."
-
-	# the build symlink needs to be relative
-	cd "$pkgdir/usr/lib/modules/$_kernver"
-		rm -rf source build
-		ln -s "/usr/src/linux-$_kernver" build
-		cd "$OLDPWD"
-
-	if [[ $_kernver != ${_basekernver}${_pkgext} ]]; then
-		cd "$pkgdir/usr/src"
-		ln -s "linux-$_kernver" "linux-${_basekernver}${_pkgext}"
-		cd "$OLDPWD"
-	fi
+        if [[ $_kernver != ${_basekernver}${_pkgext} ]]; then
+                cd "$pkgdir/usr/src"
+                ln -s "linux-$_kernver" "linux-${_basekernel}${_pkgext}"
+                cd "$OLDPWD"
+        fi
 }
 
-_generic_package_linux-docs() {
-	pkgdesc="Kernel hackers manual - HTML documentation that comes with the Linux kernel."
-	depends=("$pkgbase")
+_generic_package_kernel-docs() {
+        pkgdesc="Kernel hackers manual - HTML documentation that comes with the Linux kernel."
+        depends=("$pkgbase")
 
-	# set required variables
-	_generic_package_initialization
+        # set required variables
+        _generic_package_initialization
 
-	mkdir -p "$pkgdir/usr/src/linux-$_kernver"
-	cp -aL Documentation "$pkgdir/usr/src/linux-$_kernver/"
+        mkdir -p "$pkgdir/usr/src/linux-$_kernver"
+        cp -a Documentation "$pkgdir/usr/src/linux-$_kernver/"
 }
 
 # vim: set fenc=utf-8 ts=2 sw=2 noet:
+
diff --git a/config.saved.x86_64 b/config.saved.x86_64
index 198fda5..d2fca51 100644
--- a/config.saved.x86_64
+++ b/config.saved.x86_64
@@ -1,6 +1,6 @@
 #
 # Automatically generated file; DO NOT EDIT.
-# Linux/x86 4.7.5 Kernel Configuration
+# Linux/x86 4.10.8 Kernel Configuration
 #
 CONFIG_64BIT=y
 CONFIG_X86_64=y
@@ -39,7 +39,6 @@ CONFIG_ARCH_SUPPORTS_OPTIMIZED_INLINING=y
 CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y
 CONFIG_HAVE_INTEL_TXT=y
 CONFIG_X86_64_SMP=y
-CONFIG_ARCH_HWEIGHT_CFLAGS="-fcall-saved-rdi -fcall-saved-rsi -fcall-saved-rdx -fcall-saved-rcx -fcall-saved-r8 -fcall-saved-r9 -fcall-saved-r10 -fcall-saved-r11"
 CONFIG_ARCH_SUPPORTS_UPROBES=y
 CONFIG_FIX_EARLYCON_MEM=y
 CONFIG_DEBUG_RODATA=y
@@ -47,11 +46,11 @@ CONFIG_PGTABLE_LEVELS=4
 CONFIG_DEFCONFIG_LIST="/lib/modules/$UNAME_RELEASE/.config"
 CONFIG_IRQ_WORK=y
 CONFIG_BUILDTIME_EXTABLE_SORT=y
+CONFIG_THREAD_INFO_IN_TASK=y
 
 #
 # General setup
 #
-CONFIG_BLD=y
 CONFIG_INIT_ENV_ARG_LIMIT=32
 CONFIG_CROSS_COMPILE=""
 # CONFIG_COMPILE_TEST is not set
@@ -92,6 +91,7 @@ CONFIG_IRQ_DOMAIN_HIERARCHY=y
 CONFIG_GENERIC_MSI_IRQ=y
 CONFIG_GENERIC_MSI_IRQ_DOMAIN=y
 CONFIG_IRQ_FORCED_THREADING=y
+# CONFIG_FORCE_IRQ_THREADING is not set
 CONFIG_SPARSE_IRQ=y
 CONFIG_CLOCKSOURCE_WATCHDOG=y
 CONFIG_ARCH_CLOCKSOURCE_DATA=y
@@ -121,20 +121,22 @@ CONFIG_TICK_CPU_ACCOUNTING=y
 # CONFIG_IRQ_TIME_ACCOUNTING is not set
 CONFIG_BSD_PROCESS_ACCT=y
 CONFIG_BSD_PROCESS_ACCT_V3=y
-CONFIG_TASKSTATS=y
-CONFIG_TASK_DELAY_ACCT=y
-CONFIG_TASK_XACCT=y
-CONFIG_TASK_IO_ACCOUNTING=y
+# CONFIG_TASKSTATS is not set
 
 #
 # RCU Subsystem
 #
 CONFIG_TREE_RCU=y
-# CONFIG_RCU_EXPERT is not set
+CONFIG_RCU_EXPERT=y
 CONFIG_SRCU=y
 # CONFIG_TASKS_RCU is not set
 CONFIG_RCU_STALL_COMMON=y
+CONFIG_RCU_FANOUT=64
+CONFIG_RCU_FANOUT_LEAF=16
+CONFIG_RCU_FAST_NO_HZ=y
 # CONFIG_TREE_RCU_TRACE is not set
+CONFIG_RCU_KTHREAD_PRIO=0
+# CONFIG_RCU_NOCB_CPU is not set
 # CONFIG_RCU_EXPEDITE_BOOT is not set
 CONFIG_BUILD_BIN2C=y
 CONFIG_IKCONFIG=y
@@ -165,7 +167,9 @@ CONFIG_CPUSETS=y
 CONFIG_CGROUP_DEVICE=y
 CONFIG_CGROUP_CPUACCT=y
 # CONFIG_CGROUP_PERF is not set
+CONFIG_CGROUP_BPF=y
 # CONFIG_CGROUP_DEBUG is not set
+CONFIG_SOCK_CGROUP_DATA=y
 CONFIG_CHECKPOINT_RESTORE=y
 CONFIG_NAMESPACES=y
 CONFIG_UTS_NS=y
@@ -184,6 +188,7 @@ CONFIG_RD_GZIP=y
 # CONFIG_RD_XZ is not set
 # CONFIG_RD_LZO is not set
 CONFIG_RD_LZ4=y
+CONFIG_INITRAMFS_COMPRESSION=".gz"
 CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE=y
 # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set
 CONFIG_SYSCTL=y
@@ -196,6 +201,7 @@ CONFIG_MULTIUSER=y
 # CONFIG_SGETMASK_SYSCALL is not set
 # CONFIG_SYSFS_SYSCALL is not set
 # CONFIG_SYSCTL_SYSCALL is not set
+CONFIG_POSIX_TIMERS=y
 # CONFIG_KALLSYMS is not set
 CONFIG_PRINTK=y
 CONFIG_PRINTK_NMI=y
@@ -228,6 +234,7 @@ CONFIG_VM_EVENT_COUNTERS=y
 # CONFIG_SLAB is not set
 CONFIG_SLUB=y
 # CONFIG_SLOB is not set
+CONFIG_SLAB_FREELIST_RANDOM=y
 CONFIG_SLUB_CPU_PARTIAL=y
 # CONFIG_SYSTEM_DATA_VERIFICATION is not set
 # CONFIG_PROFILING is not set
@@ -265,11 +272,14 @@ CONFIG_HAVE_CMPXCHG_LOCAL=y
 CONFIG_HAVE_CMPXCHG_DOUBLE=y
 CONFIG_HAVE_ARCH_SECCOMP_FILTER=y
 CONFIG_SECCOMP_FILTER=y
+CONFIG_HAVE_GCC_PLUGINS=y
+# CONFIG_GCC_PLUGINS is not set
 CONFIG_HAVE_CC_STACKPROTECTOR=y
 CONFIG_CC_STACKPROTECTOR=y
 # CONFIG_CC_STACKPROTECTOR_NONE is not set
 # CONFIG_CC_STACKPROTECTOR_REGULAR is not set
 CONFIG_CC_STACKPROTECTOR_STRONG=y
+CONFIG_HAVE_ARCH_WITHIN_STACK_FRAMES=y
 CONFIG_HAVE_CONTEXT_TRACKING=y
 CONFIG_HAVE_VIRT_CPU_ACCOUNTING_GEN=y
 CONFIG_HAVE_IRQ_TIME_ACCOUNTING=y
@@ -287,6 +297,8 @@ CONFIG_HAVE_STACK_VALIDATION=y
 # CONFIG_HAVE_ARCH_HASH is not set
 # CONFIG_ISA_BUS_API is not set
 # CONFIG_CPU_NO_EFFICIENT_FFS is not set
+CONFIG_HAVE_ARCH_VMAP_STACK=y
+CONFIG_VMAP_STACK=y
 
 #
 # GCOV-based kernel profiling
@@ -301,8 +313,12 @@ CONFIG_BLOCK=y
 CONFIG_BLK_DEV_BSG=y
 CONFIG_BLK_DEV_BSGLIB=y
 CONFIG_BLK_DEV_INTEGRITY=y
+CONFIG_BLK_DEV_ZONED=y
 CONFIG_BLK_DEV_THROTTLING=y
 # CONFIG_BLK_CMDLINE_PARSER is not set
+CONFIG_BLK_WBT=y
+CONFIG_BLK_WBT_SQ=y
+CONFIG_BLK_WBT_MQ=y
 
 #
 # Partition Types
@@ -327,18 +343,16 @@ CONFIG_MSDOS_PARTITION=y
 CONFIG_EFI_PARTITION=y
 # CONFIG_SYSV68_PARTITION is not set
 # CONFIG_CMDLINE_PARTITION is not set
+CONFIG_BLK_MQ_PCI=y
 
 #
 # IO Schedulers
 #
 CONFIG_IOSCHED_NOOP=y
-CONFIG_IOSCHED_DEADLINE=y
-CONFIG_IOSCHED_CFQ=y
-CONFIG_CFQ_GROUP_IOSCHED=y
+# CONFIG_IOSCHED_DEADLINE is not set
+# CONFIG_IOSCHED_CFQ is not set
 CONFIG_IOSCHED_BFQ=y
 CONFIG_BFQ_GROUP_IOSCHED=y
-# CONFIG_DEFAULT_DEADLINE is not set
-# CONFIG_DEFAULT_CFQ is not set
 CONFIG_DEFAULT_BFQ=y
 # CONFIG_DEFAULT_NOOP is not set
 CONFIG_DEFAULT_IOSCHED="bfq"
@@ -367,8 +381,9 @@ CONFIG_SMP=y
 CONFIG_X86_FEATURE_NAMES=y
 CONFIG_X86_FAST_FEATURE_TESTS=y
 CONFIG_X86_X2APIC=y
-CONFIG_X86_MPPARSE=y
+# CONFIG_X86_MPPARSE is not set
 # CONFIG_GOLDFISH is not set
+CONFIG_INTEL_RDT_A=y
 # CONFIG_X86_EXTENDED_PLATFORM is not set
 CONFIG_X86_INTEL_LPSS=y
 # CONFIG_X86_AMD_PLATFORM_DEVICE is not set
@@ -382,10 +397,12 @@ CONFIG_NO_BOOTMEM=y
 # CONFIG_MK10 is not set
 # CONFIG_MBARCELONA is not set
 # CONFIG_MBOBCAT is not set
+# CONFIG_MJAGUAR is not set
 # CONFIG_MBULLDOZER is not set
 # CONFIG_MPILEDRIVER is not set
 # CONFIG_MSTEAMROLLER is not set
-# CONFIG_MJAGUAR is not set
+# CONFIG_MEXCAVATOR is not set
+# CONFIG_MZEN is not set
 # CONFIG_MPSC is not set
 # CONFIG_MATOM is not set
 # CONFIG_MCORE2 is not set
@@ -403,6 +420,7 @@ CONFIG_X86_INTERNODE_CACHE_SHIFT=6
 CONFIG_X86_L1_CACHE_SHIFT=6
 CONFIG_X86_INTEL_USERCOPY=y
 CONFIG_X86_USE_PPRO_CHECKSUM=y
+CONFIG_X86_P6_NOP=y
 CONFIG_X86_TSC=y
 CONFIG_X86_CMPXCHG64=y
 CONFIG_X86_CMOV=y
@@ -418,10 +436,11 @@ CONFIG_DMI=y
 # CONFIG_CALGARY_IOMMU is not set
 CONFIG_SWIOTLB=y
 CONFIG_IOMMU_HELPER=y
-# CONFIG_MAXSMP is not set
-CONFIG_NR_CPUS=16
+CONFIG_MAXSMP=y
+CONFIG_NR_CPUS=8192
 CONFIG_SCHED_SMT=y
 CONFIG_SCHED_MC=y
+CONFIG_SCHED_MC_PRIO=y
 CONFIG_PREEMPT_NONE=y
 # CONFIG_PREEMPT_VOLUNTARY is not set
 # CONFIG_PREEMPT is not set
@@ -430,7 +449,6 @@ CONFIG_X86_IO_APIC=y
 CONFIG_X86_REROUTE_FOR_BROKEN_BOOT_IRQS=y
 CONFIG_X86_MCE=y
 CONFIG_X86_MCE_INTEL=y
-# CONFIG_X86_MCE_AMD is not set
 CONFIG_X86_MCE_THRESHOLD=y
 # CONFIG_X86_MCE_INJECT is not set
 CONFIG_X86_THERMAL_VECTOR=y
@@ -492,6 +510,7 @@ CONFIG_HWPOISON_INJECT=y
 CONFIG_TRANSPARENT_HUGEPAGE=y
 CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS=y
 # CONFIG_TRANSPARENT_HUGEPAGE_MADVISE is not set
+CONFIG_TRANSPARENT_HUGE_PAGECACHE=y
 CONFIG_CLEANCACHE=y
 CONFIG_FRONTSWAP=y
 # CONFIG_CMA is not set
@@ -540,6 +559,8 @@ CONFIG_RELOCATABLE=y
 CONFIG_RANDOMIZE_BASE=y
 CONFIG_X86_NEED_RELOCS=y
 CONFIG_PHYSICAL_ALIGN=0x1000000
+CONFIG_RANDOMIZE_MEMORY=y
+CONFIG_RANDOMIZE_MEMORY_PHYSICAL_PADDING=0x0
 CONFIG_HOTPLUG_CPU=y
 # CONFIG_BOOTPARAM_HOTPLUG_CPU0 is not set
 # CONFIG_DEBUG_HOTPLUG_CPU0 is not set
@@ -583,19 +604,23 @@ CONFIG_ACPI_BUTTON=y
 CONFIG_ACPI_FAN=y
 CONFIG_ACPI_DOCK=y
 CONFIG_ACPI_CPU_FREQ_PSS=y
+CONFIG_ACPI_PROCESSOR_CSTATE=y
 CONFIG_ACPI_PROCESSOR_IDLE=y
+CONFIG_ACPI_CPPC_LIB=y
 CONFIG_ACPI_PROCESSOR=y
+CONFIG_ACPI_IPMI=y
 CONFIG_ACPI_HOTPLUG_CPU=y
 CONFIG_ACPI_PROCESSOR_AGGREGATOR=y
 CONFIG_ACPI_THERMAL=y
 # CONFIG_ACPI_CUSTOM_DSDT is not set
+CONFIG_ARCH_HAS_ACPI_TABLE_UPGRADE=y
 CONFIG_ACPI_TABLE_UPGRADE=y
 # CONFIG_ACPI_DEBUG is not set
 CONFIG_ACPI_PCI_SLOT=y
 CONFIG_X86_PM_TIMER=y
 CONFIG_ACPI_CONTAINER=y
 CONFIG_ACPI_HOTPLUG_IOAPIC=y
-CONFIG_ACPI_SBS=y
+# CONFIG_ACPI_SBS is not set
 CONFIG_ACPI_HED=y
 CONFIG_ACPI_BGRT=y
 CONFIG_ACPI_REDUCED_HARDWARE_ONLY=y
@@ -606,9 +631,11 @@ CONFIG_ACPI_APEI=y
 CONFIG_ACPI_APEI_GHES=y
 CONFIG_ACPI_APEI_PCIEAER=y
 CONFIG_ACPI_APEI_MEMORY_FAILURE=y
-# CONFIG_ACPI_APEI_ERST_DEBUG is not set
+CONFIG_ACPI_APEI_ERST_DEBUG=y
+# CONFIG_DPTF_POWER is not set
 CONFIG_ACPI_EXTLOG=y
-CONFIG_PMIC_OPREGION=y
+# CONFIG_PMIC_OPREGION is not set
+CONFIG_ACPI_CONFIGFS=y
 # CONFIG_SFI is not set
 
 #
@@ -635,9 +662,8 @@ CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y
 # CPU frequency scaling drivers
 #
 CONFIG_X86_INTEL_PSTATE=y
-CONFIG_X86_PCC_CPUFREQ=y
-CONFIG_X86_ACPI_CPUFREQ=y
-# CONFIG_X86_POWERNOW_K8 is not set
+# CONFIG_X86_PCC_CPUFREQ is not set
+# CONFIG_X86_ACPI_CPUFREQ is not set
 # CONFIG_X86_SPEEDSTEP_CENTRINO is not set
 # CONFIG_X86_P4_CLOCKMOD is not set
 
@@ -655,12 +681,6 @@ CONFIG_CPU_IDLE_GOV_MENU=y
 # CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED is not set
 CONFIG_INTEL_IDLE=y
 
-#
-# Memory power savings
-#
-CONFIG_I7300_IDLE_IOAT_CHANNEL=y
-CONFIG_I7300_IDLE=y
-
 #
 # Bus options (PCI etc.)
 #
@@ -676,18 +696,19 @@ CONFIG_PCIE_ECRC=y
 CONFIG_PCIEAER_INJECT=y
 CONFIG_PCIEASPM=y
 # CONFIG_PCIEASPM_DEBUG is not set
-# CONFIG_PCIEASPM_DEFAULT is not set
-CONFIG_PCIEASPM_POWERSAVE=y
+CONFIG_PCIEASPM_DEFAULT=y
+# CONFIG_PCIEASPM_POWERSAVE is not set
 # CONFIG_PCIEASPM_PERFORMANCE is not set
 CONFIG_PCIE_PME=y
 # CONFIG_PCIE_DPC is not set
+# CONFIG_PCIE_PTM is not set
 CONFIG_PCI_BUS_ADDR_T_64BIT=y
 CONFIG_PCI_MSI=y
 CONFIG_PCI_MSI_IRQ_DOMAIN=y
 # CONFIG_PCI_DEBUG is not set
 CONFIG_PCI_REALLOC_ENABLE_AUTO=y
-CONFIG_PCI_STUB=y
-CONFIG_HT_IRQ=y
+# CONFIG_PCI_STUB is not set
+# CONFIG_HT_IRQ is not set
 CONFIG_PCI_ATS=y
 CONFIG_PCI_IOV=y
 CONFIG_PCI_PRI=y
@@ -697,12 +718,13 @@ CONFIG_HOTPLUG_PCI=y
 CONFIG_HOTPLUG_PCI_ACPI=y
 # CONFIG_HOTPLUG_PCI_ACPI_IBM is not set
 # CONFIG_HOTPLUG_PCI_CPCI is not set
-# CONFIG_HOTPLUG_PCI_SHPC is not set
+CONFIG_HOTPLUG_PCI_SHPC=y
 
 #
 # PCI host controller drivers
 #
 # CONFIG_PCIE_DW_PLAT is not set
+# CONFIG_VMD is not set
 # CONFIG_ISA_BUS is not set
 CONFIG_ISA_DMA_API=y
 # CONFIG_PCCARD is not set
@@ -722,7 +744,6 @@ CONFIG_BINFMT_MISC=y
 # CONFIG_X86_X32 is not set
 CONFIG_X86_DEV_DMA_OPS=y
 CONFIG_PMC_ATOM=y
-# CONFIG_VMD is not set
 CONFIG_NET=y
 CONFIG_NET_INGRESS=y
 CONFIG_NET_EGRESS=y
@@ -781,6 +802,7 @@ CONFIG_TCP_CONG_CUBIC=y
 # CONFIG_TCP_CONG_HSTCP is not set
 # CONFIG_TCP_CONG_HYBLA is not set
 # CONFIG_TCP_CONG_VEGAS is not set
+# CONFIG_TCP_CONG_NV is not set
 # CONFIG_TCP_CONG_SCALABLE is not set
 CONFIG_TCP_CONG_LP=y
 # CONFIG_TCP_CONG_VENO is not set
@@ -788,6 +810,7 @@ CONFIG_TCP_CONG_LP=y
 # CONFIG_TCP_CONG_ILLINOIS is not set
 # CONFIG_TCP_CONG_DCTCP is not set
 # CONFIG_TCP_CONG_CDG is not set
+# CONFIG_TCP_CONG_BBR is not set
 CONFIG_DEFAULT_CUBIC=y
 # CONFIG_DEFAULT_RENO is not set
 CONFIG_DEFAULT_TCP_CONG="cubic"
@@ -800,7 +823,7 @@ CONFIG_INET6_AH=y
 CONFIG_INET6_ESP=y
 CONFIG_INET6_IPCOMP=y
 CONFIG_IPV6_MIP6=y
-# CONFIG_IPV6_ILA is not set
+CONFIG_IPV6_ILA=y
 CONFIG_INET6_XFRM_TUNNEL=y
 CONFIG_INET6_TUNNEL=y
 CONFIG_INET6_XFRM_MODE_TRANSPORT=y
@@ -817,8 +840,12 @@ CONFIG_IPV6_SUBTREES=y
 CONFIG_IPV6_MROUTE=y
 CONFIG_IPV6_MROUTE_MULTIPLE_TABLES=y
 CONFIG_IPV6_PIMSM_V2=y
+CONFIG_IPV6_SEG6_LWTUNNEL=y
+# CONFIG_IPV6_SEG6_INLINE is not set
+# CONFIG_IPV6_SEG6_HMAC is not set
+# CONFIG_NETLABEL is not set
 # CONFIG_NETWORK_SECMARK is not set
-CONFIG_NET_PTP_CLASSIFY=y
+# CONFIG_NET_PTP_CLASSIFY is not set
 # CONFIG_NETWORK_PHY_TIMESTAMPING is not set
 CONFIG_NETFILTER=y
 # CONFIG_NETFILTER_DEBUG is not set
@@ -835,6 +862,7 @@ CONFIG_NETFILTER_NETLINK=y
 # CONFIG_NETFILTER_NETLINK_LOG is not set
 # CONFIG_NF_CONNTRACK is not set
 CONFIG_NF_LOG_COMMON=y
+# CONFIG_NF_LOG_NETDEV is not set
 # CONFIG_NF_TABLES is not set
 CONFIG_NETFILTER_XTABLES=y
 
@@ -853,6 +881,7 @@ CONFIG_NETFILTER_XT_TARGET_CHECKSUM=y
 CONFIG_NETFILTER_XT_TARGET_HL=y
 # CONFIG_NETFILTER_XT_TARGET_HMARK is not set
 # CONFIG_NETFILTER_XT_TARGET_IDLETIMER is not set
+# CONFIG_NETFILTER_XT_TARGET_LED is not set
 CONFIG_NETFILTER_XT_TARGET_LOG=y
 # CONFIG_NETFILTER_XT_TARGET_MARK is not set
 # CONFIG_NETFILTER_XT_TARGET_NFLOG is not set
@@ -897,7 +926,6 @@ CONFIG_NETFILTER_XT_MATCH_RATEEST=y
 # CONFIG_NETFILTER_XT_MATCH_REALM is not set
 CONFIG_NETFILTER_XT_MATCH_RECENT=y
 # CONFIG_NETFILTER_XT_MATCH_SCTP is not set
-CONFIG_NETFILTER_XT_MATCH_SOCKET=y
 # CONFIG_NETFILTER_XT_MATCH_STATISTIC is not set
 CONFIG_NETFILTER_XT_MATCH_STRING=y
 CONFIG_NETFILTER_XT_MATCH_TCPMSS=y
@@ -913,6 +941,7 @@ CONFIG_IP_SET_HASH_IPMARK=y
 CONFIG_IP_SET_HASH_IPPORT=y
 CONFIG_IP_SET_HASH_IPPORTIP=y
 CONFIG_IP_SET_HASH_IPPORTNET=y
+CONFIG_IP_SET_HASH_IPMAC=y
 CONFIG_IP_SET_HASH_MAC=y
 CONFIG_IP_SET_HASH_NETPORTNET=y
 CONFIG_IP_SET_HASH_NET=y
@@ -925,7 +954,8 @@ CONFIG_IP_SET_LIST_SET=y
 #
 # IP: Netfilter Configuration
 #
-CONFIG_NF_DEFRAG_IPV4=y
+# CONFIG_NF_DEFRAG_IPV4 is not set
+# CONFIG_NF_SOCKET_IPV4 is not set
 # CONFIG_NF_DUP_IPV4 is not set
 CONFIG_NF_LOG_ARP=y
 CONFIG_NF_LOG_IPV4=y
@@ -941,12 +971,14 @@ CONFIG_IP_NF_MANGLE=y
 # CONFIG_IP_NF_TARGET_ECN is not set
 CONFIG_IP_NF_TARGET_TTL=y
 # CONFIG_IP_NF_RAW is not set
+# CONFIG_IP_NF_SECURITY is not set
 # CONFIG_IP_NF_ARPTABLES is not set
 
 #
 # IPv6: Netfilter Configuration
 #
-CONFIG_NF_DEFRAG_IPV6=y
+# CONFIG_NF_DEFRAG_IPV6 is not set
+# CONFIG_NF_SOCKET_IPV6 is not set
 # CONFIG_NF_DUP_IPV6 is not set
 CONFIG_NF_REJECT_IPV6=y
 CONFIG_NF_LOG_IPV6=y
@@ -965,15 +997,10 @@ CONFIG_IP6_NF_FILTER=y
 CONFIG_IP6_NF_TARGET_REJECT=y
 CONFIG_IP6_NF_MANGLE=y
 # CONFIG_IP6_NF_RAW is not set
+# CONFIG_IP6_NF_SECURITY is not set
 # CONFIG_BRIDGE_NF_EBTABLES is not set
 # CONFIG_IP_DCCP is not set
-CONFIG_IP_SCTP=y
-# CONFIG_SCTP_DBG_OBJCNT is not set
-CONFIG_SCTP_DEFAULT_COOKIE_HMAC_MD5=y
-# CONFIG_SCTP_DEFAULT_COOKIE_HMAC_SHA1 is not set
-# CONFIG_SCTP_DEFAULT_COOKIE_HMAC_NONE is not set
-CONFIG_SCTP_COOKIE_HMAC_MD5=y
-# CONFIG_SCTP_COOKIE_HMAC_SHA1 is not set
+# CONFIG_IP_SCTP is not set
 # CONFIG_RDS is not set
 # CONFIG_TIPC is not set
 # CONFIG_ATM is not set
@@ -985,7 +1012,6 @@ CONFIG_BRIDGE=y
 CONFIG_BRIDGE_IGMP_SNOOPING=y
 CONFIG_BRIDGE_VLAN_FILTERING=y
 CONFIG_HAVE_NET_DSA=y
-# CONFIG_NET_DSA is not set
 CONFIG_VLAN_8021Q=y
 CONFIG_VLAN_8021Q_GVRP=y
 CONFIG_VLAN_8021Q_MVRP=y
@@ -1046,6 +1072,7 @@ CONFIG_NET_CLS_FLOW=y
 CONFIG_NET_CLS_CGROUP=y
 CONFIG_NET_CLS_BPF=y
 CONFIG_NET_CLS_FLOWER=y
+# CONFIG_NET_CLS_MATCHALL is not set
 CONFIG_NET_EMATCH=y
 CONFIG_NET_EMATCH_STACK=32
 CONFIG_NET_EMATCH_CMP=y
@@ -1067,23 +1094,25 @@ CONFIG_NET_ACT_SKBEDIT=y
 CONFIG_NET_ACT_CSUM=y
 CONFIG_NET_ACT_VLAN=y
 CONFIG_NET_ACT_BPF=y
+# CONFIG_NET_ACT_SKBMOD is not set
 # CONFIG_NET_ACT_IFE is not set
+# CONFIG_NET_ACT_TUNNEL_KEY is not set
 CONFIG_NET_CLS_IND=y
 CONFIG_NET_SCH_FIFO=y
 # CONFIG_DCB is not set
 CONFIG_DNS_RESOLVER=y
 # CONFIG_BATMAN_ADV is not set
 # CONFIG_OPENVSWITCH is not set
-CONFIG_VSOCKETS=y
+# CONFIG_VSOCKETS is not set
 CONFIG_NETLINK_DIAG=y
 # CONFIG_MPLS is not set
 # CONFIG_HSR is not set
-CONFIG_NET_SWITCHDEV=y
-# CONFIG_NET_L3_MASTER_DEV is not set
+# CONFIG_NET_SWITCHDEV is not set
+CONFIG_NET_L3_MASTER_DEV=y
+# CONFIG_NET_NCSI is not set
 CONFIG_RPS=y
 CONFIG_RFS_ACCEL=y
 CONFIG_XPS=y
-CONFIG_SOCK_CGROUP_DATA=y
 CONFIG_CGROUP_NET_PRIO=y
 CONFIG_CGROUP_NET_CLASSID=y
 CONFIG_NET_RX_BUSY_POLL=y
@@ -1100,6 +1129,7 @@ CONFIG_NET_FLOW_LIMIT=y
 # CONFIG_BT is not set
 # CONFIG_AF_RXRPC is not set
 CONFIG_AF_KCM=y
+CONFIG_STREAM_PARSER=y
 CONFIG_FIB_RULES=y
 # CONFIG_WIRELESS is not set
 # CONFIG_WIMAX is not set
@@ -1109,6 +1139,7 @@ CONFIG_FIB_RULES=y
 # CONFIG_CEPH_LIB is not set
 # CONFIG_NFC is not set
 CONFIG_LWTUNNEL=y
+CONFIG_LWTUNNEL_BPF=y
 CONFIG_DST_CACHE=y
 CONFIG_NET_DEVLINK=y
 CONFIG_MAY_USE_DEVLINK=y
@@ -1133,13 +1164,14 @@ CONFIG_EXTRA_FIRMWARE=""
 # CONFIG_ALLOW_DEV_COREDUMP is not set
 # CONFIG_DEBUG_DRIVER is not set
 # CONFIG_DEBUG_DEVRES is not set
+# CONFIG_DEBUG_TEST_DRIVER_REMOVE is not set
 # CONFIG_SYS_HYPERVISOR is not set
 # CONFIG_GENERIC_CPU_DEVICES is not set
 CONFIG_GENERIC_CPU_AUTOPROBE=y
 CONFIG_REGMAP=y
 CONFIG_REGMAP_I2C=y
 CONFIG_DMA_SHARED_BUFFER=y
-# CONFIG_FENCE_TRACE is not set
+# CONFIG_DMA_FENCE_TRACE is not set
 
 #
 # Bus devices
@@ -1180,6 +1212,8 @@ CONFIG_BLK_DEV_NBD=y
 # CONFIG_BLK_DEV_RBD is not set
 # CONFIG_BLK_DEV_RSXX is not set
 # CONFIG_BLK_DEV_NVME is not set
+# CONFIG_NVME_FC is not set
+# CONFIG_NVME_TARGET is not set
 
 #
 # Misc devices
@@ -1198,12 +1232,10 @@ CONFIG_BLK_DEV_NBD=y
 # CONFIG_ISL29003 is not set
 # CONFIG_ISL29020 is not set
 # CONFIG_SENSORS_TSL2550 is not set
-# CONFIG_SENSORS_BH1780 is not set
 # CONFIG_SENSORS_BH1770 is not set
 # CONFIG_SENSORS_APDS990X is not set
 # CONFIG_HMC6352 is not set
 # CONFIG_DS1682 is not set
-# CONFIG_BMP085_I2C is not set
 # CONFIG_USB_SWITCH_FSA9480 is not set
 # CONFIG_SRAM is not set
 # CONFIG_C2PORT is not set
@@ -1220,7 +1252,6 @@ CONFIG_BLK_DEV_NBD=y
 #
 # Texas Instruments shared transport line discipline
 #
-# CONFIG_TI_ST is not set
 # CONFIG_SENSORS_LIS3_I2C is not set
 
 #
@@ -1269,8 +1300,7 @@ CONFIG_INTEL_MEI_TXE=y
 # CONFIG_GENWQE is not set
 # CONFIG_ECHO is not set
 # CONFIG_CXL_BASE is not set
-# CONFIG_CXL_KERNEL_API is not set
-# CONFIG_CXL_EEH is not set
+# CONFIG_CXL_AFU_DRIVER_OPS is not set
 CONFIG_HAVE_IDE=y
 # CONFIG_IDE is not set
 
@@ -1291,7 +1321,8 @@ CONFIG_SCSI_MQ_DEFAULT=y
 CONFIG_BLK_DEV_SD=y
 # CONFIG_CHR_DEV_ST is not set
 # CONFIG_CHR_DEV_OSST is not set
-# CONFIG_BLK_DEV_SR is not set
+CONFIG_BLK_DEV_SR=y
+# CONFIG_BLK_DEV_SR_VENDOR is not set
 CONFIG_CHR_DEV_SG=y
 # CONFIG_CHR_DEV_SCH is not set
 CONFIG_SCSI_CONSTANTS=y
@@ -1315,7 +1346,7 @@ CONFIG_ATA=y
 CONFIG_ATA_VERBOSE_ERROR=y
 CONFIG_ATA_ACPI=y
 # CONFIG_SATA_ZPODD is not set
-CONFIG_SATA_PMP=y
+# CONFIG_SATA_PMP is not set
 
 #
 # Controllers with non-SFF native interface
@@ -1336,7 +1367,6 @@ CONFIG_MD_RAID10=y
 CONFIG_MD_RAID456=y
 # CONFIG_MD_MULTIPATH is not set
 # CONFIG_MD_FAULTY is not set
-CONFIG_MD_CLUSTER=y
 CONFIG_BCACHE=y
 # CONFIG_BCACHE_DEBUG is not set
 # CONFIG_BCACHE_CLOSURES_DEBUG is not set
@@ -1345,28 +1375,22 @@ CONFIG_BLK_DEV_DM=y
 CONFIG_DM_MQ_DEFAULT=y
 # CONFIG_DM_DEBUG is not set
 CONFIG_DM_BUFIO=y
-# CONFIG_DM_DEBUG_BLOCK_STACK_TRACING is not set
-CONFIG_DM_BIO_PRISON=y
-CONFIG_DM_PERSISTENT_DATA=y
+# CONFIG_DM_DEBUG_BLOCK_MANAGER_LOCKING is not set
 CONFIG_DM_CRYPT=y
 CONFIG_DM_SNAPSHOT=y
-CONFIG_DM_THIN_PROVISIONING=y
-CONFIG_DM_CACHE=y
-CONFIG_DM_CACHE_SMQ=y
-CONFIG_DM_CACHE_CLEANER=y
-CONFIG_DM_ERA=y
-CONFIG_DM_MIRROR=y
-CONFIG_DM_LOG_USERSPACE=y
+# CONFIG_DM_THIN_PROVISIONING is not set
+# CONFIG_DM_CACHE is not set
+# CONFIG_DM_ERA is not set
+# CONFIG_DM_MIRROR is not set
 CONFIG_DM_RAID=y
-CONFIG_DM_ZERO=y
+# CONFIG_DM_ZERO is not set
 # CONFIG_DM_MULTIPATH is not set
 # CONFIG_DM_DELAY is not set
 CONFIG_DM_UEVENT=y
 # CONFIG_DM_FLAKEY is not set
-CONFIG_DM_VERITY=y
-# CONFIG_DM_VERITY_FEC is not set
-CONFIG_DM_SWITCH=y
-CONFIG_DM_LOG_WRITES=y
+# CONFIG_DM_VERITY is not set
+# CONFIG_DM_SWITCH is not set
+# CONFIG_DM_LOG_WRITES is not set
 # CONFIG_TARGET_CORE is not set
 # CONFIG_FUSION is not set
 
@@ -1385,8 +1409,7 @@ CONFIG_NET_CORE=y
 # CONFIG_NET_FC is not set
 # CONFIG_IFB is not set
 # CONFIG_NET_TEAM is not set
-CONFIG_MACVLAN=y
-CONFIG_MACVTAP=y
+# CONFIG_MACVLAN is not set
 CONFIG_IPVLAN=y
 # CONFIG_VXLAN is not set
 # CONFIG_GENEVE is not set
@@ -1398,7 +1421,8 @@ CONFIG_MACSEC=y
 # CONFIG_TUN is not set
 # CONFIG_TUN_VNET_CROSS_LE is not set
 # CONFIG_VETH is not set
-# CONFIG_NLMON is not set
+CONFIG_NLMON=y
+# CONFIG_NET_VRF is not set
 # CONFIG_ARCNET is not set
 
 #
@@ -1412,8 +1436,10 @@ CONFIG_ETHERNET=y
 # CONFIG_NET_VENDOR_3COM is not set
 # CONFIG_NET_VENDOR_ADAPTEC is not set
 # CONFIG_NET_VENDOR_AGERE is not set
+# CONFIG_NET_VENDOR_ALACRITECH is not set
 # CONFIG_NET_VENDOR_ALTEON is not set
 # CONFIG_ALTERA_TSE is not set
+# CONFIG_NET_VENDOR_AMAZON is not set
 # CONFIG_NET_VENDOR_AMD is not set
 # CONFIG_NET_VENDOR_ARC is not set
 # CONFIG_NET_VENDOR_ATHEROS is not set
@@ -1439,6 +1465,7 @@ CONFIG_E1000E=y
 CONFIG_E1000E_HWTS=y
 CONFIG_IGB=y
 CONFIG_IGB_HWMON=y
+CONFIG_IGB_DCA=y
 # CONFIG_IGBVF is not set
 # CONFIG_IXGB is not set
 # CONFIG_IXGBE is not set
@@ -1472,7 +1499,7 @@ CONFIG_R8169=y
 # CONFIG_NET_VENDOR_SEEQ is not set
 # CONFIG_NET_VENDOR_SILAN is not set
 # CONFIG_NET_VENDOR_SIS is not set
-# CONFIG_SFC is not set
+# CONFIG_NET_VENDOR_SOLARFLARE is not set
 # CONFIG_NET_VENDOR_SMSC is not set
 # CONFIG_NET_VENDOR_STMICRO is not set
 # CONFIG_NET_VENDOR_SUN is not set
@@ -1484,7 +1511,47 @@ CONFIG_R8169=y
 # CONFIG_FDDI is not set
 # CONFIG_HIPPI is not set
 # CONFIG_NET_SB1000 is not set
-# CONFIG_PHYLIB is not set
+CONFIG_PHYLIB=y
+# CONFIG_LED_TRIGGER_PHY is not set
+
+#
+# MDIO bus device drivers
+#
+# CONFIG_MDIO_BCM_UNIMAC is not set
+# CONFIG_MDIO_BITBANG is not set
+# CONFIG_MDIO_OCTEON is not set
+# CONFIG_MDIO_THUNDER is not set
+
+#
+# MII PHY device drivers
+#
+# CONFIG_AMD_PHY is not set
+# CONFIG_AQUANTIA_PHY is not set
+# CONFIG_AT803X_PHY is not set
+# CONFIG_BCM7XXX_PHY is not set
+# CONFIG_BCM87XX_PHY is not set
+# CONFIG_BROADCOM_PHY is not set
+# CONFIG_CICADA_PHY is not set
+# CONFIG_DAVICOM_PHY is not set
+# CONFIG_DP83848_PHY is not set
+# CONFIG_DP83867_PHY is not set
+# CONFIG_FIXED_PHY is not set
+# CONFIG_ICPLUS_PHY is not set
+# CONFIG_INTEL_XWAY_PHY is not set
+# CONFIG_LSI_ET1011C_PHY is not set
+# CONFIG_LXT_PHY is not set
+# CONFIG_MARVELL_PHY is not set
+# CONFIG_MICREL_PHY is not set
+# CONFIG_MICROCHIP_PHY is not set
+# CONFIG_MICROSEMI_PHY is not set
+# CONFIG_NATIONAL_PHY is not set
+# CONFIG_QSEMI_PHY is not set
+# CONFIG_REALTEK_PHY is not set
+# CONFIG_SMSC_PHY is not set
+# CONFIG_STE10XP is not set
+# CONFIG_TERANETICS_PHY is not set
+# CONFIG_VITESSE_PHY is not set
+# CONFIG_XILINX_GMII2RGMII is not set
 # CONFIG_PPP is not set
 # CONFIG_SLIP is not set
 # CONFIG_USB_NET_DRIVERS is not set
@@ -1503,10 +1570,11 @@ CONFIG_R8169=y
 # Input device support
 #
 CONFIG_INPUT=y
+CONFIG_INPUT_LEDS=y
 # CONFIG_INPUT_FF_MEMLESS is not set
 # CONFIG_INPUT_POLLDEV is not set
 # CONFIG_INPUT_SPARSEKMAP is not set
-# CONFIG_INPUT_MATRIXKMAP is not set
+CONFIG_INPUT_MATRIXKMAP=y
 
 #
 # Userland interfaces
@@ -1529,11 +1597,9 @@ CONFIG_KEYBOARD_ATKBD=y
 # CONFIG_KEYBOARD_QT1070 is not set
 # CONFIG_KEYBOARD_QT2160 is not set
 # CONFIG_KEYBOARD_LKKBD is not set
-# CONFIG_KEYBOARD_GPIO is not set
-# CONFIG_KEYBOARD_GPIO_POLLED is not set
 # CONFIG_KEYBOARD_TCA6416 is not set
 # CONFIG_KEYBOARD_TCA8418 is not set
-# CONFIG_KEYBOARD_MATRIX is not set
+# CONFIG_KEYBOARD_LM8323 is not set
 # CONFIG_KEYBOARD_LM8333 is not set
 # CONFIG_KEYBOARD_MAX7359 is not set
 # CONFIG_KEYBOARD_MCS is not set
@@ -1544,28 +1610,7 @@ CONFIG_KEYBOARD_ATKBD=y
 # CONFIG_KEYBOARD_STOWAWAY is not set
 # CONFIG_KEYBOARD_SUNKBD is not set
 # CONFIG_KEYBOARD_XTKBD is not set
-CONFIG_INPUT_MOUSE=y
-CONFIG_MOUSE_PS2=y
-# CONFIG_MOUSE_PS2_ALPS is not set
-# CONFIG_MOUSE_PS2_BYD is not set
-# CONFIG_MOUSE_PS2_LOGIPS2PP is not set
-# CONFIG_MOUSE_PS2_SYNAPTICS is not set
-# CONFIG_MOUSE_PS2_CYPRESS is not set
-# CONFIG_MOUSE_PS2_LIFEBOOK is not set
-# CONFIG_MOUSE_PS2_TRACKPOINT is not set
-# CONFIG_MOUSE_PS2_ELANTECH is not set
-# CONFIG_MOUSE_PS2_SENTELIC is not set
-# CONFIG_MOUSE_PS2_TOUCHKIT is not set
-# CONFIG_MOUSE_PS2_FOCALTECH is not set
-# CONFIG_MOUSE_SERIAL is not set
-# CONFIG_MOUSE_APPLETOUCH is not set
-# CONFIG_MOUSE_BCM5974 is not set
-# CONFIG_MOUSE_CYAPA is not set
-# CONFIG_MOUSE_ELAN_I2C is not set
-# CONFIG_MOUSE_VSXXXAA is not set
-# CONFIG_MOUSE_GPIO is not set
-# CONFIG_MOUSE_SYNAPTICS_I2C is not set
-# CONFIG_MOUSE_SYNAPTICS_USB is not set
+# CONFIG_INPUT_MOUSE is not set
 # CONFIG_INPUT_JOYSTICK is not set
 # CONFIG_INPUT_TABLET is not set
 # CONFIG_INPUT_TOUCHSCREEN is not set
@@ -1576,9 +1621,7 @@ CONFIG_INPUT_MISC=y
 CONFIG_INPUT_PCSPKR=y
 # CONFIG_INPUT_MMA8450 is not set
 # CONFIG_INPUT_MPU3050 is not set
-# CONFIG_INPUT_GP2A is not set
-# CONFIG_INPUT_GPIO_BEEPER is not set
-# CONFIG_INPUT_GPIO_TILT_POLLED is not set
+# CONFIG_INPUT_APANEL is not set
 # CONFIG_INPUT_ATLAS_BTNS is not set
 # CONFIG_INPUT_ATI_REMOTE2 is not set
 # CONFIG_INPUT_KEYSPAN_REMOTE is not set
@@ -1588,11 +1631,10 @@ CONFIG_INPUT_PCSPKR=y
 # CONFIG_INPUT_CM109 is not set
 CONFIG_INPUT_UINPUT=y
 # CONFIG_INPUT_PCF8574 is not set
-# CONFIG_INPUT_GPIO_ROTARY_ENCODER is not set
 # CONFIG_INPUT_ADXL34X is not set
+# CONFIG_INPUT_IMS_PCU is not set
 # CONFIG_INPUT_CMA3000 is not set
 # CONFIG_INPUT_IDEAPAD_SLIDEBAR is not set
-# CONFIG_INPUT_DRV260X_HAPTICS is not set
 # CONFIG_INPUT_DRV2665_HAPTICS is not set
 # CONFIG_INPUT_DRV2667_HAPTICS is not set
 # CONFIG_RMI4_CORE is not set
@@ -1642,6 +1684,7 @@ CONFIG_SERIAL_8250_DEPRECATED_OPTIONS=y
 CONFIG_SERIAL_8250_PNP=y
 # CONFIG_SERIAL_8250_FINTEK is not set
 CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_DMA=y
 CONFIG_SERIAL_8250_PCI=y
 CONFIG_SERIAL_8250_NR_UARTS=4
 CONFIG_SERIAL_8250_RUNTIME_UARTS=4
@@ -1649,6 +1692,7 @@ CONFIG_SERIAL_8250_RUNTIME_UARTS=4
 # CONFIG_SERIAL_8250_FSL is not set
 # CONFIG_SERIAL_8250_DW is not set
 # CONFIG_SERIAL_8250_RT288X is not set
+# CONFIG_SERIAL_8250_LPSS is not set
 # CONFIG_SERIAL_8250_MID is not set
 # CONFIG_SERIAL_8250_MOXA is not set
 
@@ -1671,11 +1715,15 @@ CONFIG_IPMI_HANDLER=y
 CONFIG_IPMI_PANIC_EVENT=y
 CONFIG_IPMI_PANIC_STRING=y
 CONFIG_IPMI_DEVICE_INTERFACE=y
-# CONFIG_IPMI_SI is not set
+CONFIG_IPMI_SI=y
 CONFIG_IPMI_SSIF=y
-CONFIG_IPMI_WATCHDOG=y
+# CONFIG_IPMI_WATCHDOG is not set
 CONFIG_IPMI_POWEROFF=y
-# CONFIG_HW_RANDOM is not set
+CONFIG_HW_RANDOM=y
+# CONFIG_HW_RANDOM_TIMERIOMEM is not set
+CONFIG_HW_RANDOM_INTEL=y
+# CONFIG_HW_RANDOM_AMD is not set
+# CONFIG_HW_RANDOM_VIA is not set
 CONFIG_NVRAM=y
 # CONFIG_R3964 is not set
 # CONFIG_APPLICOM is not set
@@ -1701,6 +1749,7 @@ CONFIG_I2C_COMPAT=y
 CONFIG_I2C_CHARDEV=y
 # CONFIG_I2C_MUX is not set
 CONFIG_I2C_HELPER_AUTO=y
+CONFIG_I2C_SMBUS=y
 CONFIG_I2C_ALGOBIT=y
 
 #
@@ -1734,11 +1783,9 @@ CONFIG_I2C_SCMI=y
 #
 # I2C system bus drivers (mostly embedded / system-on-chip)
 #
-# CONFIG_I2C_CBUS_GPIO is not set
 # CONFIG_I2C_DESIGNWARE_PLATFORM is not set
 # CONFIG_I2C_DESIGNWARE_PCI is not set
 # CONFIG_I2C_EMEV2 is not set
-# CONFIG_I2C_GPIO is not set
 # CONFIG_I2C_OCORES is not set
 # CONFIG_I2C_PCA_PLATFORM is not set
 # CONFIG_I2C_PXA_PCI is not set
@@ -1757,6 +1804,7 @@ CONFIG_I2C_SCMI=y
 #
 # Other I2C/SMBus bus drivers
 #
+# CONFIG_I2C_MLXCPLD is not set
 # CONFIG_I2C_SLAVE is not set
 # CONFIG_I2C_DEBUG_CORE is not set
 # CONFIG_I2C_DEBUG_ALGO is not set
@@ -1770,7 +1818,6 @@ CONFIG_I2C_SCMI=y
 #
 CONFIG_PPS=y
 # CONFIG_PPS_DEBUG is not set
-CONFIG_NTP_PPS=y
 
 #
 # PPS clients support
@@ -1786,7 +1833,7 @@ CONFIG_PPS_CLIENT_LDISC=y
 #
 # PTP clock support
 #
-CONFIG_PTP_1588_CLOCK=y
+# CONFIG_PTP_1588_CLOCK is not set
 
 #
 # Enable PHYLIB and NETWORK_PHY_TIMESTAMPING to see the additional clocks.
@@ -1797,72 +1844,14 @@ CONFIG_PINCTRL=y
 # Pin controllers
 #
 # CONFIG_DEBUG_PINCTRL is not set
-# CONFIG_PINCTRL_AMD is not set
-# CONFIG_PINCTRL_BAYTRAIL is not set
 # CONFIG_PINCTRL_CHERRYVIEW is not set
 # CONFIG_PINCTRL_BROXTON is not set
 # CONFIG_PINCTRL_SUNRISEPOINT is not set
-CONFIG_ARCH_WANT_OPTIONAL_GPIOLIB=y
-CONFIG_GPIOLIB=y
-CONFIG_GPIO_DEVRES=y
-CONFIG_GPIO_ACPI=y
-CONFIG_GPIOLIB_IRQCHIP=y
-# CONFIG_DEBUG_GPIO is not set
-CONFIG_GPIO_SYSFS=y
-CONFIG_GPIO_GENERIC=y
-
-#
-# Memory mapped GPIO drivers
-#
-# CONFIG_GPIO_AMDPT is not set
-# CONFIG_GPIO_DWAPB is not set
-CONFIG_GPIO_GENERIC_PLATFORM=y
-# CONFIG_GPIO_ICH is not set
-CONFIG_GPIO_LYNXPOINT=y
-# CONFIG_GPIO_VX855 is not set
-# CONFIG_GPIO_ZX is not set
-
-#
-# Port-mapped I/O GPIO drivers
-#
-# CONFIG_GPIO_F7188X is not set
-# CONFIG_GPIO_IT87 is not set
-# CONFIG_GPIO_SCH is not set
-# CONFIG_GPIO_SCH311X is not set
-
-#
-# I2C GPIO expanders
-#
-# CONFIG_GPIO_ADP5588 is not set
-# CONFIG_GPIO_MAX7300 is not set
-# CONFIG_GPIO_MAX732X is not set
-# CONFIG_GPIO_PCA953X is not set
-# CONFIG_GPIO_PCF857X is not set
-# CONFIG_GPIO_SX150X is not set
-# CONFIG_GPIO_TPIC2810 is not set
-
-#
-# MFD GPIO expanders
-#
-
-#
-# PCI GPIO expanders
-#
-# CONFIG_GPIO_AMD8111 is not set
-# CONFIG_GPIO_BT8XX is not set
-# CONFIG_GPIO_INTEL_MID is not set
-# CONFIG_GPIO_ML_IOH is not set
-# CONFIG_GPIO_RDC321X is not set
-
-#
-# SPI or I2C GPIO expanders
-#
-# CONFIG_GPIO_MCP23S08 is not set
-
-#
-# USB GPIO expanders
-#
+# CONFIG_GPIOLIB is not set
 # CONFIG_W1 is not set
+# CONFIG_POWER_AVS is not set
+CONFIG_POWER_RESET=y
+# CONFIG_POWER_RESET_RESTART is not set
 CONFIG_POWER_SUPPLY=y
 # CONFIG_POWER_SUPPLY_DEBUG is not set
 # CONFIG_PDA_POWER is not set
@@ -1876,18 +1865,9 @@ CONFIG_POWER_SUPPLY=y
 # CONFIG_BATTERY_MAX17042 is not set
 # CONFIG_CHARGER_MAX8903 is not set
 # CONFIG_CHARGER_LP8727 is not set
-# CONFIG_CHARGER_GPIO is not set
 # CONFIG_CHARGER_BQ2415X is not set
-# CONFIG_CHARGER_BQ24190 is not set
-# CONFIG_CHARGER_BQ24257 is not set
-# CONFIG_CHARGER_BQ24735 is not set
-# CONFIG_CHARGER_BQ25890 is not set
 # CONFIG_CHARGER_SMB347 is not set
 # CONFIG_BATTERY_GAUGE_LTC2941 is not set
-# CONFIG_CHARGER_RT9455 is not set
-CONFIG_POWER_RESET=y
-# CONFIG_POWER_RESET_RESTART is not set
-# CONFIG_POWER_AVS is not set
 CONFIG_HWMON=y
 # CONFIG_HWMON_VID is not set
 # CONFIG_HWMON_DEBUG_CHIP is not set
@@ -1924,11 +1904,11 @@ CONFIG_HWMON=y
 # CONFIG_SENSORS_F71882FG is not set
 # CONFIG_SENSORS_F75375S is not set
 # CONFIG_SENSORS_FSCHMD is not set
+# CONFIG_SENSORS_FTSTEUTATES is not set
 # CONFIG_SENSORS_GL518SM is not set
 # CONFIG_SENSORS_GL520SM is not set
 # CONFIG_SENSORS_G760A is not set
 # CONFIG_SENSORS_G762 is not set
-# CONFIG_SENSORS_GPIO_FAN is not set
 # CONFIG_SENSORS_HIH6130 is not set
 # CONFIG_SENSORS_IBMAEM is not set
 # CONFIG_SENSORS_IBMPEX is not set
@@ -1956,6 +1936,7 @@ CONFIG_SENSORS_JC42=y
 # CONFIG_SENSORS_MAX6697 is not set
 # CONFIG_SENSORS_MAX31790 is not set
 # CONFIG_SENSORS_MCP3021 is not set
+# CONFIG_SENSORS_TC654 is not set
 # CONFIG_SENSORS_LM63 is not set
 # CONFIG_SENSORS_LM73 is not set
 # CONFIG_SENSORS_LM75 is not set
@@ -1980,8 +1961,8 @@ CONFIG_SENSORS_JC42=y
 # CONFIG_SENSORS_NCT7904 is not set
 # CONFIG_SENSORS_PCF8591 is not set
 # CONFIG_PMBUS is not set
-# CONFIG_SENSORS_SHT15 is not set
 # CONFIG_SENSORS_SHT21 is not set
+# CONFIG_SENSORS_SHT3x is not set
 # CONFIG_SENSORS_SHTC1 is not set
 # CONFIG_SENSORS_SIS5595 is not set
 # CONFIG_SENSORS_DME1737 is not set
@@ -2001,10 +1982,12 @@ CONFIG_SENSORS_JC42=y
 # CONFIG_SENSORS_AMC6821 is not set
 # CONFIG_SENSORS_INA209 is not set
 # CONFIG_SENSORS_INA2XX is not set
+# CONFIG_SENSORS_INA3221 is not set
 # CONFIG_SENSORS_TC74 is not set
 # CONFIG_SENSORS_THMC50 is not set
 # CONFIG_SENSORS_TMP102 is not set
 # CONFIG_SENSORS_TMP103 is not set
+# CONFIG_SENSORS_TMP108 is not set
 # CONFIG_SENSORS_TMP401 is not set
 # CONFIG_SENSORS_TMP421 is not set
 # CONFIG_SENSORS_VIA_CPUTEMP is not set
@@ -2020,6 +2003,7 @@ CONFIG_SENSORS_JC42=y
 # CONFIG_SENSORS_W83L786NG is not set
 # CONFIG_SENSORS_W83627HF is not set
 # CONFIG_SENSORS_W83627EHF is not set
+# CONFIG_SENSORS_XGENE is not set
 
 #
 # ACPI drivers
@@ -2059,6 +2043,7 @@ CONFIG_WATCHDOG_SYSFS=y
 # Watchdog Device Drivers
 #
 # CONFIG_SOFT_WATCHDOG is not set
+# CONFIG_WDAT_WDT is not set
 # CONFIG_XILINX_WATCHDOG is not set
 # CONFIG_ZIIRAVE_WATCHDOG is not set
 # CONFIG_CADENCE_WATCHDOG is not set
@@ -2095,9 +2080,8 @@ CONFIG_ITCO_VENDOR_SUPPORT=y
 # CONFIG_W83977F_WDT is not set
 # CONFIG_MACHZ_WDT is not set
 # CONFIG_SBC_EPX_C3_WATCHDOG is not set
-CONFIG_INTEL_MEI_WDT=y
+# CONFIG_INTEL_MEI_WDT is not set
 # CONFIG_NI903X_WDT is not set
-# CONFIG_MEN_A21_WDT is not set
 
 #
 # PCI-based Watchdog Cards
@@ -2109,6 +2093,11 @@ CONFIG_INTEL_MEI_WDT=y
 # USB-based Watchdog Cards
 #
 # CONFIG_USBPCWATCHDOG is not set
+
+#
+# Watchdog Pretimeout Governors
+#
+# CONFIG_WATCHDOG_PRETIMEOUT_GOV is not set
 CONFIG_SSB_POSSIBLE=y
 
 #
@@ -2128,7 +2117,6 @@ CONFIG_BCMA_POSSIBLE=y
 CONFIG_MFD_CORE=y
 # CONFIG_MFD_AS3711 is not set
 # CONFIG_PMIC_ADP5520 is not set
-# CONFIG_MFD_AAT2870_CORE is not set
 # CONFIG_MFD_BCM590XX is not set
 # CONFIG_MFD_AXP20X_I2C is not set
 # CONFIG_MFD_CROS_EC is not set
@@ -2141,11 +2129,9 @@ CONFIG_MFD_CORE=y
 # CONFIG_MFD_DLN2 is not set
 # CONFIG_MFD_MC13XXX_I2C is not set
 # CONFIG_HTC_PASIC3 is not set
-# CONFIG_HTC_I2CPLD is not set
 # CONFIG_MFD_INTEL_QUARK_I2C_GPIO is not set
 CONFIG_LPC_ICH=y
 # CONFIG_LPC_SCH is not set
-# CONFIG_INTEL_SOC_PMIC is not set
 # CONFIG_MFD_INTEL_LPSS_ACPI is not set
 # CONFIG_MFD_INTEL_LPSS_PCI is not set
 # CONFIG_MFD_JANZ_CMODIO is not set
@@ -2170,7 +2156,6 @@ CONFIG_LPC_ICH=y
 # CONFIG_MFD_RT5033 is not set
 # CONFIG_MFD_RTSX_USB is not set
 # CONFIG_MFD_RC5T583 is not set
-# CONFIG_MFD_RN5T618 is not set
 # CONFIG_MFD_SEC_CORE is not set
 # CONFIG_MFD_SI476X_CORE is not set
 # CONFIG_MFD_SM501 is not set
@@ -2183,14 +2168,13 @@ CONFIG_LPC_ICH=y
 # CONFIG_MFD_LP8788 is not set
 # CONFIG_MFD_PALMAS is not set
 # CONFIG_TPS6105X is not set
-# CONFIG_TPS65010 is not set
 # CONFIG_TPS6507X is not set
 # CONFIG_MFD_TPS65086 is not set
 # CONFIG_MFD_TPS65090 is not set
 # CONFIG_MFD_TPS65217 is not set
+# CONFIG_MFD_TI_LP873X is not set
 # CONFIG_MFD_TPS65218 is not set
 # CONFIG_MFD_TPS6586X is not set
-# CONFIG_MFD_TPS65910 is not set
 # CONFIG_MFD_TPS65912_I2C is not set
 # CONFIG_MFD_TPS80031 is not set
 # CONFIG_TWL4030_CORE is not set
@@ -2211,10 +2195,12 @@ CONFIG_LPC_ICH=y
 # Graphics support
 #
 # CONFIG_AGP is not set
-# CONFIG_VGA_ARB is not set
+CONFIG_VGA_ARB=y
+CONFIG_VGA_ARB_MAX_GPUS=16
 # CONFIG_VGA_SWITCHEROO is not set
 CONFIG_DRM=y
 CONFIG_DRM_DP_AUX_CHARDEV=y
+# CONFIG_DRM_DEBUG_MM is not set
 CONFIG_DRM_KMS_HELPER=y
 CONFIG_DRM_KMS_FB_HELPER=y
 CONFIG_DRM_FBDEV_EMULATION=y
@@ -2224,12 +2210,9 @@ CONFIG_DRM_TTM=y
 #
 # I2C encoder or helper chips
 #
-# CONFIG_DRM_I2C_ADV7511 is not set
 # CONFIG_DRM_I2C_CH7006 is not set
 # CONFIG_DRM_I2C_SIL164 is not set
 # CONFIG_DRM_I2C_NXP_TDA998X is not set
-# CONFIG_DRM_TDFX is not set
-# CONFIG_DRM_R128 is not set
 # CONFIG_DRM_RADEON is not set
 # CONFIG_DRM_AMDGPU is not set
 
@@ -2238,9 +2221,6 @@ CONFIG_DRM_TTM=y
 #
 # CONFIG_DRM_NOUVEAU is not set
 # CONFIG_DRM_I915 is not set
-# CONFIG_DRM_MGA is not set
-# CONFIG_DRM_VIA is not set
-# CONFIG_DRM_SAVAGE is not set
 # CONFIG_DRM_VGEM is not set
 # CONFIG_DRM_VMWGFX is not set
 # CONFIG_DRM_GMA500 is not set
@@ -2256,6 +2236,8 @@ CONFIG_DRM_BRIDGE=y
 # Display Interface Bridges
 #
 # CONFIG_DRM_ANALOGIX_ANX78XX is not set
+# CONFIG_DRM_HISI_HIBMC is not set
+# CONFIG_DRM_LEGACY is not set
 
 #
 # Frame buffer Devices
@@ -2310,7 +2292,6 @@ CONFIG_FB_EFI=y
 # CONFIG_FB_S3 is not set
 # CONFIG_FB_SAVAGE is not set
 # CONFIG_FB_SIS is not set
-# CONFIG_FB_VIA is not set
 # CONFIG_FB_NEOMAGIC is not set
 # CONFIG_FB_KYRO is not set
 # CONFIG_FB_3DFX is not set
@@ -2370,8 +2351,8 @@ CONFIG_HID_GENERIC=y
 # CONFIG_HID_BETOP_FF is not set
 # CONFIG_HID_CHERRY is not set
 # CONFIG_HID_CHICONY is not set
+# CONFIG_HID_CORSAIR is not set
 # CONFIG_HID_CMEDIA is not set
-# CONFIG_HID_CP2112 is not set
 # CONFIG_HID_CYPRESS is not set
 # CONFIG_HID_DRAGONRISE is not set
 # CONFIG_HID_EMS_FF is not set
@@ -2381,6 +2362,7 @@ CONFIG_HID_GENERIC=y
 # CONFIG_HID_GEMBIRD is not set
 # CONFIG_HID_GFRM is not set
 # CONFIG_HID_HOLTEK is not set
+# CONFIG_HID_GT683R is not set
 # CONFIG_HID_KEYTOUCH is not set
 # CONFIG_HID_KYE is not set
 # CONFIG_HID_UCLOGIC is not set
@@ -2390,9 +2372,11 @@ CONFIG_HID_GENERIC=y
 # CONFIG_HID_TWINHAN is not set
 # CONFIG_HID_KENSINGTON is not set
 # CONFIG_HID_LCPOWER is not set
+# CONFIG_HID_LED is not set
 # CONFIG_HID_LENOVO is not set
 # CONFIG_HID_LOGITECH is not set
 # CONFIG_HID_MAGICMOUSE is not set
+# CONFIG_HID_MAYFLASH is not set
 # CONFIG_HID_MICROSOFT is not set
 # CONFIG_HID_MONTEREY is not set
 # CONFIG_HID_MULTITOUCH is not set
@@ -2407,6 +2391,7 @@ CONFIG_HID_GENERIC=y
 # CONFIG_HID_ROCCAT is not set
 # CONFIG_HID_SAITEK is not set
 # CONFIG_HID_SAMSUNG is not set
+# CONFIG_HID_SONY is not set
 # CONFIG_HID_SPEEDLINK is not set
 # CONFIG_HID_STEELSERIES is not set
 # CONFIG_HID_SUNPLUS is not set
@@ -2415,12 +2400,16 @@ CONFIG_HID_GENERIC=y
 # CONFIG_HID_SMARTJOYPLUS is not set
 # CONFIG_HID_TIVO is not set
 # CONFIG_HID_TOPSEED is not set
+# CONFIG_HID_THINGM is not set
 # CONFIG_HID_THRUSTMASTER is not set
+# CONFIG_HID_UDRAW_PS3 is not set
 # CONFIG_HID_WACOM is not set
+# CONFIG_HID_WIIMOTE is not set
 # CONFIG_HID_XINMO is not set
 # CONFIG_HID_ZEROPLUS is not set
 # CONFIG_HID_ZYDACRON is not set
 # CONFIG_HID_SENSOR_HUB is not set
+# CONFIG_HID_ALPS is not set
 
 #
 # USB HID support
@@ -2433,6 +2422,11 @@ CONFIG_USB_HIDDEV=y
 # I2C HID support
 #
 # CONFIG_I2C_HID is not set
+
+#
+# Intel ISH HID support
+#
+# CONFIG_INTEL_ISH_HID is not set
 CONFIG_USB_OHCI_LITTLE_ENDIAN=y
 CONFIG_USB_SUPPORT=y
 CONFIG_USB_COMMON=y
@@ -2445,11 +2439,10 @@ CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
 #
 CONFIG_USB_DEFAULT_PERSIST=y
 CONFIG_USB_DYNAMIC_MINORS=y
-CONFIG_USB_OTG=y
-CONFIG_USB_OTG_WHITELIST=y
+# CONFIG_USB_OTG is not set
+# CONFIG_USB_OTG_WHITELIST is not set
 # CONFIG_USB_OTG_BLACKLIST_HUB is not set
-# CONFIG_USB_OTG_FSM is not set
-CONFIG_USB_ULPI_BUS=y
+# CONFIG_USB_LEDS_TRIGGER_USBPORT is not set
 # CONFIG_USB_MON is not set
 # CONFIG_USB_WUSB_CBAF is not set
 
@@ -2460,11 +2453,7 @@ CONFIG_USB_ULPI_BUS=y
 CONFIG_USB_XHCI_HCD=y
 CONFIG_USB_XHCI_PCI=y
 # CONFIG_USB_XHCI_PLATFORM is not set
-CONFIG_USB_EHCI_HCD=y
-CONFIG_USB_EHCI_ROOT_HUB_TT=y
-CONFIG_USB_EHCI_TT_NEWSCHED=y
-CONFIG_USB_EHCI_PCI=y
-# CONFIG_USB_EHCI_HCD_PLATFORM is not set
+# CONFIG_USB_EHCI_HCD is not set
 # CONFIG_USB_OXU210HP_HCD is not set
 # CONFIG_USB_ISP116X_HCD is not set
 # CONFIG_USB_ISP1362_HCD is not set
@@ -2505,7 +2494,7 @@ CONFIG_USB_STORAGE=y
 # CONFIG_USB_STORAGE_KARMA is not set
 # CONFIG_USB_STORAGE_CYPRESS_ATACB is not set
 # CONFIG_USB_STORAGE_ENE_UB6250 is not set
-# CONFIG_USB_UAS is not set
+CONFIG_USB_UAS=y
 
 #
 # USB Imaging devices
@@ -2516,7 +2505,6 @@ CONFIG_USB_STORAGE=y
 # CONFIG_USB_MUSB_HDRC is not set
 # CONFIG_USB_DWC3 is not set
 # CONFIG_USB_DWC2 is not set
-# CONFIG_USB_CHIPIDEA is not set
 # CONFIG_USB_ISP1760 is not set
 
 #
@@ -2534,13 +2522,11 @@ CONFIG_USB_STORAGE=y
 # CONFIG_USB_RIO500 is not set
 # CONFIG_USB_LEGOTOWER is not set
 # CONFIG_USB_LCD is not set
-# CONFIG_USB_LED is not set
 # CONFIG_USB_CYPRESS_CY7C63 is not set
 # CONFIG_USB_CYTHERM is not set
 # CONFIG_USB_IDMOUSE is not set
 # CONFIG_USB_FTDI_ELAN is not set
 # CONFIG_USB_APPLEDISPLAY is not set
-# CONFIG_USB_SISUSBVGA is not set
 # CONFIG_USB_LD is not set
 # CONFIG_USB_TRANCEVIBRATOR is not set
 # CONFIG_USB_IOWARRIOR is not set
@@ -2550,7 +2536,9 @@ CONFIG_USB_STORAGE=y
 # CONFIG_USB_YUREX is not set
 # CONFIG_USB_EZUSB_FX2 is not set
 # CONFIG_USB_HSIC_USB3503 is not set
+# CONFIG_USB_HSIC_USB4604 is not set
 # CONFIG_USB_LINK_LAYER_TEST is not set
+# CONFIG_USB_CHAOSKEY is not set
 # CONFIG_UCSI is not set
 
 #
@@ -2558,13 +2546,64 @@ CONFIG_USB_STORAGE=y
 #
 # CONFIG_USB_PHY is not set
 # CONFIG_NOP_USB_XCEIV is not set
-# CONFIG_USB_GPIO_VBUS is not set
 # CONFIG_USB_ISP1301 is not set
 # CONFIG_USB_GADGET is not set
+# CONFIG_USB_LED_TRIG is not set
+# CONFIG_USB_ULPI_BUS is not set
 # CONFIG_UWB is not set
 # CONFIG_MMC is not set
 # CONFIG_MEMSTICK is not set
-# CONFIG_NEW_LEDS is not set
+CONFIG_NEW_LEDS=y
+CONFIG_LEDS_CLASS=y
+CONFIG_LEDS_CLASS_FLASH=y
+
+#
+# LED drivers
+#
+# CONFIG_LEDS_LM3530 is not set
+# CONFIG_LEDS_LM3642 is not set
+# CONFIG_LEDS_PCA9532 is not set
+# CONFIG_LEDS_LP3944 is not set
+# CONFIG_LEDS_LP5521 is not set
+# CONFIG_LEDS_LP5523 is not set
+# CONFIG_LEDS_LP5562 is not set
+# CONFIG_LEDS_LP8501 is not set
+# CONFIG_LEDS_LP8860 is not set
+# CONFIG_LEDS_CLEVO_MAIL is not set
+# CONFIG_LEDS_PCA955X is not set
+# CONFIG_LEDS_PCA963X is not set
+# CONFIG_LEDS_BD2802 is not set
+# CONFIG_LEDS_INTEL_SS4200 is not set
+# CONFIG_LEDS_TCA6507 is not set
+# CONFIG_LEDS_TLC591XX is not set
+# CONFIG_LEDS_LM355x is not set
+
+#
+# LED driver for blink(1) USB RGB LED is under Special HID drivers (HID_THINGM)
+#
+# CONFIG_LEDS_BLINKM is not set
+# CONFIG_LEDS_MLXCPLD is not set
+# CONFIG_LEDS_USER is not set
+# CONFIG_LEDS_NIC78BX is not set
+
+#
+# LED Triggers
+#
+CONFIG_LEDS_TRIGGERS=y
+CONFIG_LEDS_TRIGGER_TIMER=y
+CONFIG_LEDS_TRIGGER_ONESHOT=y
+CONFIG_LEDS_TRIGGER_DISK=y
+CONFIG_LEDS_TRIGGER_HEARTBEAT=y
+# CONFIG_LEDS_TRIGGER_BACKLIGHT is not set
+CONFIG_LEDS_TRIGGER_CPU=y
+CONFIG_LEDS_TRIGGER_DEFAULT_ON=y
+
+#
+# iptables trigger is under Netfilter config (LED target)
+#
+CONFIG_LEDS_TRIGGER_TRANSIENT=y
+# CONFIG_LEDS_TRIGGER_CAMERA is not set
+CONFIG_LEDS_TRIGGER_PANIC=y
 # CONFIG_ACCESSIBILITY is not set
 # CONFIG_INFINIBAND is not set
 CONFIG_EDAC_ATOMIC_SCRUB=y
@@ -2578,7 +2617,7 @@ CONFIG_EDAC_MM_EDAC=y
 # CONFIG_EDAC_I82975X is not set
 # CONFIG_EDAC_I3000 is not set
 # CONFIG_EDAC_I3200 is not set
-CONFIG_EDAC_IE31200=y
+# CONFIG_EDAC_IE31200 is not set
 # CONFIG_EDAC_X38 is not set
 # CONFIG_EDAC_I5400 is not set
 # CONFIG_EDAC_I7CORE is not set
@@ -2586,7 +2625,9 @@ CONFIG_EDAC_IE31200=y
 # CONFIG_EDAC_I5100 is not set
 # CONFIG_EDAC_I7300 is not set
 CONFIG_EDAC_SBRIDGE=y
+# CONFIG_EDAC_SKX is not set
 CONFIG_RTC_LIB=y
+CONFIG_RTC_MC146818_LIB=y
 CONFIG_RTC_CLASS=y
 CONFIG_RTC_HCTOSYS=y
 CONFIG_RTC_HCTOSYS_DEVICE="rtc0"
@@ -2615,7 +2656,6 @@ CONFIG_RTC_INTF_DEV=y
 # CONFIG_RTC_DRV_RS5C372 is not set
 # CONFIG_RTC_DRV_ISL1208 is not set
 # CONFIG_RTC_DRV_ISL12022 is not set
-# CONFIG_RTC_DRV_ISL12057 is not set
 # CONFIG_RTC_DRV_X1205 is not set
 # CONFIG_RTC_DRV_PCF8523 is not set
 # CONFIG_RTC_DRV_PCF85063 is not set
@@ -2670,12 +2710,33 @@ CONFIG_RTC_DRV_CMOS=y
 # HID Sensor RTC drivers
 #
 # CONFIG_RTC_DRV_HID_SENSOR_TIME is not set
-# CONFIG_DMADEVICES is not set
+CONFIG_DMADEVICES=y
+# CONFIG_DMADEVICES_DEBUG is not set
+
+#
+# DMA Devices
+#
+CONFIG_DMA_ENGINE=y
+CONFIG_DMA_ACPI=y
+# CONFIG_INTEL_IDMA64 is not set
+CONFIG_INTEL_IOATDMA=y
+# CONFIG_QCOM_HIDMA_MGMT is not set
+# CONFIG_QCOM_HIDMA is not set
+# CONFIG_DW_DMAC is not set
+# CONFIG_DW_DMAC_PCI is not set
+
+#
+# DMA Clients
+#
+CONFIG_ASYNC_TX_DMA=y
+# CONFIG_DMATEST is not set
+CONFIG_DMA_ENGINE_RAID=y
 
 #
 # DMABUF options
 #
 CONFIG_SYNC_FILE=y
+CONFIG_DCA=y
 # CONFIG_AUXDISPLAY is not set
 # CONFIG_UIO is not set
 # CONFIG_VFIO is not set
@@ -2691,7 +2752,6 @@ CONFIG_SYNC_FILE=y
 # Microsoft Hyper-V guest support
 #
 CONFIG_STAGING=y
-# CONFIG_SLICOSS is not set
 # CONFIG_RTS5208 is not set
 # CONFIG_FB_SM750 is not set
 # CONFIG_FB_XGI is not set
@@ -2710,8 +2770,10 @@ CONFIG_STAGING=y
 # CONFIG_CRYPTO_SKEIN is not set
 # CONFIG_UNISYSSPAR is not set
 # CONFIG_MOST is not set
+# CONFIG_GREYBUS is not set
 CONFIG_X86_PLATFORM_DEVICES=y
 # CONFIG_ACERHDF is not set
+# CONFIG_ALIENWARE_WMI is not set
 # CONFIG_DELL_WMI_AIO is not set
 # CONFIG_DELL_SMO8800 is not set
 # CONFIG_FUJITSU_TABLET is not set
@@ -2728,6 +2790,7 @@ CONFIG_ACPI_WMI=y
 # CONFIG_TOSHIBA_WMI is not set
 # CONFIG_ACPI_CMPC is not set
 # CONFIG_INTEL_HID_EVENT is not set
+# CONFIG_INTEL_VBTN is not set
 CONFIG_INTEL_IPS=y
 CONFIG_INTEL_PMC_CORE=y
 # CONFIG_IBM_RTL is not set
@@ -2736,9 +2799,12 @@ CONFIG_INTEL_PMC_CORE=y
 CONFIG_INTEL_RST=y
 CONFIG_INTEL_SMARTCONNECT=y
 # CONFIG_PVPANIC is not set
-# CONFIG_INTEL_PMC_IPC is not set
+CONFIG_INTEL_PMC_IPC=y
 # CONFIG_SURFACE_PRO3_BUTTON is not set
 CONFIG_INTEL_PUNIT_IPC=y
+# CONFIG_INTEL_TELEMETRY is not set
+# CONFIG_MLX_PLATFORM is not set
+# CONFIG_MLX_CPLD_PLATFORM is not set
 # CONFIG_CHROME_PLATFORMS is not set
 CONFIG_CLKDEV_LOOKUP=y
 CONFIG_HAVE_CLK_PREPARE=y
@@ -2753,7 +2819,13 @@ CONFIG_COMMON_CLK=y
 # CONFIG_COMMON_CLK_NXP is not set
 # CONFIG_COMMON_CLK_PXA is not set
 # CONFIG_COMMON_CLK_PIC32 is not set
-# CONFIG_COMMON_CLK_OXNAS is not set
+# CONFIG_COMMON_CLK_MT2701 is not set
+# CONFIG_COMMON_CLK_MT2701_MMSYS is not set
+# CONFIG_COMMON_CLK_MT2701_IMGSYS is not set
+# CONFIG_COMMON_CLK_MT2701_VDECSYS is not set
+# CONFIG_COMMON_CLK_MT2701_HIFSYS is not set
+# CONFIG_COMMON_CLK_MT2701_ETHSYS is not set
+# CONFIG_COMMON_CLK_MT2701_BDPSYS is not set
 
 #
 # Hardware Spinlock drivers
@@ -2770,7 +2842,9 @@ CONFIG_CLKBLD_I8253=y
 # CONFIG_SH_TIMER_MTU2 is not set
 # CONFIG_SH_TIMER_TMU is not set
 # CONFIG_EM_TIMER_STI is not set
-# CONFIG_MAILBOX is not set
+CONFIG_MAILBOX=y
+CONFIG_PCC=y
+# CONFIG_ALTERA_MBOX is not set
 CONFIG_IOMMU_API=y
 CONFIG_IOMMU_SUPPORT=y
 
@@ -2789,7 +2863,7 @@ CONFIG_IRQ_REMAP=y
 #
 # Remoteproc drivers
 #
-# CONFIG_STE_MODEM_RPROC is not set
+# CONFIG_REMOTEPROC is not set
 
 #
 # Rpmsg drivers
@@ -2798,6 +2872,10 @@ CONFIG_IRQ_REMAP=y
 #
 # SOC (System On Chip) specific Drivers
 #
+
+#
+# Broadcom SoC drivers
+#
 # CONFIG_SUNXI_SRAM is not set
 # CONFIG_SOC_TI is not set
 # CONFIG_PM_DEVFREQ is not set
@@ -2819,7 +2897,6 @@ CONFIG_ARM_GIC_MAX_NR=1
 # CONFIG_PHY_PXA_28NM_HSIC is not set
 # CONFIG_PHY_PXA_28NM_USB2 is not set
 # CONFIG_BCM_KONA_USB2_PHY is not set
-# CONFIG_PHY_TUSB1210 is not set
 CONFIG_POWERCAP=y
 CONFIG_INTEL_RAPL=y
 # CONFIG_MCB is not set
@@ -2840,7 +2917,7 @@ CONFIG_ND_BLK=y
 CONFIG_ND_CLAIM=y
 CONFIG_ND_BTT=y
 CONFIG_BTT=y
-CONFIG_DEV_DAX=y
+# CONFIG_DEV_DAX is not set
 # CONFIG_NVMEM is not set
 # CONFIG_STM is not set
 # CONFIG_INTEL_TH is not set
@@ -2853,6 +2930,7 @@ CONFIG_DEV_DAX=y
 #
 # Firmware Drivers
 #
+# CONFIG_ARM_SCPI_PROTOCOL is not set
 # CONFIG_EDD is not set
 CONFIG_FIRMWARE_MEMMAP=y
 # CONFIG_DELL_RBU is not set
@@ -2875,7 +2953,14 @@ CONFIG_EFI_VARS_PSTORE=y
 CONFIG_EFI_RUNTIME_WRAPPERS=y
 CONFIG_EFI_BOOTLOADER_CONTROL=y
 CONFIG_EFI_CAPSULE_LOADER=y
+# CONFIG_EFI_TEST is not set
+# CONFIG_APPLE_PROPERTIES is not set
 CONFIG_UEFI_CPER=y
+# CONFIG_EFI_DEV_PATH_PARSER is not set
+
+#
+# Tegra firmware driver
+#
 
 #
 # File systems
@@ -2915,6 +3000,7 @@ CONFIG_F2FS_FS_ENCRYPTION=y
 # CONFIG_FS_DAX is not set
 CONFIG_FS_POSIX_ACL=y
 CONFIG_EXPORTFS=y
+# CONFIG_EXPORTFS_BLOCK_OPS is not set
 CONFIG_FILE_LOCKING=y
 # CONFIG_MANDATORY_FILE_LOCKING is not set
 CONFIG_FS_ENCRYPTION=y
@@ -2922,11 +3008,13 @@ CONFIG_FSNOTIFY=y
 CONFIG_DNOTIFY=y
 CONFIG_INOTIFY_USER=y
 CONFIG_FANOTIFY=y
+# CONFIG_FANOTIFY_ACCESS_PERMISSIONS is not set
 # CONFIG_QUOTA is not set
 # CONFIG_QUOTACTL is not set
 CONFIG_AUTOFS4_FS=y
 # CONFIG_FUSE_FS is not set
 CONFIG_OVERLAY_FS=y
+CONFIG_OVERLAY_FS_REDIRECT_DIR=y
 
 #
 # Caches
@@ -2977,6 +3065,7 @@ CONFIG_TMPFS_POSIX_ACL=y
 CONFIG_TMPFS_XATTR=y
 CONFIG_HUGETLBFS=y
 CONFIG_HUGETLB_PAGE=y
+CONFIG_ARCH_HAS_GIGANTIC_PAGE=y
 CONFIG_CONFIGFS_FS=y
 CONFIG_EFIVAR_FS=y
 CONFIG_MISC_FILESYSTEMS=y
@@ -2990,7 +3079,6 @@ CONFIG_HFSPLUS_FS_POSIX_ACL=y
 # CONFIG_BEFS_FS is not set
 # CONFIG_BFS_FS is not set
 # CONFIG_EFS_FS is not set
-# CONFIG_LOGFS is not set
 # CONFIG_CRAMFS is not set
 # CONFIG_SQUASHFS is not set
 # CONFIG_VXFS_FS is not set
@@ -3001,16 +3089,30 @@ CONFIG_HFSPLUS_FS_POSIX_ACL=y
 # CONFIG_QNX6FS_FS is not set
 # CONFIG_ROMFS_FS is not set
 CONFIG_PSTORE=y
+# CONFIG_PSTORE_ZLIB_COMPRESS is not set
+# CONFIG_PSTORE_LZO_COMPRESS is not set
+CONFIG_PSTORE_LZ4_COMPRESS=y
 # CONFIG_PSTORE_CONSOLE is not set
 # CONFIG_PSTORE_PMSG is not set
-# CONFIG_PSTORE_RAM is not set
+CONFIG_PSTORE_RAM=y
 # CONFIG_SYSV_FS is not set
 # CONFIG_UFS_FS is not set
 CONFIG_NETWORK_FILESYSTEMS=y
 # CONFIG_NFS_FS is not set
 # CONFIG_NFSD is not set
 # CONFIG_CEPH_FS is not set
-# CONFIG_CIFS is not set
+CONFIG_CIFS=y
+# CONFIG_CIFS_STATS is not set
+CONFIG_CIFS_WEAK_PW_HASH=y
+CONFIG_CIFS_UPCALL=y
+CONFIG_CIFS_XATTR=y
+CONFIG_CIFS_POSIX=y
+CONFIG_CIFS_ACL=y
+# CONFIG_CIFS_DEBUG is not set
+CONFIG_CIFS_DFS_UPCALL=y
+CONFIG_CIFS_SMB2=y
+CONFIG_CIFS_SMB311=y
+CONFIG_CIFS_FSCACHE=y
 # CONFIG_NCP_FS is not set
 # CONFIG_CODA_FS is not set
 # CONFIG_AFS_FS is not set
@@ -3065,8 +3167,7 @@ CONFIG_NLS_CODEPAGE_949=y
 # CONFIG_NLS_MAC_ROMANIAN is not set
 # CONFIG_NLS_MAC_TURKISH is not set
 CONFIG_NLS_UTF8=y
-CONFIG_DLM=y
-# CONFIG_DLM_DEBUG is not set
+# CONFIG_DLM is not set
 
 #
 # Kernel hacking
@@ -3077,6 +3178,7 @@ CONFIG_TRACE_IRQFLAGS_SUPPORT=y
 # printk and dmesg options
 #
 CONFIG_PRINTK_TIME=y
+CONFIG_CONSOLE_LOGLEVEL_DEFAULT=7
 CONFIG_MESSAGE_LOGLEVEL_DEFAULT=4
 # CONFIG_BOOT_PRINTK_DELAY is not set
 
@@ -3084,7 +3186,7 @@ CONFIG_MESSAGE_LOGLEVEL_DEFAULT=4
 # Compile-time checks and compiler options
 #
 # CONFIG_DEBUG_INFO is not set
-CONFIG_ENABLE_WARN_DEPRECATED=y
+# CONFIG_ENABLE_WARN_DEPRECATED is not set
 # CONFIG_ENABLE_MUST_CHECK is not set
 CONFIG_FRAME_WARN=2048
 CONFIG_STRIP_ASM_SYMS=y
@@ -3123,6 +3225,7 @@ CONFIG_HAVE_DEBUG_STACKOVERFLOW=y
 CONFIG_HAVE_ARCH_KMEMCHECK=y
 # CONFIG_KMEMCHECK is not set
 CONFIG_HAVE_ARCH_KASAN=y
+# CONFIG_KASAN is not set
 CONFIG_ARCH_HAS_KCOV=y
 # CONFIG_KCOV is not set
 # CONFIG_DEBUG_SHIRQ is not set
@@ -3142,7 +3245,7 @@ CONFIG_WQ_WATCHDOG=y
 CONFIG_PANIC_ON_OOPS_VALUE=0
 CONFIG_PANIC_TIMEOUT=15
 # CONFIG_SCHED_DEBUG is not set
-CONFIG_SCHED_INFO=y
+# CONFIG_SCHED_INFO is not set
 # CONFIG_SCHEDSTATS is not set
 # CONFIG_SCHED_STACK_END_CHECK is not set
 # CONFIG_DEBUG_TIMEKEEPING is not set
@@ -3187,12 +3290,9 @@ CONFIG_RCU_CPU_STALL_TIMEOUT=21
 # CONFIG_NOTIFIER_ERROR_INJECTION is not set
 # CONFIG_FAULT_INJECTION is not set
 # CONFIG_LATENCYTOP is not set
-CONFIG_ARCH_HAS_DEBUG_STRICT_USER_COPY_CHECKS=y
-# CONFIG_DEBUG_STRICT_USER_COPY_CHECKS is not set
 CONFIG_USER_STACKTRACE_SUPPORT=y
 CONFIG_HAVE_FUNCTION_TRACER=y
 CONFIG_HAVE_FUNCTION_GRAPH_TRACER=y
-CONFIG_HAVE_FUNCTION_GRAPH_FP_TEST=y
 CONFIG_HAVE_DYNAMIC_FTRACE=y
 CONFIG_HAVE_DYNAMIC_FTRACE_WITH_REGS=y
 CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y
@@ -3223,21 +3323,21 @@ CONFIG_TRACING_SUPPORT=y
 # CONFIG_TEST_FIRMWARE is not set
 # CONFIG_TEST_UDELAY is not set
 # CONFIG_MEMTEST is not set
+# CONFIG_BUG_ON_DATA_CORRUPTION is not set
 # CONFIG_SAMPLES is not set
 CONFIG_HAVE_ARCH_KGDB=y
 # CONFIG_KGDB is not set
 CONFIG_ARCH_HAS_UBSAN_SANITIZE_ALL=y
+# CONFIG_ARCH_WANTS_UBSAN_NO_NULL is not set
 # CONFIG_UBSAN is not set
 CONFIG_ARCH_HAS_DEVMEM_IS_ALLOWED=y
-CONFIG_STRICT_DEVMEM=y
-CONFIG_IO_STRICT_DEVMEM=y
-# CONFIG_X86_VERBOSE_BOOTUP is not set
+CONFIG_X86_VERBOSE_BOOTUP=y
 # CONFIG_EARLY_PRINTK is not set
-# CONFIG_X86_PTDUMP_CORE is not set
+CONFIG_X86_PTDUMP_CORE=y
 # CONFIG_X86_PTDUMP is not set
 # CONFIG_EFI_PGT_DUMP is not set
 # CONFIG_DEBUG_RODATA_TEST is not set
-# CONFIG_DEBUG_WX is not set
+CONFIG_DEBUG_WX=y
 # CONFIG_DOUBLEFAULT is not set
 # CONFIG_DEBUG_TLBFLUSH is not set
 # CONFIG_IOMMU_STRESS is not set
@@ -3265,11 +3365,31 @@ CONFIG_KEYS=y
 CONFIG_PERSISTENT_KEYRINGS=y
 CONFIG_BIG_KEYS=y
 CONFIG_ENCRYPTED_KEYS=y
-# CONFIG_KEY_DH_OPERATIONS is not set
+CONFIG_KEY_DH_OPERATIONS=y
 CONFIG_SECURITY_DMESG_RESTRICT=y
-# CONFIG_SECURITY is not set
+CONFIG_SECURITY=y
 CONFIG_SECURITYFS=y
-# CONFIG_INTEL_TXT is not set
+# CONFIG_SECURITY_NETWORK is not set
+# CONFIG_SECURITY_PATH is not set
+CONFIG_INTEL_TXT=y
+CONFIG_HAVE_HARDENED_USERCOPY_ALLOCATOR=y
+CONFIG_HAVE_ARCH_HARDENED_USERCOPY=y
+CONFIG_HARDENED_USERCOPY=y
+CONFIG_HARDENED_USERCOPY_PAGESPAN=y
+# CONFIG_SECURITY_SMACK is not set
+# CONFIG_SECURITY_TOMOYO is not set
+# CONFIG_SECURITY_APPARMOR is not set
+# CONFIG_SECURITY_LOADPIN is not set
+CONFIG_SECURITY_YAMA=y
+CONFIG_INTEGRITY=y
+CONFIG_INTEGRITY_SIGNATURE=y
+CONFIG_INTEGRITY_ASYMMETRIC_KEYS=y
+CONFIG_INTEGRITY_TRUSTED_KEYRING=y
+# CONFIG_IMA is not set
+# CONFIG_IMA_KEYRINGS_PERMIT_SIGNED_BY_BUILTIN_OR_SECONDARY is not set
+CONFIG_EVM=y
+CONFIG_EVM_ATTR_FSUUID=y
+# CONFIG_EVM_LOAD_X509 is not set
 CONFIG_DEFAULT_SECURITY_DAC=y
 CONFIG_DEFAULT_SECURITY=""
 CONFIG_XOR_BLOCKS=y
@@ -3283,6 +3403,7 @@ CONFIG_CRYPTO=y
 #
 # Crypto core or helper
 #
+CONFIG_CRYPTO_FIPS=y
 CONFIG_CRYPTO_ALGAPI=y
 CONFIG_CRYPTO_ALGAPI2=y
 CONFIG_CRYPTO_AEAD=y
@@ -3296,7 +3417,12 @@ CONFIG_CRYPTO_RNG2=y
 CONFIG_CRYPTO_RNG_DEFAULT=y
 CONFIG_CRYPTO_AKCIPHER2=y
 CONFIG_CRYPTO_AKCIPHER=y
-# CONFIG_CRYPTO_RSA is not set
+CONFIG_CRYPTO_KPP2=y
+CONFIG_CRYPTO_KPP=y
+CONFIG_CRYPTO_ACOMP2=y
+CONFIG_CRYPTO_RSA=y
+CONFIG_CRYPTO_DH=y
+CONFIG_CRYPTO_ECDH=y
 CONFIG_CRYPTO_MANAGER=y
 CONFIG_CRYPTO_MANAGER2=y
 CONFIG_CRYPTO_USER=y
@@ -3310,6 +3436,7 @@ CONFIG_CRYPTO_CRYPTD=y
 CONFIG_CRYPTO_MCRYPTD=y
 CONFIG_CRYPTO_AUTHENC=y
 CONFIG_CRYPTO_ABLK_HELPER=y
+CONFIG_CRYPTO_SIMD=y
 CONFIG_CRYPTO_GLUE_HELPER_X86=y
 
 #
@@ -3331,7 +3458,7 @@ CONFIG_CRYPTO_ECB=y
 CONFIG_CRYPTO_LRW=y
 CONFIG_CRYPTO_PCBC=y
 CONFIG_CRYPTO_XTS=y
-# CONFIG_CRYPTO_KEYWRAP is not set
+CONFIG_CRYPTO_KEYWRAP=y
 
 #
 # Hash modes
@@ -3352,7 +3479,7 @@ CONFIG_CRYPTO_CRCT10DIF=y
 CONFIG_CRYPTO_CRCT10DIF_PCLMUL=y
 CONFIG_CRYPTO_GHASH=y
 CONFIG_CRYPTO_POLY1305=y
-# CONFIG_CRYPTO_POLY1305_X86_64 is not set
+CONFIG_CRYPTO_POLY1305_X86_64=y
 CONFIG_CRYPTO_MD4=y
 CONFIG_CRYPTO_MD5=y
 CONFIG_CRYPTO_MICHAEL_MIC=y
@@ -3365,8 +3492,11 @@ CONFIG_CRYPTO_SHA1_SSSE3=y
 CONFIG_CRYPTO_SHA256_SSSE3=y
 CONFIG_CRYPTO_SHA512_SSSE3=y
 CONFIG_CRYPTO_SHA1_MB=y
+CONFIG_CRYPTO_SHA256_MB=y
+CONFIG_CRYPTO_SHA512_MB=y
 CONFIG_CRYPTO_SHA256=y
 CONFIG_CRYPTO_SHA512=y
+CONFIG_CRYPTO_SHA3=y
 CONFIG_CRYPTO_TGR192=y
 CONFIG_CRYPTO_WP512=y
 CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL=y
@@ -3398,7 +3528,7 @@ CONFIG_CRYPTO_KHAZAD=y
 CONFIG_CRYPTO_SALSA20=y
 CONFIG_CRYPTO_SALSA20_X86_64=y
 CONFIG_CRYPTO_CHACHA20=y
-# CONFIG_CRYPTO_CHACHA20_X86_64 is not set
+CONFIG_CRYPTO_CHACHA20_X86_64=y
 CONFIG_CRYPTO_SEED=y
 CONFIG_CRYPTO_SERPENT=y
 CONFIG_CRYPTO_SERPENT_SSE2_X86_64=y
@@ -3434,21 +3564,30 @@ CONFIG_CRYPTO_JITTERENTROPY=y
 # CONFIG_CRYPTO_USER_API_SKCIPHER is not set
 # CONFIG_CRYPTO_USER_API_RNG is not set
 # CONFIG_CRYPTO_USER_API_AEAD is not set
+CONFIG_CRYPTO_HASH_INFO=y
 CONFIG_CRYPTO_HW=y
 # CONFIG_CRYPTO_DEV_PADLOCK is not set
+# CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC is not set
 # CONFIG_CRYPTO_DEV_CCP is not set
 CONFIG_CRYPTO_DEV_QAT=y
-CONFIG_CRYPTO_DEV_QAT_DH895xCC=y
+# CONFIG_CRYPTO_DEV_QAT_DH895xCC is not set
 # CONFIG_CRYPTO_DEV_QAT_C3XXX is not set
-# CONFIG_CRYPTO_DEV_QAT_C62X is not set
-CONFIG_CRYPTO_DEV_QAT_DH895xCCVF=y
+CONFIG_CRYPTO_DEV_QAT_C62X=y
+# CONFIG_CRYPTO_DEV_QAT_DH895xCCVF is not set
 # CONFIG_CRYPTO_DEV_QAT_C3XXXVF is not set
-# CONFIG_CRYPTO_DEV_QAT_C62XVF is not set
-# CONFIG_ASYMMETRIC_KEY_TYPE is not set
+CONFIG_CRYPTO_DEV_QAT_C62XVF=y
+CONFIG_ASYMMETRIC_KEY_TYPE=y
+CONFIG_ASYMMETRIC_PUBLIC_KEY_SUBTYPE=y
+CONFIG_X509_CERTIFICATE_PARSER=y
+CONFIG_PKCS7_MESSAGE_PARSER=y
 
 #
 # Certificates for signature checking
 #
+CONFIG_SYSTEM_TRUSTED_KEYRING=y
+CONFIG_SYSTEM_TRUSTED_KEYS=""
+# CONFIG_SYSTEM_EXTRA_CERTIFICATE is not set
+CONFIG_SECONDARY_TRUSTED_KEYRING=y
 CONFIG_HAVE_KVM=y
 # CONFIG_VIRTUALIZATION is not set
 # CONFIG_BINARY_PRINTF is not set
@@ -3505,6 +3644,9 @@ CONFIG_XZ_DEC_BCJ=y
 CONFIG_DECOMPRESS_GZIP=y
 CONFIG_DECOMPRESS_LZ4=y
 CONFIG_GENERIC_ALLOCATOR=y
+CONFIG_REED_SOLOMON=y
+CONFIG_REED_SOLOMON_ENC8=y
+CONFIG_REED_SOLOMON_DEC8=y
 CONFIG_TEXTSEARCH=y
 CONFIG_TEXTSEARCH_KMP=y
 CONFIG_TEXTSEARCH_BM=y
@@ -3515,15 +3657,19 @@ CONFIG_HAS_IOMEM=y
 CONFIG_HAS_IOPORT_MAP=y
 CONFIG_HAS_DMA=y
 CONFIG_CHECK_SIGNATURE=y
+CONFIG_CPUMASK_OFFSTACK=y
 CONFIG_CPU_RMAP=y
 CONFIG_DQL=y
 CONFIG_GLOB=y
 # CONFIG_GLOB_SELFTEST is not set
 CONFIG_NLATTR=y
-CONFIG_ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE=y
+CONFIG_CLZ_TAB=y
 CONFIG_CORDIC=y
 CONFIG_DDR=y
 # CONFIG_IRQ_POLL is not set
+CONFIG_MPILIB=y
+CONFIG_SIGNATURE=y
+CONFIG_OID_REGISTRY=y
 CONFIG_UCS2_STRING=y
 CONFIG_FONT_SUPPORT=y
 # CONFIG_FONTS is not set
@@ -3534,3 +3680,4 @@ CONFIG_SG_POOL=y
 CONFIG_ARCH_HAS_SG_CHAIN=y
 CONFIG_ARCH_HAS_PMEM_API=y
 CONFIG_ARCH_HAS_MMIO_FLUSH=y
+CONFIG_SBITMAP=y
diff --git a/linux-spica.install b/linux-spica.install
index 60e8431..fc68f4f 100644
--- a/linux-spica.install
+++ b/linux-spica.install
@@ -1,5 +1,5 @@
 pkgname=linux-spica
-kernver=4.7.5-1spica-dirty
+kernver=4.10.8spica-dirty
 #bootdevice="BOOT_IMAGE=/boot/vmlinuz-$pkgname root=UUID=d670564f-2cb3-4981-9d51-6ed9c1327d47"
 #option="rw quiet clocksource=hpet initrd=EFI/spi-ca/initrd intel_iommu=on pci-stub.ids=1002:683f,1002:aab0 vfio_iommu_type1.allow_unsafe_interrupts=1,kvm.ignore_msrs=1"
 #option="rw quiet clocksource=hpet initrd=EFI/spi-ca/initrd quiet intremap=no_x2apic_optout zswap.enabled=1 zswap.max_pool_percent=25 zswap.compressor=lz4"
@@ -12,19 +12,19 @@ post_install () {
 	echo "> Generating initramfs, using mkinitcpio.  Please wait..."
 	echo ">"
 	mkinitcpio -p $pkgname
-        echo "> Modifing efibootmgr..."
-        efibootmgr|awk 'match($0,/^Boot([0-9a-fA-F]{4})\* spi-ca_v(.+)$/,m){printf "efibootmgr -b %s -B;echo \">> remove entry : %s\";",m[1],m[2]}'|sh
-        echo "> Copy efistub from boot"
-	cp  -fv "boot/vmlinuz-$pkgname" "boot/efi/EFI/spi-ca/kernel.efi"
-	cp  -fv "boot/initramfs-$pkgname.img" "boot/efi/EFI/spi-ca/initrd"
-        echo "> Registering efistub "
+#        echo "> Modifing efibootmgr..."
+#        efibootmgr|awk 'match($0,/^Boot([0-9a-fA-F]{4})\* spi-ca_v(.+)$/,m){printf "efibootmgr -b %s -B;echo \">> remove entry : %s\";",m[1],m[2]}'|sh
+#        echo "> Copy efistub from boot"
+#	cp  -fv "boot/vmlinuz-$pkgname" "boot/efi/EFI/spi-ca/kernel.efi"
+#	cp  -fv "boot/initramfs-$pkgname.img" "boot/efi/EFI/spi-ca/initrd"
+#        echo "> Registering efistub "
 	#echo 'efibootmgr -c -g -d /dev/sda -p 1 -L "spi-ca_v$kernver" -l "\EFI\spi-ca\kernel.efi"  #-u "$bootdevice $option"'
-	efibootmgr -c -g -d /dev/sde -p 1 -L "spi-ca_v$kernver" -l "\EFI\spi-ca\kernel" # -u "$bootdevice $option"
-        echo "> Reordering Bootorder..."
-        newentry=`efibootmgr|awk 'match($0,/^Boot([0-9a-fA-F]{4})\* spi-ca_v(.+)$/,m){print m[1]}'`
-	prebootorder=`efibootmgr |grep BootOrder |cut -d : -f 2 |tr -d ' '`
-	efibootmgr -O
-	efibootmgr -o ${newentry},${prebootorder}	
+#	efibootmgr -c -g -d /dev/sde -p 1 -L "spi-ca_v$kernver" -l "\EFI\spi-ca\kernel" # -u "$bootdevice $option"
+#        echo "> Reordering Bootorder..."
+#        newentry=`efibootmgr|awk 'match($0,/^Boot([0-9a-fA-F]{4})\* spi-ca_v(.+)$/,m){print m[1]}'`
+#	prebootorder=`efibootmgr |grep BootOrder |cut -d : -f 2 |tr -d ' '`
+#	efibootmgr -O
+#	efibootmgr -o ${newentry},${prebootorder}	
 	echo "> OK!"
 }
 
diff --git a/patches/0001-block-cgroups-kconfig-build-bits-for-BFQ-v7r11-4.7.0.patch b/patches/0001-block-cgroups-kconfig-build-bits-for-BFQ-v7r11-4.10..patch
similarity index 94%
rename from patches/0001-block-cgroups-kconfig-build-bits-for-BFQ-v7r11-4.7.0.patch
rename to patches/0001-block-cgroups-kconfig-build-bits-for-BFQ-v7r11-4.10..patch
index ff75a8b..45f4fd2 100644
--- a/patches/0001-block-cgroups-kconfig-build-bits-for-BFQ-v7r11-4.7.0.patch
+++ b/patches/0001-block-cgroups-kconfig-build-bits-for-BFQ-v7r11-4.10..patch
@@ -1,7 +1,7 @@
-From 22ee35ec82fa543b65c1b6d516a086a21f723846 Mon Sep 17 00:00:00 2001
+From 8500f47272575b4616beb487c483019248d8c501 Mon Sep 17 00:00:00 2001
 From: Paolo Valente <paolo.valente@unimore.it>
 Date: Tue, 7 Apr 2015 13:39:12 +0200
-Subject: [PATCH 1/4] block: cgroups, kconfig, build bits for BFQ-v7r11-4.7.0
+Subject: [PATCH 1/4] block: cgroups, kconfig, build bits for BFQ-v7r11-4.10.0
 
 Update Kconfig.iosched and do the related Makefile changes to include
 kernel configuration options for BFQ. Also increase the number of
@@ -74,7 +74,7 @@ index 421bef9..0ee5f0f 100644
  
  endmenu
 diff --git a/block/Makefile b/block/Makefile
-index 9eda232..4a36683 100644
+index a827f98..3b14703 100644
 --- a/block/Makefile
 +++ b/block/Makefile
 @@ -18,6 +18,7 @@ obj-$(CONFIG_BLK_DEV_THROTTLING)	+= blk-throttle.o
@@ -86,18 +86,18 @@ index 9eda232..4a36683 100644
  obj-$(CONFIG_BLOCK_COMPAT)	+= compat_ioctl.o
  obj-$(CONFIG_BLK_CMDLINE_PARSER)	+= cmdline-parser.o
 diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
-index 3d9cf32..8d862a0 100644
+index 1ca8e8f..8e2d6ed 100644
 --- a/include/linux/blkdev.h
 +++ b/include/linux/blkdev.h
-@@ -45,7 +45,7 @@ struct pr_ops;
+@@ -47,7 +47,7 @@ struct rq_wb;
   * Maximum number of blkcg policies allowed to be registered concurrently.
   * Defined here to simplify include dependency.
   */
 -#define BLKCG_MAX_POLS		2
 +#define BLKCG_MAX_POLS		3
  
- struct request;
  typedef void (rq_end_io_fn)(struct request *, int);
+ 
 -- 
-1.9.1
+2.10.0
 
diff --git a/patches/0002-block-introduce-the-BFQ-v7r11-I-O-sched-for-4.7.0.patch b/patches/0002-block-introduce-the-BFQ-v7r11-I-O-sched-for-4.10.0.patch
similarity index 98%
rename from patches/0002-block-introduce-the-BFQ-v7r11-I-O-sched-for-4.7.0.patch
rename to patches/0002-block-introduce-the-BFQ-v7r11-I-O-sched-for-4.10.0.patch
index 368a4ff..0812a57 100644
--- a/patches/0002-block-introduce-the-BFQ-v7r11-I-O-sched-for-4.7.0.patch
+++ b/patches/0002-block-introduce-the-BFQ-v7r11-I-O-sched-for-4.10.0.patch
@@ -1,7 +1,7 @@
-From 2aae32be2a18a7d0da104ae42c08cb9bce9d9c7c Mon Sep 17 00:00:00 2001
+From 2f56e91506b329ffc29d0f184924ad0123c9ba9e Mon Sep 17 00:00:00 2001
 From: Paolo Valente <paolo.valente@unimore.it>
 Date: Thu, 9 May 2013 19:10:02 +0200
-Subject: [PATCH 2/4] block: introduce the BFQ-v7r11 I/O sched for 4.7.0
+Subject: [PATCH 2/4] block: introduce the BFQ-v7r11 I/O sched for 4.10.0
 
 The general structure is borrowed from CFQ, as much of the code for
 handling I/O contexts. Over time, several useful features have been
@@ -56,12 +56,12 @@ Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
 Signed-off-by: Arianna Avanzini <avanzini@google.com>
 ---
  block/Kconfig.iosched |    6 +-
- block/bfq-cgroup.c    | 1182 ++++++++++++++++
+ block/bfq-cgroup.c    | 1186 ++++++++++++++++
  block/bfq-ioc.c       |   36 +
- block/bfq-iosched.c   | 3754 +++++++++++++++++++++++++++++++++++++++++++++++++
- block/bfq-sched.c     | 1200 ++++++++++++++++
+ block/bfq-iosched.c   | 3763 +++++++++++++++++++++++++++++++++++++++++++++++++
+ block/bfq-sched.c     | 1199 ++++++++++++++++
  block/bfq.h           |  801 +++++++++++
- 6 files changed, 6975 insertions(+), 4 deletions(-)
+ 6 files changed, 6987 insertions(+), 4 deletions(-)
  create mode 100644 block/bfq-cgroup.c
  create mode 100644 block/bfq-ioc.c
  create mode 100644 block/bfq-iosched.c
@@ -91,10 +91,10 @@ index 0ee5f0f..f78cd1a 100644
  	prompt "Default I/O scheduler"
 diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c
 new file mode 100644
-index 0000000..8610cd6
+index 0000000..8b08a57
 --- /dev/null
 +++ b/block/bfq-cgroup.c
-@@ -0,0 +1,1182 @@
+@@ -0,0 +1,1186 @@
 +/*
 + * BFQ: CGROUPS support.
 + *
@@ -259,7 +259,9 @@ index 0000000..8610cd6
 +static struct bfq_group *blkg_to_bfqg(struct blkcg_gq *blkg)
 +{
 +	struct blkg_policy_data *pd = blkg_to_pd(blkg, &blkcg_policy_bfq);
++
 +	BUG_ON(!pd);
++
 +	return pd_to_bfqg(pd);
 +}
 +
@@ -379,7 +381,8 @@ index 0000000..8610cd6
 +	blkg_stat_add_aux(&from->time, &from->time);
 +	blkg_stat_add_aux(&to->unaccounted_time, &from->unaccounted_time);
 +	blkg_stat_add_aux(&to->avg_queue_size_sum, &from->avg_queue_size_sum);
-+	blkg_stat_add_aux(&to->avg_queue_size_samples, &from->avg_queue_size_samples);
++	blkg_stat_add_aux(&to->avg_queue_size_samples,
++			  &from->avg_queue_size_samples);
 +	blkg_stat_add_aux(&to->dequeue, &from->dequeue);
 +	blkg_stat_add_aux(&to->group_wait_time, &from->group_wait_time);
 +	blkg_stat_add_aux(&to->idle_time, &from->idle_time);
@@ -471,9 +474,9 @@ index 0000000..8610cd6
 +}
 +
 +static struct bfq_group_data *cpd_to_bfqgd(struct blkcg_policy_data *cpd)
-+ {
++{
 +	return cpd ? container_of(cpd, struct bfq_group_data, pd) : NULL;
-+ }
++}
 +
 +static struct bfq_group_data *blkcg_to_bfqgd(struct blkcg *blkcg)
 +{
@@ -562,8 +565,8 @@ index 0000000..8610cd6
 +}
 +
 +/* to be used by recursive prfill, sums live and dead rwstats recursively */
-+static struct blkg_rwstat bfqg_rwstat_pd_recursive_sum(struct blkg_policy_data *pd,
-+						       int off)
++static struct blkg_rwstat
++bfqg_rwstat_pd_recursive_sum(struct blkg_policy_data *pd, int off)
 +{
 +	struct blkg_rwstat a, b;
 +
@@ -776,7 +779,6 @@ index 0000000..8610cd6
 +
 +	BUG_ON(!bfqq);
 +	bfq_bfqq_move(bfqd, bfqq, entity, bfqd->root_group);
-+	return;
 +}
 +
 +/**
@@ -804,8 +806,6 @@ index 0000000..8610cd6
 +	if (bfqg->sched_data.in_service_entity)
 +		bfq_reparent_leaf_entity(bfqd,
 +			bfqg->sched_data.in_service_entity);
-+
-+	return;
 +}
 +
 +/**
@@ -930,6 +930,7 @@ index 0000000..8610cd6
 +	bfqgd->weight = (unsigned short)val;
 +	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
 +		struct bfq_group *bfqg = blkg_to_bfqg(blkg);
++
 +		if (!bfqg)
 +			continue;
 +		/*
@@ -1043,7 +1044,8 @@ index 0000000..8610cd6
 +	return 0;
 +}
 +
-+static struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)
++static struct bfq_group *
++bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)
 +{
 +	int ret;
 +
@@ -1051,22 +1053,22 @@ index 0000000..8610cd6
 +	if (ret)
 +		return NULL;
 +
-+        return blkg_to_bfqg(bfqd->queue->root_blkg);
++	return blkg_to_bfqg(bfqd->queue->root_blkg);
 +}
 +
 +static struct blkcg_policy_data *bfq_cpd_alloc(gfp_t gfp)
 +{
-+        struct bfq_group_data *bgd;
++	struct bfq_group_data *bgd;
 +
-+        bgd = kzalloc(sizeof(*bgd), GFP_KERNEL);
-+        if (!bgd)
-+                return NULL;
-+        return &bgd->pd;
++	bgd = kzalloc(sizeof(*bgd), GFP_KERNEL);
++	if (!bgd)
++		return NULL;
++	return &bgd->pd;
 +}
 +
 +static void bfq_cpd_free(struct blkcg_policy_data *cpd)
 +{
-+        kfree(cpd_to_bfqgd(cpd));
++	kfree(cpd_to_bfqgd(cpd));
 +}
 +
 +static struct cftype bfqio_files_dfl[] = {
@@ -1201,20 +1203,19 @@ index 0000000..8610cd6
 +};
 +
 +static struct blkcg_policy blkcg_policy_bfq = {
-+       .dfl_cftypes            = bfqio_files_dfl,
-+       .legacy_cftypes         = bfqio_files,
++	.dfl_cftypes            = bfqio_files_dfl,
++	.legacy_cftypes		= bfqio_files,
 +
-+       .pd_alloc_fn            = bfq_pd_alloc,
-+       .pd_init_fn             = bfq_pd_init,
-+       .pd_offline_fn          = bfq_pd_offline,
-+       .pd_free_fn             = bfq_pd_free,
-+       .pd_reset_stats_fn      = bfq_pd_reset_stats,
-+
-+       .cpd_alloc_fn           = bfq_cpd_alloc,
-+       .cpd_init_fn            = bfq_cpd_init,
-+       .cpd_bind_fn	       = bfq_cpd_init,
-+       .cpd_free_fn            = bfq_cpd_free,
++	.pd_alloc_fn		= bfq_pd_alloc,
++	.pd_init_fn		= bfq_pd_init,
++	.pd_offline_fn		= bfq_pd_offline,
++	.pd_free_fn		= bfq_pd_free,
++	.pd_reset_stats_fn	= bfq_pd_reset_stats,
 +
++	.cpd_alloc_fn		= bfq_cpd_alloc,
++	.cpd_init_fn		= bfq_cpd_init,
++	.cpd_bind_fn		= bfq_cpd_init,
++	.cpd_free_fn		= bfq_cpd_free,
 +};
 +
 +#else
@@ -1223,6 +1224,7 @@ index 0000000..8610cd6
 +			    struct bfq_group *bfqg)
 +{
 +	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
++
 +	entity->weight = entity->new_weight;
 +	entity->orig_weight = entity->new_weight;
 +	if (bfqq) {
@@ -1236,6 +1238,7 @@ index 0000000..8610cd6
 +bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio)
 +{
 +	struct bfq_data *bfqd = bic_to_bfqd(bic);
++
 +	return bfqd->root_group;
 +}
 +
@@ -1257,12 +1260,13 @@ index 0000000..8610cd6
 +}
 +
 +static struct bfq_group *bfq_find_alloc_group(struct bfq_data *bfqd,
-+                                              struct blkcg *blkcg)
++					      struct blkcg *blkcg)
 +{
 +	return bfqd->root_group;
 +}
 +
-+static struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)
++static struct bfq_group *
++bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)
 +{
 +	struct bfq_group *bfqg;
 +	int i;
@@ -1321,10 +1325,10 @@ index 0000000..fb7bb8f
 +}
 diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
 new file mode 100644
-index 0000000..f9787a6
+index 0000000..85e2169
 --- /dev/null
 +++ b/block/bfq-iosched.c
-@@ -0,0 +1,3754 @@
+@@ -0,0 +1,3763 @@
 +/*
 + * Budget Fair Queueing (BFQ) disk scheduler.
 + *
@@ -1542,7 +1546,7 @@ index 0000000..f9787a6
 +	unsigned long back_max;
 +#define BFQ_RQ1_WRAP	0x01 /* request 1 wraps */
 +#define BFQ_RQ2_WRAP	0x02 /* request 2 wraps */
-+	unsigned wrap = 0; /* bit mask: requests behind the disk head? */
++	unsigned int wrap = 0; /* bit mask: requests behind the disk head? */
 +
 +	if (!rq1 || rq1 == rq2)
 +		return rq2;
@@ -1597,12 +1601,11 @@ index 0000000..f9787a6
 +			return rq1;
 +		else if (d2 < d1)
 +			return rq2;
-+		else {
-+			if (s1 >= s2)
-+				return rq1;
-+			else
-+				return rq2;
-+		}
++
++		if (s1 >= s2)
++			return rq1;
++		else
++			return rq2;
 +
 +	case BFQ_RQ2_WRAP:
 +		return rq1;
@@ -1889,7 +1892,7 @@ index 0000000..f9787a6
 +		 */
 +		hlist_for_each_entry(bfqq_item, &bfqd->burst_list,
 +				     burst_list_node)
-+		        bfq_mark_bfqq_in_large_burst(bfqq_item);
++			bfq_mark_bfqq_in_large_burst(bfqq_item);
 +		bfq_mark_bfqq_in_large_burst(bfqq);
 +
 +		/*
@@ -2288,7 +2291,7 @@ index 0000000..f9787a6
 +	bfqd->rq_in_driver++;
 +	bfqd->last_position = blk_rq_pos(rq) + blk_rq_sectors(rq);
 +	bfq_log(bfqd, "activate_request: new bfqd->last_position %llu",
-+		(long long unsigned)bfqd->last_position);
++		(unsigned long long) bfqd->last_position);
 +}
 +
 +static void bfq_deactivate_request(struct request_queue *q, struct request *rq)
@@ -2595,6 +2598,7 @@ index 0000000..f9787a6
 +{
 +	struct bfq_queue *bfqq = bfqd->in_service_queue;
 +	unsigned int timeout_coeff;
++
 +	if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time)
 +		timeout_coeff = 1;
 +	else
@@ -2667,6 +2671,7 @@ index 0000000..f9787a6
 +static int bfq_bfqq_budget_left(struct bfq_queue *bfqq)
 +{
 +	struct bfq_entity *entity = &bfqq->entity;
++
 +	return entity->budget - entity->service;
 +}
 +
@@ -2906,6 +2911,7 @@ index 0000000..f9787a6
 +		if (bfqd->peak_rate_samples == BFQ_PEAK_RATE_SAMPLES &&
 +		    update) {
 +			int dev_type = blk_queue_nonrot(bfqd->queue);
++
 +			if (bfqd->bfq_user_max_budget == 0) {
 +				bfqd->bfq_max_budget =
 +					bfq_calc_max_budget(bfqd->peak_rate,
@@ -3065,6 +3071,7 @@ index 0000000..f9787a6
 +			    enum bfqq_expiration reason)
 +{
 +	bool slow;
++
 +	BUG_ON(bfqq != bfqd->in_service_queue);
 +
 +	/*
@@ -3098,7 +3105,7 @@ index 0000000..f9787a6
 +	}
 +
 +	if (reason == BFQ_BFQQ_TOO_IDLE &&
-+	    bfqq->entity.service <= 2 * bfqq->entity.budget / 10 )
++	    bfqq->entity.service <= 2 * bfqq->entity.budget / 10)
 +		bfq_clear_bfqq_IO_bound(bfqq);
 +
 +	if (bfqd->low_latency && bfqq->wr_coeff == 1)
@@ -3244,7 +3251,7 @@ index 0000000..f9787a6
 +	 */
 +	idling_boosts_thr = !bfqd->hw_tag ||
 +		(!blk_queue_nonrot(bfqd->queue) && bfq_bfqq_IO_bound(bfqq) &&
-+		 bfq_bfqq_idle_window(bfqq)) ;
++		 bfq_bfqq_idle_window(bfqq));
 +
 +	/*
 +	 * The value of the next variable,
@@ -3356,7 +3363,7 @@ index 0000000..f9787a6
 +	 * (i)  each of these processes must get the same throughput as
 +	 *      the others;
 +	 * (ii) all these processes have the same I/O pattern
-+	        (either sequential or random).
++	 *      (either sequential or random).
 +	 * In fact, in such a scenario, the drive will tend to treat
 +	 * the requests of each of these processes in about the same
 +	 * way as the requests of the others, and thus to provide
@@ -3553,6 +3560,7 @@ index 0000000..f9787a6
 +static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq)
 +{
 +	struct bfq_entity *entity = &bfqq->entity;
++
 +	if (bfqq->wr_coeff > 1) { /* queue is being weight-raised */
 +		bfq_log_bfqq(bfqd, bfqq,
 +			"raising period dur %u/%u msec, old coeff %u, w %d(%d)",
@@ -3643,7 +3651,7 @@ index 0000000..f9787a6
 +	bfq_log_bfqq(bfqd, bfqq,
 +			"dispatched %u sec req (%llu), budg left %d",
 +			blk_rq_sectors(rq),
-+			(long long unsigned)blk_rq_pos(rq),
++			(unsigned long long) blk_rq_pos(rq),
 +			bfq_bfqq_budget_left(bfqq));
 +
 +	dispatched++;
@@ -3841,7 +3849,8 @@ index 0000000..f9787a6
 + * Update the entity prio values; note that the new values will not
 + * be used until the next (re)activation.
 + */
-+static void bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic)
++static void
++bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic)
 +{
 +	struct task_struct *tsk = current;
 +	int ioprio_class;
@@ -3874,8 +3883,8 @@ index 0000000..f9787a6
 +	}
 +
 +	if (bfqq->new_ioprio < 0 || bfqq->new_ioprio >= IOPRIO_BE_NR) {
-+		printk(KERN_CRIT "bfq_set_next_ioprio_data: new_ioprio %d\n",
-+				 bfqq->new_ioprio);
++		pr_crit("bfq_set_next_ioprio_data: new_ioprio %d\n",
++			bfqq->new_ioprio);
 +		BUG();
 +	}
 +
@@ -3999,7 +4008,7 @@ index 0000000..f9787a6
 +
 +		if (bfqq) {
 +			bfq_init_bfqq(bfqd, bfqq, bic, current->pid,
-+                                      is_sync);
++				      is_sync);
 +			bfq_init_entity(&bfqq->entity, bfqg);
 +			bfq_log_bfqq(bfqd, bfqq, "allocated");
 +		} else {
@@ -4187,7 +4196,7 @@ index 0000000..f9787a6
 +	bfq_log_bfqq(bfqd, bfqq,
 +		     "rq_enqueued: idle_window=%d (seeky %d, mean %llu)",
 +		     bfq_bfqq_idle_window(bfqq), BFQQ_SEEKY(bfqq),
-+		     (long long unsigned)bfqq->seek_mean);
++		     (unsigned long long) bfqq->seek_mean);
 +
 +	bfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq);
 +
@@ -4738,8 +4747,7 @@ index 0000000..f9787a6
 +
 +static void bfq_slab_kill(void)
 +{
-+	if (bfq_pool)
-+		kmem_cache_destroy(bfq_pool);
++	kmem_cache_destroy(bfq_pool);
 +}
 +
 +static int __init bfq_slab_setup(void)
@@ -4770,6 +4778,7 @@ index 0000000..f9787a6
 +static ssize_t bfq_wr_max_time_show(struct elevator_queue *e, char *page)
 +{
 +	struct bfq_data *bfqd = e->elevator_data;
++
 +	return sprintf(page, "%d\n", bfqd->bfq_wr_max_time > 0 ?
 +		       jiffies_to_msecs(bfqd->bfq_wr_max_time) :
 +		       jiffies_to_msecs(bfq_wr_duration(bfqd)));
@@ -4788,25 +4797,29 @@ index 0000000..f9787a6
 +
 +	num_char += sprintf(page + num_char, "Active:\n");
 +	list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) {
-+	  num_char += sprintf(page + num_char,
-+			      "pid%d: weight %hu, nr_queued %d %d, dur %d/%u\n",
-+			      bfqq->pid,
-+			      bfqq->entity.weight,
-+			      bfqq->queued[0],
-+			      bfqq->queued[1],
-+			jiffies_to_msecs(jiffies - bfqq->last_wr_start_finish),
-+			jiffies_to_msecs(bfqq->wr_cur_max_time));
++		num_char += sprintf(page + num_char,
++				    "pid%d: weight %hu, nr_queued %d %d, ",
++				    bfqq->pid,
++				    bfqq->entity.weight,
++				    bfqq->queued[0],
++				    bfqq->queued[1]);
++		num_char += sprintf(page + num_char,
++				    "dur %d/%u\n",
++				    jiffies_to_msecs(
++					    jiffies -
++					    bfqq->last_wr_start_finish),
++				    jiffies_to_msecs(bfqq->wr_cur_max_time));
 +	}
 +
 +	num_char += sprintf(page + num_char, "Idle:\n");
 +	list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list) {
-+			num_char += sprintf(page + num_char,
-+				"pid%d: weight %hu, dur %d/%u\n",
-+				bfqq->pid,
-+				bfqq->entity.weight,
-+				jiffies_to_msecs(jiffies -
-+					bfqq->last_wr_start_finish),
-+				jiffies_to_msecs(bfqq->wr_cur_max_time));
++		num_char += sprintf(page + num_char,
++				    "pid%d: weight %hu, dur %d/%u\n",
++				    bfqq->pid,
++				    bfqq->entity.weight,
++				    jiffies_to_msecs(jiffies -
++						     bfqq->last_wr_start_finish),
++				    jiffies_to_msecs(bfqq->wr_cur_max_time));
 +	}
 +
 +	spin_unlock_irq(bfqd->queue->queue_lock);
@@ -5081,10 +5094,10 @@ index 0000000..f9787a6
 +MODULE_LICENSE("GPL");
 diff --git a/block/bfq-sched.c b/block/bfq-sched.c
 new file mode 100644
-index 0000000..a64fec1
+index 0000000..a5ed694
 --- /dev/null
 +++ b/block/bfq-sched.c
-@@ -0,0 +1,1200 @@
+@@ -0,0 +1,1199 @@
 +/*
 + * BFQ: Hierarchical B-WF2Q+ scheduler.
 + *
@@ -5715,8 +5728,7 @@ index 0000000..a64fec1
 +		if (entity->new_weight != entity->orig_weight) {
 +			if (entity->new_weight < BFQ_MIN_WEIGHT ||
 +			    entity->new_weight > BFQ_MAX_WEIGHT) {
-+				printk(KERN_CRIT "update_weight_prio: "
-+						 "new_weight %d\n",
++				pr_crit("update_weight_prio: new_weight %d\n",
 +					entity->new_weight);
 +				BUG();
 +			}
@@ -6287,7 +6299,7 @@ index 0000000..a64fec1
 +}
 diff --git a/block/bfq.h b/block/bfq.h
 new file mode 100644
-index 0000000..485d0c9
+index 0000000..2bf54ae
 --- /dev/null
 +++ b/block/bfq.h
 @@ -0,0 +1,801 @@
@@ -6722,10 +6734,10 @@ index 0000000..485d0c9
 + *                      @last_ins_in_burst.
 + * @burst_size: number of queues in the current burst of queue activations.
 + * @bfq_large_burst_thresh: maximum burst size above which the current
-+ * 			    queue-activation burst is deemed as 'large'.
++ *			    queue-activation burst is deemed as 'large'.
 + * @large_burst: true if a large queue-activation burst is in progress.
 + * @burst_list: head of the burst list (as for the above fields, more details
-+ * 		in the comments to the function bfq_handle_burst).
++ *		in the comments to the function bfq_handle_burst).
 + * @low_latency: if set to true, low-latency heuristics are enabled.
 + * @bfq_wr_coeff: maximum factor by which the weight of a weight-raised
 + *                queue is multiplied.
@@ -7093,5 +7105,5 @@ index 0000000..485d0c9
 +
 +#endif /* _BFQ_H */
 -- 
-1.9.1
+2.10.0
 
diff --git a/patches/0003-block-bfq-add-Early-Queue-Merge-EQM-to-BFQ-v7r11-for.patch b/patches/0003-block-bfq-add-Early-Queue-Merge-EQM-to-BFQ-v7r11-for.patch
index a9876aa..28eeb1f 100644
--- a/patches/0003-block-bfq-add-Early-Queue-Merge-EQM-to-BFQ-v7r11-for.patch
+++ b/patches/0003-block-bfq-add-Early-Queue-Merge-EQM-to-BFQ-v7r11-for.patch
@@ -1,8 +1,8 @@
-From 47de1e46ef5f462e9694e5b0607aec6ad658f1e0 Mon Sep 17 00:00:00 2001
+From e4d9bed2dfdec562b23491e44602c89c4a2a5ea4 Mon Sep 17 00:00:00 2001
 From: Mauro Andreolini <mauro.andreolini@unimore.it>
 Date: Sun, 6 Sep 2015 16:09:05 +0200
 Subject: [PATCH 3/4] block, bfq: add Early Queue Merge (EQM) to BFQ-v7r11 for
- 4.7.0
+ 4.10.0
 
 A set of processes may happen  to  perform interleaved reads, i.e.,requests
 whose union would give rise to a  sequential read  pattern.  There are two
@@ -35,16 +35,16 @@ Signed-off-by: Arianna Avanzini <avanzini@google.com>
 Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
 Signed-off-by: Linus Walleij <linus.walleij@linaro.org>
 ---
- block/bfq-cgroup.c  |   4 +
- block/bfq-iosched.c | 687 ++++++++++++++++++++++++++++++++++++++++++++++++++--
+ block/bfq-cgroup.c  |   5 +
+ block/bfq-iosched.c | 685 +++++++++++++++++++++++++++++++++++++++++++++++++++-
  block/bfq.h         |  66 +++++
- 3 files changed, 743 insertions(+), 14 deletions(-)
+ 3 files changed, 743 insertions(+), 13 deletions(-)
 
 diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c
-index 8610cd6..5ee99ec 100644
+index 8b08a57..0367996 100644
 --- a/block/bfq-cgroup.c
 +++ b/block/bfq-cgroup.c
-@@ -437,6 +437,7 @@ static void bfq_pd_init(struct blkg_policy_data *pd)
+@@ -440,6 +440,7 @@ static void bfq_pd_init(struct blkg_policy_data *pd)
  				   */
  	bfqg->bfqd = bfqd;
  	bfqg->active_entities = 0;
@@ -52,16 +52,17 @@ index 8610cd6..5ee99ec 100644
  }
  
  static void bfq_pd_free(struct blkg_policy_data *pd)
-@@ -530,6 +531,8 @@ static struct bfq_group *bfq_find_alloc_group(struct bfq_data *bfqd,
+@@ -533,6 +534,9 @@ static struct bfq_group *bfq_find_alloc_group(struct bfq_data *bfqd,
  	return bfqg;
  }
  
-+static void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq);
++static void bfq_pos_tree_add_move(struct bfq_data *bfqd,
++				  struct bfq_queue *bfqq);
 +
  /**
   * bfq_bfqq_move - migrate @bfqq to @bfqg.
   * @bfqd: queue descriptor.
-@@ -577,6 +580,7 @@ static void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -580,6 +584,7 @@ static void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  	bfqg_get(bfqg);
  
  	if (busy) {
@@ -70,10 +71,10 @@ index 8610cd6..5ee99ec 100644
  			bfq_activate_bfqq(bfqd, bfqq);
  	}
 diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
-index f9787a6..d1f648d 100644
+index 85e2169..cf3e9b1 100644
 --- a/block/bfq-iosched.c
 +++ b/block/bfq-iosched.c
-@@ -296,6 +296,72 @@ static struct request *bfq_choose_req(struct bfq_data *bfqd,
+@@ -295,6 +295,72 @@ static struct request *bfq_choose_req(struct bfq_data *bfqd,
  	}
  }
  
@@ -112,7 +113,7 @@ index f9787a6..d1f648d 100644
 +		*rb_link = p;
 +
 +	bfq_log(bfqd, "rq_pos_tree_lookup %llu: returning %d",
-+		(long long unsigned)sector,
++		(unsigned long long) sector,
 +		bfqq ? bfqq->pid : 0);
 +
 +	return bfqq;
@@ -146,11 +147,11 @@ index f9787a6..d1f648d 100644
  /*
   * Tell whether there are active queues or groups with differentiated weights.
   */
-@@ -528,6 +594,57 @@ static unsigned int bfq_wr_duration(struct bfq_data *bfqd)
+@@ -527,6 +593,57 @@ static unsigned int bfq_wr_duration(struct bfq_data *bfqd)
  	return dur;
  }
  
-+static unsigned bfq_bfqq_cooperations(struct bfq_queue *bfqq)
++static unsigned int bfq_bfqq_cooperations(struct bfq_queue *bfqq)
 +{
 +	return bfqq->bic ? bfqq->bic->cooperations : 0;
 +}
@@ -204,7 +205,7 @@ index f9787a6..d1f648d 100644
  /* Empty burst list and add just bfqq (see comments to bfq_handle_burst) */
  static void bfq_reset_burst_list(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  {
-@@ -764,8 +881,14 @@ static void bfq_add_request(struct request *rq)
+@@ -763,8 +880,14 @@ static void bfq_add_request(struct request *rq)
  	BUG_ON(!next_rq);
  	bfqq->next_rq = next_rq;
  
@@ -220,7 +221,7 @@ index f9787a6..d1f648d 100644
  		     idle_for_long_time = time_is_before_jiffies(
  						bfqq->budget_timeout +
  						bfqd->bfq_wr_min_idle_time);
-@@ -793,11 +916,12 @@ static void bfq_add_request(struct request *rq)
+@@ -792,11 +915,12 @@ static void bfq_add_request(struct request *rq)
  				bfqd->last_ins_in_burst = jiffies;
  		}
  
@@ -236,7 +237,7 @@ index f9787a6..d1f648d 100644
  		entity->budget = max_t(unsigned long, bfqq->max_budget,
  				       bfq_serv_to_charge(next_rq, bfqq));
  
-@@ -816,6 +940,9 @@ static void bfq_add_request(struct request *rq)
+@@ -815,6 +939,9 @@ static void bfq_add_request(struct request *rq)
  		if (!bfqd->low_latency)
  			goto add_bfqq_busy;
  
@@ -246,7 +247,7 @@ index f9787a6..d1f648d 100644
  		/*
  		 * If the queue:
  		 * - is not being boosted,
-@@ -840,7 +967,7 @@ static void bfq_add_request(struct request *rq)
+@@ -839,7 +966,7 @@ static void bfq_add_request(struct request *rq)
  		} else if (old_wr_coeff > 1) {
  			if (interactive)
  				bfqq->wr_cur_max_time = bfq_wr_duration(bfqd);
@@ -255,7 +256,7 @@ index f9787a6..d1f648d 100644
  				 (bfqq->wr_cur_max_time ==
  				  bfqd->bfq_wr_rt_max_time &&
  				  !soft_rt)) {
-@@ -905,6 +1032,7 @@ static void bfq_add_request(struct request *rq)
+@@ -904,6 +1031,7 @@ static void bfq_add_request(struct request *rq)
  					bfqd->bfq_wr_rt_max_time;
  			}
  		}
@@ -263,7 +264,7 @@ index f9787a6..d1f648d 100644
  		if (old_wr_coeff != bfqq->wr_coeff)
  			entity->prio_changed = 1;
  add_bfqq_busy:
-@@ -1047,6 +1175,15 @@ static void bfq_merged_request(struct request_queue *q, struct request *req,
+@@ -1046,6 +1174,15 @@ static void bfq_merged_request(struct request_queue *q, struct request *req,
  					 bfqd->last_position);
  		BUG_ON(!next_rq);
  		bfqq->next_rq = next_rq;
@@ -279,7 +280,7 @@ index f9787a6..d1f648d 100644
  	}
  }
  
-@@ -1129,11 +1266,346 @@ static void bfq_end_wr(struct bfq_data *bfqd)
+@@ -1128,11 +1265,346 @@ static void bfq_end_wr(struct bfq_data *bfqd)
  	spin_unlock_irq(bfqd->queue->queue_lock);
  }
  
@@ -572,7 +573,7 @@ index f9787a6..d1f648d 100644
 +		struct bfq_queue *bfqq, struct bfq_queue *new_bfqq)
 +{
 +	bfq_log_bfqq(bfqd, bfqq, "merging with queue %lu",
-+		(long unsigned)new_bfqq->pid);
++		     (unsigned long) new_bfqq->pid);
 +	/* Save weight raising and idle window of the merged queues */
 +	bfq_bfqq_save_state(bfqq);
 +	bfq_bfqq_save_state(new_bfqq);
@@ -626,7 +627,7 @@ index f9787a6..d1f648d 100644
  
  	/*
  	 * Disallow merge of a sync bio into an async request.
-@@ -1150,7 +1622,26 @@ static int bfq_allow_merge(struct request_queue *q, struct request *rq,
+@@ -1149,7 +1621,26 @@ static int bfq_allow_merge(struct request_queue *q, struct request *rq,
  	if (!bic)
  		return 0;
  
@@ -654,7 +655,7 @@ index f9787a6..d1f648d 100644
  }
  
  static void __bfq_set_in_service_queue(struct bfq_data *bfqd,
-@@ -1349,6 +1840,15 @@ static void __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+@@ -1350,6 +1841,15 @@ static void __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  
  	__bfq_bfqd_reset_in_service(bfqd);
  
@@ -670,7 +671,7 @@ index f9787a6..d1f648d 100644
  	if (RB_EMPTY_ROOT(&bfqq->sort_list)) {
  		/*
  		 * Overloading budget_timeout field to store the time
-@@ -1357,8 +1857,13 @@ static void __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+@@ -1358,8 +1858,13 @@ static void __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  		 */
  		bfqq->budget_timeout = jiffies;
  		bfq_del_bfqq_busy(bfqd, bfqq, 1);
@@ -685,7 +686,7 @@ index f9787a6..d1f648d 100644
  }
  
  /**
-@@ -2242,10 +2747,12 @@ static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+@@ -2246,10 +2751,12 @@ static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  		/*
  		 * If the queue was activated in a burst, or
  		 * too much time has elapsed from the beginning
@@ -700,7 +701,7 @@ index f9787a6..d1f648d 100644
  		    time_is_before_jiffies(bfqq->last_wr_start_finish +
  					   bfqq->wr_cur_max_time)) {
  			bfqq->last_wr_start_finish = jiffies;
-@@ -2474,6 +2981,25 @@ static void bfq_put_queue(struct bfq_queue *bfqq)
+@@ -2478,6 +2985,25 @@ static void bfq_put_queue(struct bfq_queue *bfqq)
  #endif
  }
  
@@ -726,7 +727,7 @@ index f9787a6..d1f648d 100644
  static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  {
  	if (bfqq == bfqd->in_service_queue) {
-@@ -2484,6 +3010,8 @@ static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+@@ -2488,6 +3014,8 @@ static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  	bfq_log_bfqq(bfqd, bfqq, "exit_bfqq: %p, %d", bfqq,
  		     atomic_read(&bfqq->ref));
  
@@ -735,7 +736,7 @@ index f9787a6..d1f648d 100644
  	bfq_put_queue(bfqq);
  }
  
-@@ -2492,6 +3020,25 @@ static void bfq_init_icq(struct io_cq *icq)
+@@ -2496,6 +3024,25 @@ static void bfq_init_icq(struct io_cq *icq)
  	struct bfq_io_cq *bic = icq_to_bic(icq);
  
  	bic->ttime.last_end_request = jiffies;
@@ -761,7 +762,7 @@ index f9787a6..d1f648d 100644
  }
  
  static void bfq_exit_icq(struct io_cq *icq)
-@@ -2505,6 +3052,13 @@ static void bfq_exit_icq(struct io_cq *icq)
+@@ -2509,6 +3056,13 @@ static void bfq_exit_icq(struct io_cq *icq)
  	}
  
  	if (bic->bfqq[BLK_RW_SYNC]) {
@@ -775,7 +776,7 @@ index f9787a6..d1f648d 100644
  		bfq_exit_bfqq(bfqd, bic->bfqq[BLK_RW_SYNC]);
  		bic->bfqq[BLK_RW_SYNC] = NULL;
  	}
-@@ -2809,6 +3363,10 @@ static void bfq_update_idle_window(struct bfq_data *bfqd,
+@@ -2814,6 +3368,10 @@ static void bfq_update_idle_window(struct bfq_data *bfqd,
  	if (!bfq_bfqq_sync(bfqq) || bfq_class_idle(bfqq))
  		return;
  
@@ -786,7 +787,7 @@ index f9787a6..d1f648d 100644
  	enable_idle = bfq_bfqq_idle_window(bfqq);
  
  	if (atomic_read(&bic->icq.ioc->active_ref) == 0 ||
-@@ -2856,6 +3414,7 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -2861,6 +3419,7 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  	if (bfqq->entity.service > bfq_max_budget(bfqd) / 8 ||
  	    !BFQQ_SEEKY(bfqq))
  		bfq_update_idle_window(bfqd, bfqq, bic);
@@ -794,7 +795,7 @@ index f9787a6..d1f648d 100644
  
  	bfq_log_bfqq(bfqd, bfqq,
  		     "rq_enqueued: idle_window=%d (seeky %d, mean %llu)",
-@@ -2920,12 +3479,47 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -2925,12 +3484,47 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  static void bfq_insert_request(struct request_queue *q, struct request *rq)
  {
  	struct bfq_data *bfqd = q->elevator->elevator_data;
@@ -843,7 +844,7 @@ index f9787a6..d1f648d 100644
  	rq->fifo_time = jiffies + bfqd->bfq_fifo_expire[rq_is_sync(rq)];
  	list_add_tail(&rq->queuelist, &bfqq->fifo);
  
-@@ -3094,6 +3688,32 @@ static void bfq_put_request(struct request *rq)
+@@ -3099,6 +3693,32 @@ static void bfq_put_request(struct request *rq)
  }
  
  /*
@@ -876,7 +877,7 @@ index f9787a6..d1f648d 100644
   * Allocate bfq data structures associated with this request.
   */
  static int bfq_set_request(struct request_queue *q, struct request *rq,
-@@ -3105,6 +3725,7 @@ static int bfq_set_request(struct request_queue *q, struct request *rq,
+@@ -3110,6 +3730,7 @@ static int bfq_set_request(struct request_queue *q, struct request *rq,
  	const int is_sync = rq_is_sync(rq);
  	struct bfq_queue *bfqq;
  	unsigned long flags;
@@ -884,7 +885,7 @@ index f9787a6..d1f648d 100644
  
  	might_sleep_if(gfpflags_allow_blocking(gfp_mask));
  
-@@ -3117,15 +3738,30 @@ static int bfq_set_request(struct request_queue *q, struct request *rq,
+@@ -3122,15 +3743,30 @@ static int bfq_set_request(struct request_queue *q, struct request *rq,
  
  	bfq_bic_update_cgroup(bic, bio);
  
@@ -900,12 +901,11 @@ index f9787a6..d1f648d 100644
 +			    bic->saved_in_large_burst)
  				bfq_mark_bfqq_in_large_burst(bfqq);
 -			else
--				bfq_clear_bfqq_in_large_burst(bfqq);
 +			else {
-+			    bfq_clear_bfqq_in_large_burst(bfqq);
-+			    if (bic->was_in_burst_list)
-+			       hlist_add_head(&bfqq->burst_list_node,
-+				              &bfqd->burst_list);
+ 				bfq_clear_bfqq_in_large_burst(bfqq);
++				if (bic->was_in_burst_list)
++					hlist_add_head(&bfqq->burst_list_node,
++						       &bfqd->burst_list);
 +			}
 +		}
 +	} else {
@@ -919,7 +919,7 @@ index f9787a6..d1f648d 100644
  		}
  	}
  
-@@ -3137,6 +3773,26 @@ static int bfq_set_request(struct request_queue *q, struct request *rq,
+@@ -3142,6 +3778,26 @@ static int bfq_set_request(struct request_queue *q, struct request *rq,
  	rq->elv.priv[0] = bic;
  	rq->elv.priv[1] = bfqq;
  
@@ -946,7 +946,7 @@ index f9787a6..d1f648d 100644
  	spin_unlock_irqrestore(q->queue_lock, flags);
  
  	return 0;
-@@ -3290,6 +3946,7 @@ static void bfq_init_root_group(struct bfq_group *root_group,
+@@ -3295,6 +3951,7 @@ static void bfq_init_root_group(struct bfq_group *root_group,
  	root_group->my_entity = NULL;
  	root_group->bfqd = bfqd;
  #endif
@@ -954,7 +954,7 @@ index f9787a6..d1f648d 100644
  	for (i = 0; i < BFQ_IOPRIO_CLASSES; i++)
  		root_group->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT;
  }
-@@ -3370,6 +4027,8 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
+@@ -3375,6 +4032,8 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
  	bfqd->bfq_timeout[BLK_RW_ASYNC] = bfq_timeout_async;
  	bfqd->bfq_timeout[BLK_RW_SYNC] = bfq_timeout_sync;
  
@@ -964,7 +964,7 @@ index f9787a6..d1f648d 100644
  
  	bfqd->bfq_large_burst_thresh = 11;
 diff --git a/block/bfq.h b/block/bfq.h
-index 485d0c9..f73c942 100644
+index 2bf54ae..fcce855 100644
 --- a/block/bfq.h
 +++ b/block/bfq.h
 @@ -183,6 +183,8 @@ struct bfq_group;
@@ -1097,5 +1097,5 @@ index 485d0c9..f73c942 100644
  static void bfq_put_queue(struct bfq_queue *bfqq);
  static void bfq_dispatch_insert(struct request_queue *q, struct request *rq);
 -- 
-1.9.1
+2.10.0
 
diff --git a/patches/0004-block-bfq-turn-BFQ-v7r11-for-4.7.0-into-BFQ-v8r3-for.patch b/patches/0004-Turn-BFQ-v7r11-for-4.10.0-into-BFQ-v8r8-for-4.10.0.patch
similarity index 60%
rename from patches/0004-block-bfq-turn-BFQ-v7r11-for-4.7.0-into-BFQ-v8r3-for.patch
rename to patches/0004-Turn-BFQ-v7r11-for-4.10.0-into-BFQ-v8r8-for-4.10.0.patch
index bf56ac7..48e64d9 100644
--- a/patches/0004-block-bfq-turn-BFQ-v7r11-for-4.7.0-into-BFQ-v8r3-for.patch
+++ b/patches/0004-Turn-BFQ-v7r11-for-4.10.0-into-BFQ-v8r8-for-4.10.0.patch
@@ -1,22 +1,588 @@
-From d384ccf796a992e27691b7359ce54534db57e74c Mon Sep 17 00:00:00 2001
+From b782bbfcb5e08e92c0448d0c6a870b44db198837 Mon Sep 17 00:00:00 2001
 From: Paolo Valente <paolo.valente@linaro.org>
-Date: Tue, 17 May 2016 08:28:04 +0200
-Subject: [PATCH 4/4] block, bfq: turn BFQ-v7r11 for 4.7.0 into BFQ-v8r3 for
- 4.7.0
+Date: Mon, 16 May 2016 11:16:17 +0200
+Subject: [PATCH 4/4] Turn BFQ-v7r11 for 4.10.0 into BFQ-v8r8 for 4.10.0
 
+Signed-off-by: Paolo Valente <paolo.valente@linaro.org>
 ---
- block/Kconfig.iosched |    2 +-
- block/bfq-cgroup.c    |  480 +++++----
- block/bfq-iosched.c   | 2602 +++++++++++++++++++++++++++++--------------------
- block/bfq-sched.c     |  441 +++++++--
- block/bfq.h           |  708 +++++++-------
- 5 files changed, 2484 insertions(+), 1749 deletions(-)
+ Documentation/block/00-INDEX        |    2 +
+ Documentation/block/bfq-iosched.txt |  530 ++++++
+ block/Kconfig.iosched               |   18 +-
+ block/bfq-cgroup.c                  |  510 +++---
+ block/bfq-iosched.c                 | 3414 ++++++++++++++++++++++-------------
+ block/bfq-sched.c                   | 1290 ++++++++++---
+ block/bfq.h                         |  800 ++++----
+ 7 files changed, 4390 insertions(+), 2174 deletions(-)
+ create mode 100644 Documentation/block/bfq-iosched.txt
 
+diff --git a/Documentation/block/00-INDEX b/Documentation/block/00-INDEX
+index e55103a..8d55b4b 100644
+--- a/Documentation/block/00-INDEX
++++ b/Documentation/block/00-INDEX
+@@ -1,5 +1,7 @@
+ 00-INDEX
+ 	- This file
++bfq-iosched.txt
++	- BFQ IO scheduler and its tunables
+ biodoc.txt
+ 	- Notes on the Generic Block Layer Rewrite in Linux 2.5
+ biovecs.txt
+diff --git a/Documentation/block/bfq-iosched.txt b/Documentation/block/bfq-iosched.txt
+new file mode 100644
+index 0000000..13b5248
+--- /dev/null
++++ b/Documentation/block/bfq-iosched.txt
+@@ -0,0 +1,530 @@
++BFQ (Budget Fair Queueing)
++==========================
++
++BFQ is a proportional-share I/O scheduler, with some extra
++low-latency capabilities. In addition to cgroups support (blkio or io
++controllers), BFQ's main features are:
++- BFQ guarantees a high system and application responsiveness, and a
++  low latency for time-sensitive applications, such as audio or video
++  players;
++- BFQ distributes bandwidth, and not just time, among processes or
++  groups (switching back to time distribution when needed to keep
++  throughput high).
++
++On average CPUs, the current version of BFQ can handle devices
++performing at most ~30K IOPS; at most ~50 KIOPS on faster CPUs. As a
++reference, 30-50 KIOPS correspond to very high bandwidths with
++sequential I/O (e.g., 8-12 GB/s if I/O requests are 256 KB large), and
++to 120-200 MB/s with 4KB random I/O.
++
++The table of contents follow. Impatients can just jump to Section 3.
++
++CONTENTS
++
++1. When may BFQ be useful?
++ 1-1 Personal systems
++ 1-2 Server systems
++2. How does BFQ work?
++3. What are BFQ's tunable?
++4. BFQ group scheduling
++ 4-1 Service guarantees provided
++ 4-2 Interface
++
++1. When may BFQ be useful?
++==========================
++
++BFQ provides the following benefits on personal and server systems.
++
++1-1 Personal systems
++--------------------
++
++Low latency for interactive applications
++
++Regardless of the actual background workload, BFQ guarantees that, for
++interactive tasks, the storage device is virtually as responsive as if
++it was idle. For example, even if one or more of the following
++background workloads are being executed:
++- one or more large files are being read, written or copied,
++- a tree of source files is being compiled,
++- one or more virtual machines are performing I/O,
++- a software update is in progress,
++- indexing daemons are scanning filesystems and updating their
++  databases,
++starting an application or loading a file from within an application
++takes about the same time as if the storage device was idle. As a
++comparison, with CFQ, NOOP or DEADLINE, and in the same conditions,
++applications experience high latencies, or even become unresponsive
++until the background workload terminates (also on SSDs).
++
++Low latency for soft real-time applications
++
++Also soft real-time applications, such as audio and video
++players/streamers, enjoy a low latency and a low drop rate, regardless
++of the background I/O workload. As a consequence, these applications
++do not suffer from almost any glitch due to the background workload.
++
++Higher speed for code-development tasks
++
++If some additional workload happens to be executed in parallel, then
++BFQ executes the I/O-related components of typical code-development
++tasks (compilation, checkout, merge, ...) much more quickly than CFQ,
++NOOP or DEADLINE.
++
++High throughput
++
++On hard disks, BFQ achieves up to 30% higher throughput than CFQ, and
++up to 150% higher throughput than DEADLINE and NOOP, with all the
++sequential workloads considered in our tests. With random workloads,
++and with all the workloads on flash-based devices, BFQ achieves,
++instead, about the same throughput as the other schedulers.
++
++Strong fairness, bandwidth and delay guarantees
++
++BFQ distributes the device throughput, and not just the device time,
++among I/O-bound applications in proportion their weights, with any
++workload and regardless of the device parameters. From these bandwidth
++guarantees, it is possible to compute tight per-I/O-request delay
++guarantees by a simple formula. If not configured for strict service
++guarantees, BFQ switches to time-based resource sharing (only) for
++applications that would otherwise cause a throughput loss.
++
++1-2 Server systems
++------------------
++
++Most benefits for server systems follow from the same service
++properties as above. In particular, regardless of whether additional,
++possibly heavy workloads are being served, BFQ guarantees:
++
++. audio and video-streaming with zero or very low jitter and drop
++  rate;
++
++. fast retrieval of WEB pages and embedded objects;
++
++. real-time recording of data in live-dumping applications (e.g.,
++  packet logging);
++
++. responsiveness in local and remote access to a server.
++
++
++2. How does BFQ work?
++=====================
++
++BFQ is a proportional-share I/O scheduler, whose general structure,
++plus a lot of code, are borrowed from CFQ.
++
++- Each process doing I/O on a device is associated with a weight and a
++  (bfq_)queue.
++
++- BFQ grants exclusive access to the device, for a while, to one queue
++  (process) at a time, and implements this service model by
++  associating every queue with a budget, measured in number of
++  sectors.
++
++  - After a queue is granted access to the device, the budget of the
++    queue is decremented, on each request dispatch, by the size of the
++    request.
++
++  - The in-service queue is expired, i.e., its service is suspended,
++    only if one of the following events occurs: 1) the queue finishes
++    its budget, 2) the queue empties, 3) a "budget timeout" fires.
++
++    - The budget timeout prevents processes doing random I/O from
++      holding the device for too long and dramatically reducing
++      throughput.
++
++    - Actually, as in CFQ, a queue associated with a process issuing
++      sync requests may not be expired immediately when it empties. In
++      contrast, BFQ may idle the device for a short time interval,
++      giving the process the chance to go on being served if it issues
++      a new request in time. Device idling typically boosts the
++      throughput on rotational devices, if processes do synchronous
++      and sequential I/O. In addition, under BFQ, device idling is
++      also instrumental in guaranteeing the desired throughput
++      fraction to processes issuing sync requests (see the description
++      of the slice_idle tunable in this document, or [1, 2], for more
++      details).
++
++      - With respect to idling for service guarantees, if several
++	processes are competing for the device at the same time, but
++	all processes (and groups, after the following commit) have
++	the same weight, then BFQ guarantees the expected throughput
++	distribution without ever idling the device. Throughput is
++	thus as high as possible in this common scenario.
++
++  - If low-latency mode is enabled (default configuration), BFQ
++    executes some special heuristics to detect interactive and soft
++    real-time applications (e.g., video or audio players/streamers),
++    and to reduce their latency. The most important action taken to
++    achieve this goal is to give to the queues associated with these
++    applications more than their fair share of the device
++    throughput. For brevity, we call just "weight-raising" the whole
++    sets of actions taken by BFQ to privilege these queues. In
++    particular, BFQ provides a milder form of weight-raising for
++    interactive applications, and a stronger form for soft real-time
++    applications.
++
++  - BFQ automatically deactivates idling for queues born in a burst of
++    queue creations. In fact, these queues are usually associated with
++    the processes of applications and services that benefit mostly
++    from a high throughput. Examples are systemd during boot, or git
++    grep.
++
++  - As CFQ, BFQ merges queues performing interleaved I/O, i.e.,
++    performing random I/O that becomes mostly sequential if
++    merged. Differently from CFQ, BFQ achieves this goal with a more
++    reactive mechanism, called Early Queue Merge (EQM). EQM is so
++    responsive in detecting interleaved I/O (cooperating processes),
++    that it enables BFQ to achieve a high throughput, by queue
++    merging, even for queues for which CFQ needs a different
++    mechanism, preemption, to get a high throughput. As such EQM is a
++    unified mechanism to achieve a high throughput with interleaved
++    I/O.
++
++  - Queues are scheduled according to a variant of WF2Q+, named
++    B-WF2Q+, and implemented using an augmented rb-tree to preserve an
++    O(log N) overall complexity.  See [2] for more details. B-WF2Q+ is
++    also ready for hierarchical scheduling. However, for a cleaner
++    logical breakdown, the code that enables and completes
++    hierarchical support is provided in the next commit, which focuses
++    exactly on this feature.
++
++  - B-WF2Q+ guarantees a tight deviation with respect to an ideal,
++    perfectly fair, and smooth service. In particular, B-WF2Q+
++    guarantees that each queue receives a fraction of the device
++    throughput proportional to its weight, even if the throughput
++    fluctuates, and regardless of: the device parameters, the current
++    workload and the budgets assigned to the queue.
++
++  - The last, budget-independence, property (although probably
++    counterintuitive in the first place) is definitely beneficial, for
++    the following reasons:
++
++    - First, with any proportional-share scheduler, the maximum
++      deviation with respect to an ideal service is proportional to
++      the maximum budget (slice) assigned to queues. As a consequence,
++      BFQ can keep this deviation tight not only because of the
++      accurate service of B-WF2Q+, but also because BFQ *does not*
++      need to assign a larger budget to a queue to let the queue
++      receive a higher fraction of the device throughput.
++
++    - Second, BFQ is free to choose, for every process (queue), the
++      budget that best fits the needs of the process, or best
++      leverages the I/O pattern of the process. In particular, BFQ
++      updates queue budgets with a simple feedback-loop algorithm that
++      allows a high throughput to be achieved, while still providing
++      tight latency guarantees to time-sensitive applications. When
++      the in-service queue expires, this algorithm computes the next
++      budget of the queue so as to:
++
++      - Let large budgets be eventually assigned to the queues
++	associated with I/O-bound applications performing sequential
++	I/O: in fact, the longer these applications are served once
++	got access to the device, the higher the throughput is.
++
++      - Let small budgets be eventually assigned to the queues
++	associated with time-sensitive applications (which typically
++	perform sporadic and short I/O), because, the smaller the
++	budget assigned to a queue waiting for service is, the sooner
++	B-WF2Q+ will serve that queue (Subsec 3.3 in [2]).
++
++- If several processes are competing for the device at the same time,
++  but all processes and groups have the same weight, then BFQ
++  guarantees the expected throughput distribution without ever idling
++  the device. It uses preemption instead. Throughput is then much
++  higher in this common scenario.
++
++- ioprio classes are served in strict priority order, i.e.,
++  lower-priority queues are not served as long as there are
++  higher-priority queues.  Among queues in the same class, the
++  bandwidth is distributed in proportion to the weight of each
++  queue. A very thin extra bandwidth is however guaranteed to
++  the Idle class, to prevent it from starving.
++
++
++3. What are BFQ's tunable?
++==========================
++
++The tunables back_seek-max, back_seek_penalty, fifo_expire_async and
++fifo_expire_sync below are the same as in CFQ. Their description is
++just copied from that for CFQ. Some considerations in the description
++of slice_idle are copied from CFQ too.
++
++per-process ioprio and weight
++-----------------------------
++
++Unless the cgroups interface is used (see "4. BFQ group scheduling"),
++weights can be assigned to processes only indirectly, through I/O
++priorities, and according to the relation:
++weight = (IOPRIO_BE_NR - ioprio) * 10.
++
++Beware that, if low-latency is set, then BFQ automatically raises the
++weight of the queues associated with interactive and soft real-time
++applications. Unset this tunable if you need/want to control weights.
++
++slice_idle
++----------
++
++This parameter specifies how long BFQ should idle for next I/O
++request, when certain sync BFQ queues become empty. By default
++slice_idle is a non-zero value. Idling has a double purpose: boosting
++throughput and making sure that the desired throughput distribution is
++respected (see the description of how BFQ works, and, if needed, the
++papers referred there).
++
++As for throughput, idling can be very helpful on highly seeky media
++like single spindle SATA/SAS disks where we can cut down on overall
++number of seeks and see improved throughput.
++
++Setting slice_idle to 0 will remove all the idling on queues and one
++should see an overall improved throughput on faster storage devices
++like multiple SATA/SAS disks in hardware RAID configuration.
++
++So depending on storage and workload, it might be useful to set
++slice_idle=0.  In general for SATA/SAS disks and software RAID of
++SATA/SAS disks keeping slice_idle enabled should be useful. For any
++configurations where there are multiple spindles behind single LUN
++(Host based hardware RAID controller or for storage arrays), setting
++slice_idle=0 might end up in better throughput and acceptable
++latencies.
++
++Idling is however necessary to have service guarantees enforced in
++case of differentiated weights or differentiated I/O-request lengths.
++To see why, suppose that a given BFQ queue A must get several I/O
++requests served for each request served for another queue B. Idling
++ensures that, if A makes a new I/O request slightly after becoming
++empty, then no request of B is dispatched in the middle, and thus A
++does not lose the possibility to get more than one request dispatched
++before the next request of B is dispatched. Note that idling
++guarantees the desired differentiated treatment of queues only in
++terms of I/O-request dispatches. To guarantee that the actual service
++order then corresponds to the dispatch order, the strict_guarantees
++tunable must be set too.
++
++There is an important flipside for idling: apart from the above cases
++where it is beneficial also for throughput, idling can severely impact
++throughput. One important case is random workload. Because of this
++issue, BFQ tends to avoid idling as much as possible, when it is not
++beneficial also for throughput. As a consequence of this behavior, and
++of further issues described for the strict_guarantees tunable,
++short-term service guarantees may be occasionally violated. And, in
++some cases, these guarantees may be more important than guaranteeing
++maximum throughput. For example, in video playing/streaming, a very
++low drop rate may be more important than maximum throughput. In these
++cases, consider setting the strict_guarantees parameter.
++
++strict_guarantees
++-----------------
++
++If this parameter is set (default: unset), then BFQ
++
++- always performs idling when the in-service queue becomes empty;
++
++- forces the device to serve one I/O request at a time, by dispatching a
++  new request only if there is no outstanding request.
++
++In the presence of differentiated weights or I/O-request sizes, both
++the above conditions are needed to guarantee that every BFQ queue
++receives its allotted share of the bandwidth. The first condition is
++needed for the reasons explained in the description of the slice_idle
++tunable.  The second condition is needed because all modern storage
++devices reorder internally-queued requests, which may trivially break
++the service guarantees enforced by the I/O scheduler.
++
++Setting strict_guarantees may evidently affect throughput.
++
++back_seek_max
++-------------
++
++This specifies, given in Kbytes, the maximum "distance" for backward seeking.
++The distance is the amount of space from the current head location to the
++sectors that are backward in terms of distance.
++
++This parameter allows the scheduler to anticipate requests in the "backward"
++direction and consider them as being the "next" if they are within this
++distance from the current head location.
++
++back_seek_penalty
++-----------------
++
++This parameter is used to compute the cost of backward seeking. If the
++backward distance of request is just 1/back_seek_penalty from a "front"
++request, then the seeking cost of two requests is considered equivalent.
++
++So scheduler will not bias toward one or the other request (otherwise scheduler
++will bias toward front request). Default value of back_seek_penalty is 2.
++
++fifo_expire_async
++-----------------
++
++This parameter is used to set the timeout of asynchronous requests. Default
++value of this is 248ms.
++
++fifo_expire_sync
++----------------
++
++This parameter is used to set the timeout of synchronous requests. Default
++value of this is 124ms. In case to favor synchronous requests over asynchronous
++one, this value should be decreased relative to fifo_expire_async.
++
++low_latency
++-----------
++
++This parameter is used to enable/disable BFQ's low latency mode. By
++default, low latency mode is enabled. If enabled, interactive and soft
++real-time applications are privileged and experience a lower latency,
++as explained in more detail in the description of how BFQ works.
++
++DO NOT enable this mode if you need full control on bandwidth
++distribution. In fact, if it is enabled, then BFQ automatically
++increases the bandwidth share of privileged applications, as the main
++means to guarantee a lower latency to them.
++
++timeout_sync
++------------
++
++Maximum amount of device time that can be given to a task (queue) once
++it has been selected for service. On devices with costly seeks,
++increasing this time usually increases maximum throughput. On the
++opposite end, increasing this time coarsens the granularity of the
++short-term bandwidth and latency guarantees, especially if the
++following parameter is set to zero.
++
++max_budget
++----------
++
++Maximum amount of service, measured in sectors, that can be provided
++to a BFQ queue once it is set in service (of course within the limits
++of the above timeout). According to what said in the description of
++the algorithm, larger values increase the throughput in proportion to
++the percentage of sequential I/O requests issued. The price of larger
++values is that they coarsen the granularity of short-term bandwidth
++and latency guarantees.
++
++The default value is 0, which enables auto-tuning: BFQ sets max_budget
++to the maximum number of sectors that can be served during
++timeout_sync, according to the estimated peak rate.
++
++weights
++-------
++
++Read-only parameter, used to show the weights of the currently active
++BFQ queues.
++
++
++wr_ tunables
++------------
++
++BFQ exports a few parameters to control/tune the behavior of
++low-latency heuristics.
++
++wr_coeff
++
++Factor by which the weight of a weight-raised queue is multiplied. If
++the queue is deemed soft real-time, then the weight is further
++multiplied by an additional, constant factor.
++
++wr_max_time
++
++Maximum duration of a weight-raising period for an interactive task
++(ms). If set to zero (default value), then this value is computed
++automatically, as a function of the peak rate of the device. In any
++case, when the value of this parameter is read, it always reports the
++current duration, regardless of whether it has been set manually or
++computed automatically.
++
++wr_max_softrt_rate
++
++Maximum service rate below which a queue is deemed to be associated
++with a soft real-time application, and is then weight-raised
++accordingly (sectors/sec).
++
++wr_min_idle_time
++
++Minimum idle period after which interactive weight-raising may be
++reactivated for a queue (in ms).
++
++wr_rt_max_time
++
++Maximum weight-raising duration for soft real-time queues (in ms). The
++start time from which this duration is considered is automatically
++moved forward if the queue is detected to be still soft real-time
++before the current soft real-time weight-raising period finishes.
++
++wr_min_inter_arr_async
++
++Minimum period between I/O request arrivals after which weight-raising
++may be reactivated for an already busy async queue (in ms).
++
++
++4. Group scheduling with BFQ
++============================
++
++BFQ supports both cgroups-v1 and cgroups-v2 io controllers, namely
++blkio and io. In particular, BFQ supports weight-based proportional
++share. To activate cgroups support, set BFQ_GROUP_IOSCHED.
++
++4-1 Service guarantees provided
++-------------------------------
++
++With BFQ, proportional share means true proportional share of the
++device bandwidth, according to group weights. For example, a group
++with weight 200 gets twice the bandwidth, and not just twice the time,
++of a group with weight 100.
++
++BFQ supports hierarchies (group trees) of any depth. Bandwidth is
++distributed among groups and processes in the expected way: for each
++group, the children of the group share the whole bandwidth of the
++group in proportion to their weights. In particular, this implies
++that, for each leaf group, every process of the group receives the
++same share of the whole group bandwidth, unless the ioprio of the
++process is modified.
++
++The resource-sharing guarantee for a group may partially or totally
++switch from bandwidth to time, if providing bandwidth guarantees to
++the group lowers the throughput too much. This switch occurs on a
++per-process basis: if a process of a leaf group causes throughput loss
++if served in such a way to receive its share of the bandwidth, then
++BFQ switches back to just time-based proportional share for that
++process.
++
++4-2 Interface
++-------------
++
++To get proportional sharing of bandwidth with BFQ for a given device,
++BFQ must of course be the active scheduler for that device.
++
++Within each group directory, the names of the files associated with
++BFQ-specific cgroup parameters and stats begin with the "bfq."
++prefix. So, with cgroups-v1 or cgroups-v2, the full prefix for
++BFQ-specific files is "blkio.bfq." or "io.bfq." For example, the group
++parameter to set the weight of a group with BFQ is blkio.bfq.weight
++or io.bfq.weight.
++
++Parameters to set
++-----------------
++
++For each group, there is only the following parameter to set.
++
++weight (namely blkio.bfq.weight or io.bfq-weight): the weight of the
++group inside its parent. Available values: 1..10000 (default 100). The
++linear mapping between ioprio and weights, described at the beginning
++of the tunable section, is still valid, but all weights higher than
++IOPRIO_BE_NR*10 are mapped to ioprio 0.
++
++Recall that, if low-latency is set, then BFQ automatically raises the
++weight of the queues associated with interactive and soft real-time
++applications. Unset this tunable if you need/want to control weights.
++
++
++[1] P. Valente, A. Avanzini, "Evolution of the BFQ Storage I/O
++    Scheduler", Proceedings of the First Workshop on Mobile System
++    Technologies (MST-2015), May 2015.
++    http://algogroup.unimore.it/people/paolo/disk_sched/mst-2015.pdf
++
++[2] P. Valente and M. Andreolini, "Improving Application
++    Responsiveness with the BFQ Disk I/O Scheduler", Proceedings of
++    the 5th Annual International Systems and Storage Conference
++    (SYSTOR '12), June 2012.
++    Slightly extended version:
++    http://algogroup.unimore.it/people/paolo/disk_sched/bfq-v1-suite-
++							results.pdf
 diff --git a/block/Kconfig.iosched b/block/Kconfig.iosched
-index f78cd1a..6d92579 100644
+index f78cd1a..f2cd945 100644
 --- a/block/Kconfig.iosched
 +++ b/block/Kconfig.iosched
-@@ -53,7 +53,7 @@ config IOSCHED_BFQ
+@@ -43,20 +43,20 @@ config IOSCHED_BFQ
+ 	tristate "BFQ I/O scheduler"
+ 	default n
+ 	---help---
+-	  The BFQ I/O scheduler tries to distribute bandwidth among
+-	  all processes according to their weights.
+-	  It aims at distributing the bandwidth as desired, independently of
+-	  the disk parameters and with any workload. It also tries to
+-	  guarantee low latency to interactive and soft real-time
+-	  applications. If compiled built-in (saying Y here), BFQ can
+-	  be configured to support hierarchical scheduling.
++	The BFQ I/O scheduler distributes bandwidth among all
++	processes according to their weights, regardless of the
++	device parameters and with any workload. It also guarantees
++	a low latency to interactive and soft real-time applications.
++	Details in Documentation/block/bfq-iosched.txt
  
  config BFQ_GROUP_IOSCHED
  	bool "BFQ hierarchical scheduling support"
@@ -24,35 +590,91 @@ index f78cd1a..6d92579 100644
 +	depends on IOSCHED_BFQ && BLK_CGROUP
  	default n
  	---help---
- 	  Enable hierarchical scheduling in BFQ, using the blkio controller.
+-	  Enable hierarchical scheduling in BFQ, using the blkio controller.
++
++	Enable hierarchical scheduling in BFQ, using the blkio
++	(cgroups-v1) or io (cgroups-v2) controller.
+ 
+ choice
+ 	prompt "Default I/O scheduler"
 diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c
-index 5ee99ec..c83d90c 100644
+index 0367996..0125275 100644
 --- a/block/bfq-cgroup.c
 +++ b/block/bfq-cgroup.c
-@@ -162,7 +162,6 @@ static struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg)
- static struct bfq_group *blkg_to_bfqg(struct blkcg_gq *blkg)
+@@ -7,7 +7,9 @@
+  * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
+  *		      Paolo Valente <paolo.valente@unimore.it>
+  *
+- * Copyright (C) 2010 Paolo Valente <paolo.valente@unimore.it>
++ * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
++ *
++ * Copyright (C) 2016 Paolo Valente <paolo.valente@linaro.org>
+  *
+  * Licensed under the GPL-2 as detailed in the accompanying COPYING.BFQ
+  * file.
+@@ -163,8 +165,6 @@ static struct bfq_group *blkg_to_bfqg(struct blkcg_gq *blkg)
  {
  	struct blkg_policy_data *pd = blkg_to_pd(blkg, &blkcg_policy_bfq);
+ 
 -	BUG_ON(!pd);
+-
  	return pd_to_bfqg(pd);
  }
  
-@@ -224,14 +223,6 @@ static void bfqg_stats_update_io_merged(struct bfq_group *bfqg, int rw)
- 	blkg_rwstat_add(&bfqg->stats.merged, rw, 1);
+@@ -208,59 +208,47 @@ static void bfqg_put(struct bfq_group *bfqg)
+ 
+ static void bfqg_stats_update_io_add(struct bfq_group *bfqg,
+ 				     struct bfq_queue *bfqq,
+-				     int rw)
++				     unsigned int op)
+ {
+-	blkg_rwstat_add(&bfqg->stats.queued, rw, 1);
++	blkg_rwstat_add(&bfqg->stats.queued, op, 1);
+ 	bfqg_stats_end_empty_time(&bfqg->stats);
+ 	if (!(bfqq == ((struct bfq_data *)bfqg->bfqd)->in_service_queue))
+ 		bfqg_stats_set_start_group_wait_time(bfqg, bfqq_group(bfqq));
+ }
+ 
+-static void bfqg_stats_update_io_remove(struct bfq_group *bfqg, int rw)
+-{
+-	blkg_rwstat_add(&bfqg->stats.queued, rw, -1);
+-}
+-
+-static void bfqg_stats_update_io_merged(struct bfq_group *bfqg, int rw)
++static void bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op)
+ {
+-	blkg_rwstat_add(&bfqg->stats.merged, rw, 1);
++	blkg_rwstat_add(&bfqg->stats.queued, op, -1);
  }
  
 -static void bfqg_stats_update_dispatch(struct bfq_group *bfqg,
 -					      uint64_t bytes, int rw)
--{
++static void bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op)
+ {
 -	blkg_stat_add(&bfqg->stats.sectors, bytes >> 9);
 -	blkg_rwstat_add(&bfqg->stats.serviced, rw, 1);
 -	blkg_rwstat_add(&bfqg->stats.service_bytes, rw, bytes);
--}
--
++	blkg_rwstat_add(&bfqg->stats.merged, op, 1);
+ }
+ 
  static void bfqg_stats_update_completion(struct bfq_group *bfqg,
- 			uint64_t start_time, uint64_t io_start_time, int rw)
+-			uint64_t start_time, uint64_t io_start_time, int rw)
++			uint64_t start_time, uint64_t io_start_time,
++			unsigned int op)
  {
-@@ -248,17 +239,11 @@ static void bfqg_stats_update_completion(struct bfq_group *bfqg,
+ 	struct bfqg_stats *stats = &bfqg->stats;
+ 	unsigned long long now = sched_clock();
+ 
+ 	if (time_after64(now, io_start_time))
+-		blkg_rwstat_add(&stats->service_time, rw, now - io_start_time);
++		blkg_rwstat_add(&stats->service_time, op,
++				now - io_start_time);
+ 	if (time_after64(io_start_time, start_time))
+-		blkg_rwstat_add(&stats->wait_time, rw,
++		blkg_rwstat_add(&stats->wait_time, op,
+ 				io_start_time - start_time);
+ }
+ 
  /* @stats = 0 */
  static void bfqg_stats_reset(struct bfqg_stats *stats)
  {
@@ -70,7 +692,7 @@ index 5ee99ec..c83d90c 100644
  	blkg_stat_reset(&stats->avg_queue_size_sum);
  	blkg_stat_reset(&stats->avg_queue_size_samples);
  	blkg_stat_reset(&stats->dequeue);
-@@ -268,21 +253,19 @@ static void bfqg_stats_reset(struct bfqg_stats *stats)
+@@ -270,19 +258,16 @@ static void bfqg_stats_reset(struct bfqg_stats *stats)
  }
  
  /* @to += @from */
@@ -89,13 +711,9 @@ index 5ee99ec..c83d90c 100644
  	blkg_stat_add_aux(&from->time, &from->time);
 -	blkg_stat_add_aux(&to->unaccounted_time, &from->unaccounted_time);
  	blkg_stat_add_aux(&to->avg_queue_size_sum, &from->avg_queue_size_sum);
--	blkg_stat_add_aux(&to->avg_queue_size_samples, &from->avg_queue_size_samples);
-+	blkg_stat_add_aux(&to->avg_queue_size_samples,
-+			  &from->avg_queue_size_samples);
- 	blkg_stat_add_aux(&to->dequeue, &from->dequeue);
- 	blkg_stat_add_aux(&to->group_wait_time, &from->group_wait_time);
- 	blkg_stat_add_aux(&to->idle_time, &from->idle_time);
-@@ -308,10 +291,8 @@ static void bfqg_stats_xfer_dead(struct bfq_group *bfqg)
+ 	blkg_stat_add_aux(&to->avg_queue_size_samples,
+ 			  &from->avg_queue_size_samples);
+@@ -311,10 +296,8 @@ static void bfqg_stats_xfer_dead(struct bfq_group *bfqg)
  	if (unlikely(!parent))
  		return;
  
@@ -107,7 +725,14 @@ index 5ee99ec..c83d90c 100644
  }
  
  static void bfq_init_entity(struct bfq_entity *entity,
-@@ -332,15 +313,11 @@ static void bfq_init_entity(struct bfq_entity *entity,
+@@ -329,21 +312,17 @@ static void bfq_init_entity(struct bfq_entity *entity,
+ 		bfqq->ioprio_class = bfqq->new_ioprio_class;
+ 		bfqg_get(bfqg);
+ 	}
+-	entity->parent = bfqg->my_entity;
++	entity->parent = bfqg->my_entity; /* NULL for root group */
+ 	entity->sched_data = &bfqg->sched_data;
+ }
  
  static void bfqg_stats_exit(struct bfqg_stats *stats)
  {
@@ -123,7 +748,7 @@ index 5ee99ec..c83d90c 100644
  	blkg_stat_exit(&stats->avg_queue_size_sum);
  	blkg_stat_exit(&stats->avg_queue_size_samples);
  	blkg_stat_exit(&stats->dequeue);
-@@ -351,15 +328,11 @@ static void bfqg_stats_exit(struct bfqg_stats *stats)
+@@ -354,15 +333,11 @@ static void bfqg_stats_exit(struct bfqg_stats *stats)
  
  static int bfqg_stats_init(struct bfqg_stats *stats, gfp_t gfp)
  {
@@ -140,18 +765,7 @@ index 5ee99ec..c83d90c 100644
  	    blkg_stat_init(&stats->avg_queue_size_sum, gfp) ||
  	    blkg_stat_init(&stats->avg_queue_size_samples, gfp) ||
  	    blkg_stat_init(&stats->dequeue, gfp) ||
-@@ -374,20 +347,36 @@ static int bfqg_stats_init(struct bfqg_stats *stats, gfp_t gfp)
- }
- 
- static struct bfq_group_data *cpd_to_bfqgd(struct blkcg_policy_data *cpd)
-- {
-+{
- 	return cpd ? container_of(cpd, struct bfq_group_data, pd) : NULL;
-- }
-+}
- 
- static struct bfq_group_data *blkcg_to_bfqgd(struct blkcg *blkcg)
- {
+@@ -386,11 +361,27 @@ static struct bfq_group_data *blkcg_to_bfqgd(struct blkcg *blkcg)
  	return cpd_to_bfqgd(blkcg_to_cpd(blkcg, &blkcg_policy_bfq));
  }
  
@@ -159,7 +773,7 @@ index 5ee99ec..c83d90c 100644
 +{
 +	struct bfq_group_data *bgd;
 +
-+	bgd = kzalloc(sizeof(*bgd), GFP_KERNEL);
++	bgd = kzalloc(sizeof(*bgd), gfp);
 +	if (!bgd)
 +		return NULL;
 +	return &bgd->pd;
@@ -180,7 +794,7 @@ index 5ee99ec..c83d90c 100644
  }
  
  static struct blkg_policy_data *bfq_pd_alloc(gfp_t gfp, int node)
-@@ -398,8 +387,7 @@ static struct blkg_policy_data *bfq_pd_alloc(gfp_t gfp, int node)
+@@ -401,8 +392,7 @@ static struct blkg_policy_data *bfq_pd_alloc(gfp_t gfp, int node)
  	if (!bfqg)
  		return NULL;
  
@@ -190,7 +804,7 @@ index 5ee99ec..c83d90c 100644
  		kfree(bfqg);
  		return NULL;
  	}
-@@ -407,27 +395,20 @@ static struct blkg_policy_data *bfq_pd_alloc(gfp_t gfp, int node)
+@@ -410,27 +400,20 @@ static struct blkg_policy_data *bfq_pd_alloc(gfp_t gfp, int node)
  	return &bfqg->pd;
  }
  
@@ -228,7 +842,7 @@ index 5ee99ec..c83d90c 100644
  
  	entity->orig_weight = entity->weight = entity->new_weight = d->weight;
  	entity->my_sched_data = &bfqg->sched_data;
-@@ -445,70 +426,53 @@ static void bfq_pd_free(struct blkg_policy_data *pd)
+@@ -448,70 +431,53 @@ static void bfq_pd_free(struct blkg_policy_data *pd)
  	struct bfq_group *bfqg = pd_to_bfqg(pd);
  
  	bfqg_stats_exit(&bfqg->stats);
@@ -256,23 +870,23 @@ index 5ee99ec..c83d90c 100644
  }
  
 -/* to be used by recursive prfill, sums live and dead rwstats recursively */
--static struct blkg_rwstat bfqg_rwstat_pd_recursive_sum(struct blkg_policy_data *pd,
--						       int off)
+-static struct blkg_rwstat
+-bfqg_rwstat_pd_recursive_sum(struct blkg_policy_data *pd, int off)
 +static void bfq_group_set_parent(struct bfq_group *bfqg,
 +					struct bfq_group *parent)
  {
 -	struct blkg_rwstat a, b;
 +	struct bfq_entity *entity;
-+
-+	BUG_ON(!parent);
-+	BUG_ON(!bfqg);
-+	BUG_ON(bfqg == parent);
  
 -	a = blkg_rwstat_recursive_sum(pd_to_blkg(pd), &blkcg_policy_bfq, off);
 -	b = blkg_rwstat_recursive_sum(pd_to_blkg(pd), &blkcg_policy_bfq,
 -				      off + dead_stats_off_delta);
 -	blkg_rwstat_add_aux(&a, &b);
 -	return a;
++	BUG_ON(!parent);
++	BUG_ON(!bfqg);
++	BUG_ON(bfqg == parent);
++
 +	entity = &bfqg->entity;
 +	entity->parent = parent->my_entity;
 +	entity->sched_data = &parent->sched_data;
@@ -326,19 +940,15 @@ index 5ee99ec..c83d90c 100644
  
  	/*
  	 * Update chain of bfq_groups as we might be handling a leaf group
-@@ -531,13 +495,18 @@ static struct bfq_group *bfq_find_alloc_group(struct bfq_data *bfqd,
- 	return bfqg;
- }
+@@ -537,11 +503,15 @@ static struct bfq_group *bfq_find_alloc_group(struct bfq_data *bfqd,
+ static void bfq_pos_tree_add_move(struct bfq_data *bfqd,
+ 				  struct bfq_queue *bfqq);
  
--static void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq);
-+static void bfq_pos_tree_add_move(struct bfq_data *bfqd,
-+				  struct bfq_queue *bfqq);
-+
 +static void bfq_bfqq_expire(struct bfq_data *bfqd,
 +			    struct bfq_queue *bfqq,
 +			    bool compensate,
 +			    enum bfqq_expiration reason);
- 
++
  /**
   * bfq_bfqq_move - migrate @bfqq to @bfqg.
   * @bfqd: queue descriptor.
@@ -347,7 +957,7 @@ index 5ee99ec..c83d90c 100644
   * @bfqg: the group to move to.
   *
   * Move @bfqq to @bfqg, deactivating it from its old group and reactivating
-@@ -548,26 +517,40 @@ static void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+@@ -552,26 +522,40 @@ static void bfq_pos_tree_add_move(struct bfq_data *bfqd,
   * rcu_read_lock()).
   */
  static void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
@@ -393,7 +1003,7 @@ index 5ee99ec..c83d90c 100644
 -			bfq_deactivate_bfqq(bfqd, bfqq, 0);
 -	} else if (entity->on_st)
 +	if (bfq_bfqq_busy(bfqq))
-+		bfq_deactivate_bfqq(bfqd, bfqq, 0);
++		bfq_deactivate_bfqq(bfqd, bfqq, false, false);
 +	else if (entity->on_st) {
 +		BUG_ON(&bfq_entity_service_tree(entity)->idle !=
 +		       entity->tree);
@@ -402,7 +1012,7 @@ index 5ee99ec..c83d90c 100644
  	bfqg_put(bfqq_group(bfqq));
  
  	/*
-@@ -579,14 +562,17 @@ static void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -583,14 +567,17 @@ static void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  	entity->sched_data = &bfqg->sched_data;
  	bfqg_get(bfqg);
  
@@ -423,16 +1033,20 @@ index 5ee99ec..c83d90c 100644
  }
  
  /**
-@@ -613,7 +599,7 @@ static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd,
+@@ -617,7 +604,11 @@ static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd,
  
  	lockdep_assert_held(bfqd->queue->queue_lock);
  
 -	bfqg = bfq_find_alloc_group(bfqd, blkcg);
 +	bfqg = bfq_find_set_group(bfqd, blkcg);
++
++	if (unlikely(!bfqg))
++		bfqg = bfqd->root_group;
++
  	if (async_bfqq) {
  		entity = &async_bfqq->entity;
  
-@@ -621,7 +607,8 @@ static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd,
+@@ -625,7 +616,8 @@ static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd,
  			bic_set_bfqq(bic, NULL, 0);
  			bfq_log_bfqq(bfqd, async_bfqq,
  				     "bic_change_group: %p %d",
@@ -442,7 +1056,7 @@ index 5ee99ec..c83d90c 100644
  			bfq_put_queue(async_bfqq);
  		}
  	}
-@@ -629,7 +616,7 @@ static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd,
+@@ -633,7 +625,7 @@ static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd,
  	if (sync_bfqq) {
  		entity = &sync_bfqq->entity;
  		if (entity->sched_data != &bfqg->sched_data)
@@ -451,7 +1065,7 @@ index 5ee99ec..c83d90c 100644
  	}
  
  	return bfqg;
-@@ -638,25 +625,23 @@ static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd,
+@@ -642,25 +634,23 @@ static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd,
  static void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio)
  {
  	struct bfq_data *bfqd = bic_to_bfqd(bic);
@@ -485,22 +1099,25 @@ index 5ee99ec..c83d90c 100644
  }
  
  /**
-@@ -682,8 +667,7 @@ static void bfq_reparent_leaf_entity(struct bfq_data *bfqd,
+@@ -672,7 +662,7 @@ static void bfq_flush_idle_tree(struct bfq_service_tree *st)
+ 	struct bfq_entity *entity = st->first_idle;
+ 
+ 	for (; entity ; entity = st->first_idle)
+-		__bfq_deactivate_entity(entity, 0);
++		__bfq_deactivate_entity(entity, false);
+ }
+ 
+ /**
+@@ -686,7 +676,7 @@ static void bfq_reparent_leaf_entity(struct bfq_data *bfqd,
  	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
  
  	BUG_ON(!bfqq);
 -	bfq_bfqq_move(bfqd, bfqq, entity, bfqd->root_group);
--	return;
 +	bfq_bfqq_move(bfqd, bfqq, bfqd->root_group);
  }
  
  /**
-@@ -711,16 +695,15 @@ static void bfq_reparent_active_entities(struct bfq_data *bfqd,
- 	if (bfqg->sched_data.in_service_entity)
- 		bfq_reparent_leaf_entity(bfqd,
- 			bfqg->sched_data.in_service_entity);
--
--	return;
+@@ -717,11 +707,12 @@ static void bfq_reparent_active_entities(struct bfq_data *bfqd,
  }
  
  /**
@@ -517,7 +1134,12 @@ index 5ee99ec..c83d90c 100644
   */
  static void bfq_pd_offline(struct blkg_policy_data *pd)
  {
-@@ -779,6 +762,12 @@ static void bfq_pd_offline(struct blkg_policy_data *pd)
+@@ -776,10 +767,16 @@ static void bfq_pd_offline(struct blkg_policy_data *pd)
+ 	BUG_ON(bfqg->sched_data.next_in_service);
+ 	BUG_ON(bfqg->sched_data.in_service_entity);
+ 
+-	__bfq_deactivate_entity(entity, 0);
++	__bfq_deactivate_entity(entity, false);
  	bfq_put_async_queues(bfqd, bfqg);
  	BUG_ON(entity->tree);
  
@@ -530,7 +1152,7 @@ index 5ee99ec..c83d90c 100644
  	bfqg_stats_xfer_dead(bfqg);
  }
  
-@@ -788,46 +777,35 @@ static void bfq_end_wr_async(struct bfq_data *bfqd)
+@@ -789,46 +786,35 @@ static void bfq_end_wr_async(struct bfq_data *bfqd)
  
  	list_for_each_entry(blkg, &bfqd->queue->blkg_list, q_node) {
  		struct bfq_group *bfqg = blkg_to_bfqg(blkg);
@@ -588,15 +1210,7 @@ index 5ee99ec..c83d90c 100644
  
  	if (val < BFQ_MIN_WEIGHT || val > BFQ_MAX_WEIGHT)
  		return ret;
-@@ -837,6 +815,7 @@ static int bfqio_cgroup_weight_write(struct cgroup_subsys_state *css,
- 	bfqgd->weight = (unsigned short)val;
- 	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
- 		struct bfq_group *bfqg = blkg_to_bfqg(blkg);
-+
- 		if (!bfqg)
- 			continue;
- 		/*
-@@ -871,13 +850,18 @@ static int bfqio_cgroup_weight_write(struct cgroup_subsys_state *css,
+@@ -873,13 +859,18 @@ static int bfqio_cgroup_weight_write(struct cgroup_subsys_state *css,
  	return ret;
  }
  
@@ -620,7 +1234,7 @@ index 5ee99ec..c83d90c 100644
  }
  
  static int bfqg_print_stat(struct seq_file *sf, void *v)
-@@ -897,16 +881,17 @@ static int bfqg_print_rwstat(struct seq_file *sf, void *v)
+@@ -899,16 +890,17 @@ static int bfqg_print_rwstat(struct seq_file *sf, void *v)
  static u64 bfqg_prfill_stat_recursive(struct seq_file *sf,
  				      struct blkg_policy_data *pd, int off)
  {
@@ -642,7 +1256,7 @@ index 5ee99ec..c83d90c 100644
  	return __blkg_prfill_rwstat(sf, pd, &sum);
  }
  
-@@ -926,6 +911,41 @@ static int bfqg_print_rwstat_recursive(struct seq_file *sf, void *v)
+@@ -928,6 +920,41 @@ static int bfqg_print_rwstat_recursive(struct seq_file *sf, void *v)
  	return 0;
  }
  
@@ -684,37 +1298,23 @@ index 5ee99ec..c83d90c 100644
  static u64 bfqg_prfill_avg_queue_size(struct seq_file *sf,
  				      struct blkg_policy_data *pd, int off)
  {
-@@ -950,7 +970,8 @@ static int bfqg_print_avg_queue_size(struct seq_file *sf, void *v)
- 	return 0;
- }
- 
--static struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)
-+static struct bfq_group *
-+bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)
- {
- 	int ret;
- 
-@@ -958,41 +979,18 @@ static struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int n
- 	if (ret)
- 		return NULL;
- 
--        return blkg_to_bfqg(bfqd->queue->root_blkg);
-+	return blkg_to_bfqg(bfqd->queue->root_blkg);
+@@ -964,38 +991,15 @@ bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)
+ 	return blkg_to_bfqg(bfqd->queue->root_blkg);
  }
  
 -static struct blkcg_policy_data *bfq_cpd_alloc(gfp_t gfp)
 -{
--        struct bfq_group_data *bgd;
+-	struct bfq_group_data *bgd;
 -
--        bgd = kzalloc(sizeof(*bgd), GFP_KERNEL);
--        if (!bgd)
--                return NULL;
--        return &bgd->pd;
+-	bgd = kzalloc(sizeof(*bgd), GFP_KERNEL);
+-	if (!bgd)
+-		return NULL;
+-	return &bgd->pd;
 -}
 -
 -static void bfq_cpd_free(struct blkcg_policy_data *cpd)
 -{
--        kfree(cpd_to_bfqgd(cpd));
+-	kfree(cpd_to_bfqgd(cpd));
 -}
 -
 -static struct cftype bfqio_files_dfl[] = {
@@ -742,7 +1342,7 @@ index 5ee99ec..c83d90c 100644
  	{
  		.name = "bfq.time",
  		.private = offsetof(struct bfq_group, stats.time),
-@@ -1000,18 +998,17 @@ static struct cftype bfqio_files[] = {
+@@ -1003,18 +1007,17 @@ static struct cftype bfqio_files[] = {
  	},
  	{
  		.name = "bfq.sectors",
@@ -766,7 +1366,7 @@ index 5ee99ec..c83d90c 100644
  	},
  	{
  		.name = "bfq.io_service_time",
-@@ -1042,18 +1039,17 @@ static struct cftype bfqio_files[] = {
+@@ -1045,18 +1048,17 @@ static struct cftype bfqio_files[] = {
  	},
  	{
  		.name = "bfq.sectors_recursive",
@@ -790,7 +1390,7 @@ index 5ee99ec..c83d90c 100644
  	},
  	{
  		.name = "bfq.io_service_time_recursive",
-@@ -1099,32 +1095,35 @@ static struct cftype bfqio_files[] = {
+@@ -1102,31 +1104,42 @@ static struct cftype bfqio_files[] = {
  		.private = offsetof(struct bfq_group, stats.dequeue),
  		.seq_show = bfqg_print_stat,
  	},
@@ -803,20 +1403,19 @@ index 5ee99ec..c83d90c 100644
  };
  
 -static struct blkcg_policy blkcg_policy_bfq = {
--       .dfl_cftypes            = bfqio_files_dfl,
--       .legacy_cftypes         = bfqio_files,
+-	.dfl_cftypes            = bfqio_files_dfl,
+-	.legacy_cftypes		= bfqio_files,
 -
--       .pd_alloc_fn            = bfq_pd_alloc,
--       .pd_init_fn             = bfq_pd_init,
--       .pd_offline_fn          = bfq_pd_offline,
--       .pd_free_fn             = bfq_pd_free,
--       .pd_reset_stats_fn      = bfq_pd_reset_stats,
--
--       .cpd_alloc_fn           = bfq_cpd_alloc,
--       .cpd_init_fn            = bfq_cpd_init,
--       .cpd_bind_fn	       = bfq_cpd_init,
--       .cpd_free_fn            = bfq_cpd_free,
+-	.pd_alloc_fn		= bfq_pd_alloc,
+-	.pd_init_fn		= bfq_pd_init,
+-	.pd_offline_fn		= bfq_pd_offline,
+-	.pd_free_fn		= bfq_pd_free,
+-	.pd_reset_stats_fn	= bfq_pd_reset_stats,
 -
+-	.cpd_alloc_fn		= bfq_cpd_alloc,
+-	.cpd_init_fn		= bfq_cpd_init,
+-	.cpd_bind_fn		= bfq_cpd_init,
+-	.cpd_free_fn		= bfq_cpd_free,
 +static struct cftype bfq_blkg_files[] = {
 +	{
 +		.name = "bfq.weight",
@@ -831,33 +1430,49 @@ index 5ee99ec..c83d90c 100644
 +#else /* CONFIG_BFQ_GROUP_IOSCHED */
 +
 +static inline void bfqg_stats_update_io_add(struct bfq_group *bfqg,
-+			struct bfq_queue *bfqq, int rw) { }
-+static inline void bfqg_stats_update_io_remove(struct bfq_group *bfqg, int rw) { }
-+static inline void bfqg_stats_update_io_merged(struct bfq_group *bfqg, int rw) { }
++			struct bfq_queue *bfqq, unsigned int op) { }
++static inline void
++bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op) { }
++static inline void
++bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op) { }
 +static inline void bfqg_stats_update_completion(struct bfq_group *bfqg,
-+			uint64_t start_time, uint64_t io_start_time, int rw) { }
-+static inline void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg,
-+struct bfq_group *curr_bfqg) { }
++			uint64_t start_time, uint64_t io_start_time,
++			unsigned int op) { }
++static inline void
++bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg,
++				     struct bfq_group *curr_bfqg) { }
 +static inline void bfqg_stats_end_empty_time(struct bfqg_stats *stats) { }
 +static inline void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { }
 +static inline void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg) { }
 +static inline void bfqg_stats_update_idle_time(struct bfq_group *bfqg) { }
 +static inline void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { }
 +static inline void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg) { }
++
++static void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
++			  struct bfq_group *bfqg) {}
  
  static void bfq_init_entity(struct bfq_entity *entity,
  			    struct bfq_group *bfqg)
-@@ -1146,27 +1145,20 @@ bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio)
- 	return bfqd->root_group;
+@@ -1142,35 +1155,22 @@ static void bfq_init_entity(struct bfq_entity *entity,
+ 	entity->sched_data = &bfqg->sched_data;
  }
  
+-static struct bfq_group *
+-bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio)
+-{
+-	struct bfq_data *bfqd = bic_to_bfqd(bic);
+-
+-	return bfqd->root_group;
+-}
+-
 -static void bfq_bfqq_move(struct bfq_data *bfqd,
 -			  struct bfq_queue *bfqq,
 -			  struct bfq_entity *entity,
 -			  struct bfq_group *bfqg)
 -{
 -}
--
++static void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio) {}
+ 
  static void bfq_end_wr_async(struct bfq_data *bfqd)
  {
  	bfq_end_wr_async_queues(bfqd, bfqd->root_group);
@@ -865,31 +1480,40 @@ index 5ee99ec..c83d90c 100644
  
 -static void bfq_disconnect_groups(struct bfq_data *bfqd)
 +static struct bfq_group *bfq_find_set_group(struct bfq_data *bfqd,
-+                                              struct blkcg *blkcg)
++					    struct blkcg *blkcg)
  {
 -	bfq_put_async_queues(bfqd, bfqd->root_group);
 +	return bfqd->root_group;
  }
  
 -static struct bfq_group *bfq_find_alloc_group(struct bfq_data *bfqd,
--                                              struct blkcg *blkcg)
+-					      struct blkcg *blkcg)
 +static struct bfq_group *bfqq_group(struct bfq_queue *bfqq)
  {
 -	return bfqd->root_group;
 +	return bfqq->bfqd->root_group;
  }
  
- static struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)
+ static struct bfq_group *
 diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
-index d1f648d..3bc1f8b 100644
+index cf3e9b1..e5dfa5a 100644
 --- a/block/bfq-iosched.c
 +++ b/block/bfq-iosched.c
-@@ -7,25 +7,26 @@
+@@ -1,5 +1,5 @@
+ /*
+- * Budget Fair Queueing (BFQ) disk scheduler.
++ * Budget Fair Queueing (BFQ) I/O scheduler.
+  *
+  * Based on ideas and code from CFQ:
+  * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
+@@ -7,25 +7,34 @@
   * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
   *		      Paolo Valente <paolo.valente@unimore.it>
   *
 - * Copyright (C) 2010 Paolo Valente <paolo.valente@unimore.it>
-+ * Copyright (C) 2016 Paolo Valente <paolo.valente@unimore.it>
++ * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
++ *
++ * Copyright (C) 2017 Paolo Valente <paolo.valente@linaro.org>
   *
   * Licensed under the GPL-2 as detailed in the accompanying COPYING.BFQ
   * file.
@@ -908,6 +1532,12 @@ index d1f648d..3bc1f8b 100644
 - * I/O-bound processes issuing sequential requests (to boost the
 - * throughput), and yet guarantee a low latency to interactive and soft
 - * real-time applications.
++ * BFQ is a proportional-share I/O scheduler, with some extra
++ * low-latency capabilities. BFQ also supports full hierarchical
++ * scheduling through cgroups. Next paragraphs provide an introduction
++ * on BFQ inner workings. Details on BFQ benefits and usage can be
++ * found in Documentation/block/bfq-iosched.txt.
++ *
 + * BFQ is a proportional-share storage-I/O scheduling algorithm based
 + * on the slice-by-slice service scheme of CFQ. But BFQ assigns
 + * budgets, measured in number of sectors, to processes instead of
@@ -926,37 +1556,91 @@ index d1f648d..3bc1f8b 100644
   *
   * BFQ is described in [1], where also a reference to the initial, more
   * theoretical paper on BFQ can be found. The interested reader can find
-@@ -87,7 +88,6 @@ static const int bfq_stats_min_budgets = 194;
+@@ -40,10 +49,10 @@
+  * H-WF2Q+, while the augmented tree used to implement B-WF2Q+ with O(log N)
+  * complexity derives from the one introduced with EEVDF in [3].
+  *
+- * [1] P. Valente and M. Andreolini, ``Improving Application Responsiveness
+- *     with the BFQ Disk I/O Scheduler'',
+- *     Proceedings of the 5th Annual International Systems and Storage
+- *     Conference (SYSTOR '12), June 2012.
++ * [1] P. Valente, A. Avanzini, "Evolution of the BFQ Storage I/O
++ *   Scheduler", Proceedings of the First Workshop on Mobile System
++ *   Technologies (MST-2015), May 2015.
++ *   http://algogroup.unimore.it/people/paolo/disk_sched/mst-2015.pdf
+  *
+  * http://algogroup.unimo.it/people/paolo/disk_sched/bf1-v1-suite-results.pdf
+  *
+@@ -70,24 +79,23 @@
+ #include "bfq.h"
+ #include "blk.h"
+ 
+-/* Expiration time of sync (0) and async (1) requests, in jiffies. */
+-static const int bfq_fifo_expire[2] = { HZ / 4, HZ / 8 };
++/* Expiration time of sync (0) and async (1) requests, in ns. */
++static const u64 bfq_fifo_expire[2] = { NSEC_PER_SEC / 4, NSEC_PER_SEC / 8 };
+ 
+ /* Maximum backwards seek, in KiB. */
+-static const int bfq_back_max = 16 * 1024;
++static const int bfq_back_max = (16 * 1024);
+ 
+ /* Penalty of a backwards seek, in number of sectors. */
+ static const int bfq_back_penalty = 2;
+ 
+-/* Idling period duration, in jiffies. */
+-static int bfq_slice_idle = HZ / 125;
++/* Idling period duration, in ns. */
++static u32 bfq_slice_idle = (NSEC_PER_SEC / 125);
+ 
+ /* Minimum number of assigned budgets for which stats are safe to compute. */
+ static const int bfq_stats_min_budgets = 194;
  
  /* Default maximum budget values, in sectors and number of requests. */
- static const int bfq_default_max_budget = 16 * 1024;
+-static const int bfq_default_max_budget = 16 * 1024;
 -static const int bfq_max_budget_async_rq = 4;
++static const int bfq_default_max_budget = (16 * 1024);
  
  /*
   * Async to sync throughput distribution is controlled as follows:
-@@ -97,8 +97,7 @@ static const int bfq_max_budget_async_rq = 4;
+@@ -97,23 +105,28 @@ static const int bfq_max_budget_async_rq = 4;
  static const int bfq_async_charge_factor = 10;
  
  /* Default timeout values, in jiffies, approximating CFQ defaults. */
 -static const int bfq_timeout_sync = HZ / 8;
 -static int bfq_timeout_async = HZ / 25;
-+static const int bfq_timeout = HZ / 8;
++static const int bfq_timeout = (HZ / 8);
  
- struct kmem_cache *bfq_pool;
+-struct kmem_cache *bfq_pool;
++static struct kmem_cache *bfq_pool;
  
-@@ -109,8 +108,9 @@ struct kmem_cache *bfq_pool;
+-/* Below this threshold (in ms), we consider thinktime immediate. */
+-#define BFQ_MIN_TT		2
++/* Below this threshold (in ns), we consider thinktime immediate. */
++#define BFQ_MIN_TT		(2 * NSEC_PER_MSEC)
+ 
+ /* hw_tag detection: parallel requests threshold and min samples needed. */
  #define BFQ_HW_QUEUE_THRESHOLD	4
  #define BFQ_HW_QUEUE_SAMPLES	32
  
 -#define BFQQ_SEEK_THR	 (sector_t)(8 * 1024)
 -#define BFQQ_SEEKY(bfqq) ((bfqq)->seek_mean > BFQQ_SEEK_THR)
-+#define BFQQ_SEEK_THR	 	(sector_t)(8 * 100)
++#define BFQQ_SEEK_THR		(sector_t)(8 * 100)
++#define BFQQ_SECT_THR_NONROT	(sector_t)(2 * 32)
 +#define BFQQ_CLOSE_THR		(sector_t)(8 * 1024)
 +#define BFQQ_SEEKY(bfqq)	(hweight32(bfqq->seek_history) > 32/8)
  
- /* Min samples used for peak rate estimation (for autotuning). */
- #define BFQ_PEAK_RATE_SAMPLES	32
-@@ -141,16 +141,24 @@ struct kmem_cache *bfq_pool;
+-/* Min samples used for peak rate estimation (for autotuning). */
+-#define BFQ_PEAK_RATE_SAMPLES	32
++/* Min number of samples required to perform peak-rate update */
++#define BFQ_RATE_MIN_SAMPLES	32
++/* Min observation time interval required to perform a peak-rate update (ns) */
++#define BFQ_RATE_MIN_INTERVAL	(300*NSEC_PER_MSEC)
++/* Target observation time interval for a peak-rate update (ns) */
++#define BFQ_RATE_REF_INTERVAL	NSEC_PER_SEC
+ 
+ /* Shift used for peak rate fixed precision calculations. */
+ #define BFQ_RATE_SHIFT		16
+@@ -141,16 +154,24 @@ struct kmem_cache *bfq_pool;
   * The device's speed class is dynamically (re)detected in
   * bfq_update_peak_rate() every time the estimated peak rate is updated.
   *
@@ -988,7 +1672,26 @@ index d1f648d..3bc1f8b 100644
  /*
   * To improve readability, a conversion function is used to initialize the
   * following arrays, which entails that they can be initialized only in a
-@@ -410,11 +418,7 @@ static bool bfq_differentiated_weights(struct bfq_data *bfqd)
+@@ -178,18 +199,6 @@ static void bfq_schedule_dispatch(struct bfq_data *bfqd);
+ #define bfq_sample_valid(samples)	((samples) > 80)
+ 
+ /*
+- * We regard a request as SYNC, if either it's a read or has the SYNC bit
+- * set (in which case it could also be a direct WRITE).
+- */
+-static int bfq_bio_sync(struct bio *bio)
+-{
+-	if (bio_data_dir(bio) == READ || (bio->bi_rw & REQ_SYNC))
+-		return 1;
+-
+-	return 0;
+-}
+-
+-/*
+  * Scheduler run of queue, if there are requests pending and no one in the
+  * driver that will restart queueing.
+  */
+@@ -409,11 +418,7 @@ static bool bfq_differentiated_weights(struct bfq_data *bfqd)
   */
  static bool bfq_symmetric_scenario(struct bfq_data *bfqd)
  {
@@ -1001,7 +1704,54 @@ index d1f648d..3bc1f8b 100644
  }
  
  /*
-@@ -534,9 +538,19 @@ static struct request *bfq_find_next_rq(struct bfq_data *bfqd,
+@@ -505,13 +510,45 @@ static void bfq_weights_tree_remove(struct bfq_data *bfqd,
+ 	entity->weight_counter = NULL;
+ }
+ 
++/*
++ * Return expired entry, or NULL to just start from scratch in rbtree.
++ */
++static struct request *bfq_check_fifo(struct bfq_queue *bfqq,
++				      struct request *last)
++{
++	struct request *rq;
++
++	if (bfq_bfqq_fifo_expire(bfqq))
++		return NULL;
++
++	bfq_mark_bfqq_fifo_expire(bfqq);
++
++	rq = rq_entry_fifo(bfqq->fifo.next);
++
++	if (rq == last || ktime_get_ns() < rq->fifo_time)
++		return NULL;
++
++	bfq_log_bfqq(bfqq->bfqd, bfqq, "check_fifo: returned %p", rq);
++	BUG_ON(RB_EMPTY_NODE(&rq->rb_node));
++	return rq;
++}
++
+ static struct request *bfq_find_next_rq(struct bfq_data *bfqd,
+ 					struct bfq_queue *bfqq,
+ 					struct request *last)
+ {
+ 	struct rb_node *rbnext = rb_next(&last->rb_node);
+ 	struct rb_node *rbprev = rb_prev(&last->rb_node);
+-	struct request *next = NULL, *prev = NULL;
++	struct request *next, *prev = NULL;
++
++	BUG_ON(list_empty(&bfqq->fifo));
++
++	/* Follow expired path, else get first next available. */
++	next = bfq_check_fifo(bfqq, last);
++	if (next) {
++		BUG_ON(next == last);
++		return next;
++	}
+ 
+ 	BUG_ON(RB_EMPTY_NODE(&last->rb_node));
+ 
+@@ -533,9 +570,19 @@ static struct request *bfq_find_next_rq(struct bfq_data *bfqd,
  static unsigned long bfq_serv_to_charge(struct request *rq,
  					struct bfq_queue *bfqq)
  {
@@ -1024,7 +1774,16 @@ index d1f648d..3bc1f8b 100644
  }
  
  /**
-@@ -591,12 +605,23 @@ static unsigned int bfq_wr_duration(struct bfq_data *bfqd)
+@@ -576,7 +623,7 @@ static void bfq_updated_next_req(struct bfq_data *bfqd,
+ 		entity->budget = new_budget;
+ 		bfq_log_bfqq(bfqd, bfqq, "updated next rq: new budget %lu",
+ 					 new_budget);
+-		bfq_activate_bfqq(bfqd, bfqq);
++		bfq_requeue_bfqq(bfqd, bfqq);
+ 	}
+ }
+ 
+@@ -590,12 +637,23 @@ static unsigned int bfq_wr_duration(struct bfq_data *bfqd)
  	dur = bfqd->RT_prod;
  	do_div(dur, bfqd->peak_rate);
  
@@ -1046,14 +1805,14 @@ index d1f648d..3bc1f8b 100644
 +	else if (dur < msecs_to_jiffies(3000))
 +		dur = msecs_to_jiffies(3000);
  
--static unsigned bfq_bfqq_cooperations(struct bfq_queue *bfqq)
+-static unsigned int bfq_bfqq_cooperations(struct bfq_queue *bfqq)
 -{
 -	return bfqq->bic ? bfqq->bic->cooperations : 0;
 +	return dur;
  }
  
  static void
-@@ -606,31 +631,11 @@ bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_io_cq *bic)
+@@ -605,31 +663,31 @@ bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_io_cq *bic)
  		bfq_mark_bfqq_idle_window(bfqq);
  	else
  		bfq_clear_bfqq_idle_window(bfqq);
@@ -1076,17 +1835,36 @@ index d1f648d..3bc1f8b 100644
 -		bfqq->wr_cur_max_time = bic->wr_time_left;
 -		bfqq->last_wr_start_finish = jiffies;
 -		bfqq->entity.prio_changed = 1;
--	}
++
++	bfqq->wr_coeff = bic->saved_wr_coeff;
++	bfqq->wr_start_at_switch_to_srt = bic->saved_wr_start_at_switch_to_srt;
++	BUG_ON(time_is_after_jiffies(bfqq->wr_start_at_switch_to_srt));
++	bfqq->last_wr_start_finish = bic->saved_last_wr_start_finish;
++	bfqq->wr_cur_max_time = bic->saved_wr_cur_max_time;
++	BUG_ON(time_is_after_jiffies(bfqq->last_wr_start_finish));
++
++	if (bfqq->wr_coeff > 1 && (bfq_bfqq_in_large_burst(bfqq) ||
++	    time_is_before_jiffies(bfqq->last_wr_start_finish +
++				   bfqq->wr_cur_max_time))) {
++		bfq_log_bfqq(bfqq->bfqd, bfqq,
++			     "resume state: switching off wr (%lu + %lu < %lu)",
++			     bfqq->last_wr_start_finish, bfqq->wr_cur_max_time,
++			     jiffies);
++
++		bfqq->wr_coeff = 1;
+ 	}
 -	/*
 -	 * Clear wr_time_left to prevent bfq_bfqq_save_state() from
 -	 * getting confused about the queue's need of a weight-raising
 -	 * period.
 -	 */
 -	bic->wr_time_left = 0;
++	/* make sure weight will be updated, however we got here */
++	bfqq->entity.prio_changed = 1;
  }
  
  static int bfqq_process_refs(struct bfq_queue *bfqq)
-@@ -640,7 +645,7 @@ static int bfqq_process_refs(struct bfq_queue *bfqq)
+@@ -639,7 +697,7 @@ static int bfqq_process_refs(struct bfq_queue *bfqq)
  	lockdep_assert_held(bfqq->bfqd->queue->queue_lock);
  
  	io_refs = bfqq->allocated[READ] + bfqq->allocated[WRITE];
@@ -1095,7 +1873,7 @@ index d1f648d..3bc1f8b 100644
  	BUG_ON(process_refs < 0);
  	return process_refs;
  }
-@@ -655,6 +660,7 @@ static void bfq_reset_burst_list(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+@@ -654,6 +712,7 @@ static void bfq_reset_burst_list(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  		hlist_del_init(&item->burst_list_node);
  	hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list);
  	bfqd->burst_size = 1;
@@ -1103,7 +1881,7 @@ index d1f648d..3bc1f8b 100644
  }
  
  /* Add bfqq to the list of queues in current burst (see bfq_handle_burst) */
-@@ -663,6 +669,10 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+@@ -662,6 +721,10 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  	/* Increment burst size to take into account also bfqq */
  	bfqd->burst_size++;
  
@@ -1114,7 +1892,7 @@ index d1f648d..3bc1f8b 100644
  	if (bfqd->burst_size == bfqd->bfq_large_burst_thresh) {
  		struct bfq_queue *pos, *bfqq_item;
  		struct hlist_node *n;
-@@ -672,15 +682,19 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+@@ -671,15 +734,19 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  		 * other to consider this burst as large.
  		 */
  		bfqd->large_burst = true;
@@ -1127,7 +1905,7 @@ index d1f648d..3bc1f8b 100644
  		hlist_for_each_entry(bfqq_item, &bfqd->burst_list,
 -				     burst_list_node)
 +				     burst_list_node) {
- 		        bfq_mark_bfqq_in_large_burst(bfqq_item);
+ 			bfq_mark_bfqq_in_large_burst(bfqq_item);
 +			bfq_log_bfqq(bfqd, bfqq_item, "marked in large burst");
 +		}
  		bfq_mark_bfqq_in_large_burst(bfqq);
@@ -1135,7 +1913,7 @@ index d1f648d..3bc1f8b 100644
  
  		/*
  		 * From now on, and until the current burst finishes, any
-@@ -692,67 +706,79 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+@@ -691,67 +758,79 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  		hlist_for_each_entry_safe(pos, n, &bfqd->burst_list,
  					  burst_list_node)
  			hlist_del_init(&pos->burst_list_node);
@@ -1249,7 +2027,8 @@ index d1f648d..3bc1f8b 100644
 + * enjoy weight raising as expected. Fortunately these false positives
 + * are very rare. They typically occur if some service happens to
 + * start doing I/O exactly when the interactive task starts.
-+ *
+  *
+- * . when the very first queue is activated, the queue is inserted into the
 + * Turning back to the next function, it implements all the steps
 + * needed to detect the occurrence of a large burst and to properly
 + * mark all the queues belonging to it (so that they can then be
@@ -1258,13 +2037,12 @@ index d1f648d..3bc1f8b 100644
 + * burst in progress. The list is then used to mark these queues as
 + * belonging to a large burst if the burst does become large. The main
 + * steps are the following.
-  *
-- * . when the very first queue is activated, the queue is inserted into the
++ *
 + * . when the very first queue is created, the queue is inserted into the
   *   list (as it could be the first queue in a possible burst)
   *
   * . if the current burst has not yet become large, and a queue Q that does
-@@ -773,13 +799,13 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+@@ -772,13 +851,13 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq)
   *
   *     . the device enters a large-burst mode
   *
@@ -1280,7 +2058,7 @@ index d1f648d..3bc1f8b 100644
   *   later, i.e., not shortly after, than the last time at which a queue
   *   either entered the burst list or was marked as belonging to the
   *   current large burst, then the current burst is deemed as finished and:
-@@ -792,52 +818,44 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+@@ -791,52 +870,44 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq)
   *          in a possible new burst (then the burst list contains just Q
   *          after this step).
   */
@@ -1357,7 +2135,7 @@ index d1f648d..3bc1f8b 100644
  	}
  
  	/*
-@@ -846,8 +864,9 @@ static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -845,8 +916,9 @@ static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  	 * bfqq as belonging to this large burst immediately.
  	 */
  	if (bfqd->large_burst) {
@@ -1368,7 +2146,7 @@ index d1f648d..3bc1f8b 100644
  	}
  
  	/*
-@@ -856,25 +875,498 @@ static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -855,25 +927,490 @@ static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  	 * queue. Then we add bfqq to the burst.
  	 */
  	bfq_add_to_burst(bfqd, bfqq);
@@ -1388,6 +2166,7 @@ index d1f648d..3bc1f8b 100644
 +static int bfq_bfqq_budget_left(struct bfq_queue *bfqq)
 +{
 +	struct bfq_entity *entity = &bfqq->entity;
++
 +	return entity->budget - entity->service;
 +}
 +
@@ -1550,6 +2329,7 @@ index d1f648d..3bc1f8b 100644
 +		 * operation, is reset only when bfqq is selected for
 +		 * service (see bfq_get_next_queue).
 +		 */
++		BUG_ON(bfqq->max_budget < 0);
 +		entity->budget = min_t(unsigned long,
 +				       bfq_bfqq_budget_left(bfqq),
 +				       bfqq->max_budget);
@@ -1558,8 +2338,9 @@ index d1f648d..3bc1f8b 100644
 +		return true;
 +	}
 +
++	BUG_ON(bfqq->max_budget < 0);
 +	entity->budget = max_t(unsigned long, bfqq->max_budget,
-+			       bfq_serv_to_charge(bfqq->next_rq,bfqq));
++			       bfq_serv_to_charge(bfqq->next_rq, bfqq));
 +	BUG_ON(entity->budget < 0);
 +
 +	bfq_clear_bfqq_non_blocking_wait_rq(bfqq);
@@ -1580,6 +2361,7 @@ index d1f648d..3bc1f8b 100644
 +			bfqq->wr_coeff = bfqd->bfq_wr_coeff;
 +			bfqq->wr_cur_max_time = bfq_wr_duration(bfqd);
 +		} else {
++			bfqq->wr_start_at_switch_to_srt = jiffies;
 +			bfqq->wr_coeff = bfqd->bfq_wr_coeff *
 +				BFQ_SOFTRT_WEIGHT_FACTOR;
 +			bfqq->wr_cur_max_time =
@@ -1613,32 +2395,13 @@ index d1f648d..3bc1f8b 100644
 +				     jiffies,
 +				     jiffies_to_msecs(bfqq->
 +						      wr_cur_max_time));
-+		} else if (time_before(
-+				   bfqq->last_wr_start_finish +
-+				   bfqq->wr_cur_max_time,
-+				   jiffies +
-+				   bfqd->bfq_wr_rt_max_time) &&
-+			   soft_rt) {
++		} else if (soft_rt) {
 +			/*
-+			 * The remaining weight-raising time is lower
-+			 * than bfqd->bfq_wr_rt_max_time, which means
-+			 * that the application is enjoying weight
-+			 * raising either because deemed soft-rt in
-+			 * the near past, or because deemed interactive
-+			 * a long ago.
-+			 * In both cases, resetting now the current
-+			 * remaining weight-raising time for the
-+			 * application to the weight-raising duration
-+			 * for soft rt applications would not cause any
-+			 * latency increase for the application (as the
-+			 * new duration would be higher than the
-+			 * remaining time).
-+			 *
-+			 * In addition, the application is now meeting
-+			 * the requirements for being deemed soft rt.
-+			 * In the end we can correctly and safely
-+			 * (re)charge the weight-raising duration for
-+			 * the application with the weight-raising
++			 * The application is now or still meeting the
++			 * requirements for being deemed soft rt.  We
++			 * can then correctly and safely (re)charge
++			 * the weight-raising duration for the
++			 * application with the weight-raising
 +			 * duration for soft rt applications.
 +			 *
 +			 * In particular, doing this recharge now, i.e.,
@@ -1662,14 +2425,22 @@ index d1f648d..3bc1f8b 100644
 +			 *    latency because the application is not
 +			 *    weight-raised while they are pending.
 +			 */
++			if (bfqq->wr_cur_max_time !=
++				bfqd->bfq_wr_rt_max_time) {
++				bfqq->wr_start_at_switch_to_srt =
++					bfqq->last_wr_start_finish;
++                BUG_ON(time_is_after_jiffies(bfqq->last_wr_start_finish));
++
++				bfqq->wr_cur_max_time =
++					bfqd->bfq_wr_rt_max_time;
++				bfqq->wr_coeff = bfqd->bfq_wr_coeff *
++					BFQ_SOFTRT_WEIGHT_FACTOR;
++				bfq_log_bfqq(bfqd, bfqq,
++					     "switching to soft_rt wr");
++			} else
++				bfq_log_bfqq(bfqd, bfqq,
++					"moving forward soft_rt wr duration");
 +			bfqq->last_wr_start_finish = jiffies;
-+			bfqq->wr_cur_max_time =
-+				bfqd->bfq_wr_rt_max_time;
-+			bfqq->wr_coeff = bfqd->bfq_wr_coeff *
-+				BFQ_SOFTRT_WEIGHT_FACTOR;
-+			bfq_log_bfqq(bfqd, bfqq,
-+				     "switching to soft_rt wr, or "
-+				     " just moving forward duration");
 +		}
 +	}
 +}
@@ -1697,9 +2468,9 @@ index d1f648d..3bc1f8b 100644
 +		 * bfq_bfqq_update_budg_for_activation for
 +		 * details on the usage of the next variable.
 +		 */
-+		arrived_in_time = time_is_after_jiffies(
++		arrived_in_time =  ktime_get_ns() <=
 +			RQ_BIC(rq)->ttime.last_end_request +
-+			bfqd->bfq_slice_idle * 3);
++			bfqd->bfq_slice_idle * 3;
 +
 +	bfq_log_bfqq(bfqd, bfqq,
 +		     "bfq_add_request non-busy: "
@@ -1713,8 +2484,7 @@ index d1f648d..3bc1f8b 100644
 +	BUG_ON(bfqq->entity.budget < bfqq->entity.service);
 +
 +	BUG_ON(bfqq == bfqd->in_service_queue);
-+	bfqg_stats_update_io_add(bfqq_group(RQ_BFQQ(rq)), bfqq,
-+				 rq->cmd_flags);
++	bfqg_stats_update_io_add(bfqq_group(RQ_BFQQ(rq)), bfqq, rq->cmd_flags);
 +
 +	/*
 +	 * bfqq deserves to be weight-raised if:
@@ -1871,7 +2641,7 @@ index d1f648d..3bc1f8b 100644
  	 */
  	prev = bfqq->next_rq;
  	next_rq = bfq_choose_req(bfqd, bfqq->next_rq, rq, bfqd->last_position);
-@@ -887,160 +1379,10 @@ static void bfq_add_request(struct request *rq)
+@@ -886,160 +1423,10 @@ static void bfq_add_request(struct request *rq)
  	if (prev != bfqq->next_rq)
  		bfq_pos_tree_add_move(bfqd, bfqq);
  
@@ -2036,7 +2806,7 @@ index d1f648d..3bc1f8b 100644
  		if (bfqd->low_latency && old_wr_coeff == 1 && !rq_is_sync(rq) &&
  		    time_is_before_jiffies(
  				bfqq->last_wr_start_finish +
-@@ -1049,16 +1391,43 @@ add_bfqq_busy:
+@@ -1048,16 +1435,43 @@ static void bfq_add_request(struct request *rq)
  			bfqq->wr_cur_max_time = bfq_wr_duration(bfqd);
  
  			bfqd->wr_busy_queues++;
@@ -2084,7 +2854,49 @@ index d1f648d..3bc1f8b 100644
  	if (bfqd->low_latency &&
  		(old_wr_coeff == 1 || bfqq->wr_coeff == 1 || interactive))
  		bfqq->last_wr_start_finish = jiffies;
-@@ -1106,6 +1475,9 @@ static void bfq_remove_request(struct request *rq)
+@@ -1074,22 +1488,32 @@ static struct request *bfq_find_rq_fmerge(struct bfq_data *bfqd,
+ 	if (!bic)
+ 		return NULL;
+ 
+-	bfqq = bic_to_bfqq(bic, bfq_bio_sync(bio));
++	bfqq = bic_to_bfqq(bic, op_is_sync(bio->bi_opf));
+ 	if (bfqq)
+ 		return elv_rb_find(&bfqq->sort_list, bio_end_sector(bio));
+ 
+ 	return NULL;
+ }
+ 
+-static void bfq_activate_request(struct request_queue *q, struct request *rq)
++static sector_t get_sdist(sector_t last_pos, struct request *rq)
+ {
+-	struct bfq_data *bfqd = q->elevator->elevator_data;
+-
+-	bfqd->rq_in_driver++;
+-	bfqd->last_position = blk_rq_pos(rq) + blk_rq_sectors(rq);
+-	bfq_log(bfqd, "activate_request: new bfqd->last_position %llu",
+-		(unsigned long long) bfqd->last_position);
+-}
++	sector_t sdist = 0;
++
++	if (last_pos) {
++		if (last_pos < blk_rq_pos(rq))
++			sdist = blk_rq_pos(rq) - last_pos;
++		else
++			sdist = last_pos - blk_rq_pos(rq);
++	}
++
++	return sdist;
++}
++
++static void bfq_activate_request(struct request_queue *q, struct request *rq)
++{
++	struct bfq_data *bfqd = q->elevator->elevator_data;
++	bfqd->rq_in_driver++;
++}
+ 
+ static void bfq_deactivate_request(struct request_queue *q, struct request *rq)
+ {
+@@ -1105,6 +1529,9 @@ static void bfq_remove_request(struct request *rq)
  	struct bfq_data *bfqd = bfqq->bfqd;
  	const int sync = rq_is_sync(rq);
  
@@ -2094,16 +2906,18 @@ index d1f648d..3bc1f8b 100644
  	if (bfqq->next_rq == rq) {
  		bfqq->next_rq = bfq_find_next_rq(bfqd, bfqq, rq);
  		bfq_updated_next_req(bfqd, bfqq);
-@@ -1119,8 +1491,25 @@ static void bfq_remove_request(struct request *rq)
+@@ -1118,8 +1545,26 @@ static void bfq_remove_request(struct request *rq)
  	elv_rb_del(&bfqq->sort_list, rq);
  
  	if (RB_EMPTY_ROOT(&bfqq->sort_list)) {
 -		if (bfq_bfqq_busy(bfqq) && bfqq != bfqd->in_service_queue)
+-			bfq_del_bfqq_busy(bfqd, bfqq, 1);
++		bfqq->next_rq = NULL;
++
 +		BUG_ON(bfqq->entity.budget < 0);
 +
 +		if (bfq_bfqq_busy(bfqq) && bfqq != bfqd->in_service_queue) {
- 			bfq_del_bfqq_busy(bfqd, bfqq, 1);
-+
++			bfq_del_bfqq_busy(bfqd, bfqq, false);
 +			/* bfqq emptied. In normal operation, when
 +			 * bfqq is empty, bfqq->entity.service and
 +			 * bfqq->entity.budget must contain,
@@ -2121,7 +2935,7 @@ index d1f648d..3bc1f8b 100644
  		/*
  		 * Remove queue from request-position tree as it is empty.
  		 */
-@@ -1134,9 +1523,7 @@ static void bfq_remove_request(struct request *rq)
+@@ -1133,9 +1578,7 @@ static void bfq_remove_request(struct request *rq)
  		BUG_ON(bfqq->meta_pending == 0);
  		bfqq->meta_pending--;
  	}
@@ -2131,7 +2945,34 @@ index d1f648d..3bc1f8b 100644
  }
  
  static int bfq_merge(struct request_queue *q, struct request **req,
-@@ -1221,21 +1608,25 @@ static void bfq_merged_requests(struct request_queue *q, struct request *rq,
+@@ -1145,7 +1588,7 @@ static int bfq_merge(struct request_queue *q, struct request **req,
+ 	struct request *__rq;
+ 
+ 	__rq = bfq_find_rq_fmerge(bfqd, bio);
+-	if (__rq && elv_rq_merge_ok(__rq, bio)) {
++	if (__rq && elv_bio_merge_ok(__rq, bio)) {
+ 		*req = __rq;
+ 		return ELEVATOR_FRONT_MERGE;
+ 	}
+@@ -1190,7 +1633,7 @@ static void bfq_merged_request(struct request_queue *q, struct request *req,
+ static void bfq_bio_merged(struct request_queue *q, struct request *req,
+ 			   struct bio *bio)
+ {
+-	bfqg_stats_update_io_merged(bfqq_group(RQ_BFQQ(req)), bio->bi_rw);
++	bfqg_stats_update_io_merged(bfqq_group(RQ_BFQQ(req)), bio->bi_opf);
+ }
+ #endif
+ 
+@@ -1210,7 +1653,7 @@ static void bfq_merged_requests(struct request_queue *q, struct request *rq,
+ 	 */
+ 	if (bfqq == next_bfqq &&
+ 	    !list_empty(&rq->queuelist) && !list_empty(&next->queuelist) &&
+-	    time_before(next->fifo_time, rq->fifo_time)) {
++	    next->fifo_time < rq->fifo_time) {
+ 		list_del_init(&rq->queuelist);
+ 		list_replace_init(&next->queuelist, &rq->queuelist);
+ 		rq->fifo_time = next->fifo_time;
+@@ -1220,21 +1663,30 @@ static void bfq_merged_requests(struct request_queue *q, struct request *rq,
  		bfqq->next_rq = rq;
  
  	bfq_remove_request(next);
@@ -2150,17 +2991,22 @@ index d1f648d..3bc1f8b 100644
  	bfqq->wr_coeff = 1;
  	bfqq->wr_cur_max_time = 0;
 -	/* Trigger a weight change on the next activation of the queue */
++	bfqq->last_wr_start_finish = jiffies;
 +	/*
 +	 * Trigger a weight change on the next invocation of
 +	 * __bfq_entity_update_weight_prio.
 +	 */
  	bfqq->entity.prio_changed = 1;
++	bfq_log_bfqq(bfqq->bfqd, bfqq,
++		     "end_wr: wrais ending at %lu, rais_max_time %u",
++		     bfqq->last_wr_start_finish,
++		     jiffies_to_msecs(bfqq->wr_cur_max_time));
 +	bfq_log_bfqq(bfqq->bfqd, bfqq, "end_wr: wr_busy %d",
 +		     bfqq->bfqd->wr_busy_queues);
  }
  
  static void bfq_end_wr_async_queues(struct bfq_data *bfqd,
-@@ -1278,7 +1669,7 @@ static int bfq_rq_close_to_sector(void *io_struct, bool request,
+@@ -1277,7 +1729,7 @@ static int bfq_rq_close_to_sector(void *io_struct, bool request,
  				  sector_t sector)
  {
  	return abs(bfq_io_struct_pos(io_struct, request) - sector) <=
@@ -2169,7 +3015,7 @@ index d1f648d..3bc1f8b 100644
  }
  
  static struct bfq_queue *bfqq_find_close(struct bfq_data *bfqd,
-@@ -1400,7 +1791,7 @@ bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq)
+@@ -1399,7 +1851,7 @@ bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq)
  	 * throughput.
  	 */
  	bfqq->new_bfqq = new_bfqq;
@@ -2178,7 +3024,7 @@ index d1f648d..3bc1f8b 100644
  	return new_bfqq;
  }
  
-@@ -1431,9 +1822,23 @@ static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq,
+@@ -1430,9 +1882,23 @@ static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq,
  }
  
  /*
@@ -2191,7 +3037,7 @@ index d1f648d..3bc1f8b 100644
 + * positives. In case bfqq is weight-raised, such false positives
 + * would evidently degrade latency guarantees for bfqq.
 + */
-+bool wr_from_too_long(struct bfq_queue *bfqq)
++static bool wr_from_too_long(struct bfq_queue *bfqq)
 +{
 +	return bfqq->wr_coeff > 1 &&
 +		time_is_before_jiffies(bfqq->last_wr_start_finish +
@@ -2205,7 +3051,7 @@ index d1f648d..3bc1f8b 100644
   * structure otherwise.
   *
   * The OOM queue is not allowed to participate to cooperation: in fact, since
-@@ -1442,6 +1847,18 @@ static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq,
+@@ -1441,6 +1907,18 @@ static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq,
   * handle merging with the OOM queue would be quite complex and expensive
   * to maintain. Besides, in such a critical condition as an out of memory,
   * the benefits of queue merging may be little relevant, or even negligible.
@@ -2224,7 +3070,7 @@ index d1f648d..3bc1f8b 100644
   */
  static struct bfq_queue *
  bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,
-@@ -1451,16 +1868,32 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -1450,16 +1928,32 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  
  	if (bfqq->new_bfqq)
  		return bfqq->new_bfqq;
@@ -2260,7 +3106,7 @@ index d1f648d..3bc1f8b 100644
  	    unlikely(in_service_bfqq == &bfqd->oom_bfqq))
  		goto check_scheduled;
  
-@@ -1482,7 +1915,15 @@ check_scheduled:
+@@ -1481,7 +1975,15 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  
  	BUG_ON(new_bfqq && bfqq->entity.parent != new_bfqq->entity.parent);
  
@@ -2277,9 +3123,19 @@ index d1f648d..3bc1f8b 100644
  	    bfq_may_be_close_cooperator(bfqq, new_bfqq))
  		return bfq_setup_merge(bfqq, new_bfqq);
  
-@@ -1498,46 +1939,11 @@ static void bfq_bfqq_save_state(struct bfq_queue *bfqq)
+@@ -1490,53 +1992,25 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+ 
+ static void bfq_bfqq_save_state(struct bfq_queue *bfqq)
+ {
++	struct bfq_io_cq *bic = bfqq->bic;
++
+ 	/*
+ 	 * If !bfqq->bic, the queue is already shared or its requests
+ 	 * have already been redirected to a shared queue; both idle window
+ 	 * and weight raising state have already been saved. Do nothing.
  	 */
- 	if (!bfqq->bic)
+-	if (!bfqq->bic)
++	if (!bic)
  		return;
 -	if (bfqq->bic->wr_time_left)
 -		/*
@@ -2315,17 +3171,26 @@ index d1f648d..3bc1f8b 100644
 -		bfq_bfqq_end_wr(bfqq);
 -	} else
 -		bfqq->bic->wr_time_left = 0;
-+
- 	bfqq->bic->saved_idle_window = bfq_bfqq_idle_window(bfqq);
- 	bfqq->bic->saved_IO_bound = bfq_bfqq_IO_bound(bfqq);
- 	bfqq->bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq);
- 	bfqq->bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node);
+-	bfqq->bic->saved_idle_window = bfq_bfqq_idle_window(bfqq);
+-	bfqq->bic->saved_IO_bound = bfq_bfqq_IO_bound(bfqq);
+-	bfqq->bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq);
+-	bfqq->bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node);
 -	bfqq->bic->cooperations++;
 -	bfqq->bic->failed_cooperations = 0;
++
++	bic->saved_idle_window = bfq_bfqq_idle_window(bfqq);
++	bic->saved_IO_bound = bfq_bfqq_IO_bound(bfqq);
++	bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq);
++	bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node);
++	bic->saved_wr_coeff = bfqq->wr_coeff;
++	bic->saved_wr_start_at_switch_to_srt = bfqq->wr_start_at_switch_to_srt;
++	bic->saved_last_wr_start_finish = bfqq->last_wr_start_finish;
++	bic->saved_wr_cur_max_time = bfqq->wr_cur_max_time;
++	BUG_ON(time_is_after_jiffies(bfqq->last_wr_start_finish));
  }
  
  static void bfq_get_bic_reference(struct bfq_queue *bfqq)
-@@ -1562,6 +1968,40 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic,
+@@ -1561,6 +2035,40 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic,
  	if (bfq_bfqq_IO_bound(bfqq))
  		bfq_mark_bfqq_IO_bound(new_bfqq);
  	bfq_clear_bfqq_IO_bound(bfqq);
@@ -2343,12 +3208,12 @@ index d1f648d..3bc1f8b 100644
 +		new_bfqq->wr_coeff = bfqq->wr_coeff;
 +		new_bfqq->wr_cur_max_time = bfqq->wr_cur_max_time;
 +		new_bfqq->last_wr_start_finish = bfqq->last_wr_start_finish;
++		new_bfqq->wr_start_at_switch_to_srt = bfqq->wr_start_at_switch_to_srt;
 +		if (bfq_bfqq_busy(new_bfqq))
-+		    bfqd->wr_busy_queues++;
++			bfqd->wr_busy_queues++;
 +		new_bfqq->entity.prio_changed = 1;
 +		bfq_log_bfqq(bfqd, new_bfqq,
-+			     "wr starting after merge with %d, "
-+			     "rais_max_time %u",
++			     "wr start after merge with %d, rais_max_time %u",
 +			     bfqq->pid,
 +			     jiffies_to_msecs(bfqq->wr_cur_max_time));
 +	}
@@ -2366,7 +3231,7 @@ index d1f648d..3bc1f8b 100644
  	/*
  	 * Grab a reference to the bic, to prevent it from being destroyed
  	 * before being possibly touched by a bfq_split_bfqq().
-@@ -1588,18 +2028,6 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic,
+@@ -1587,30 +2095,19 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic,
  	bfq_put_queue(bfqq);
  }
  
@@ -2382,10 +3247,39 @@ index d1f648d..3bc1f8b 100644
 -	}
 -}
 -
- static int bfq_allow_merge(struct request_queue *q, struct request *rq,
- 			   struct bio *bio)
+-static int bfq_allow_merge(struct request_queue *q, struct request *rq,
+-			   struct bio *bio)
++static int bfq_allow_bio_merge(struct request_queue *q, struct request *rq,
++			       struct bio *bio)
  {
-@@ -1637,30 +2065,86 @@ static int bfq_allow_merge(struct request_queue *q, struct request *rq,
+ 	struct bfq_data *bfqd = q->elevator->elevator_data;
++	bool is_sync = op_is_sync(bio->bi_opf);
+ 	struct bfq_io_cq *bic;
+ 	struct bfq_queue *bfqq, *new_bfqq;
+ 
+ 	/*
+ 	 * Disallow merge of a sync bio into an async request.
+ 	 */
+-	if (bfq_bio_sync(bio) && !rq_is_sync(rq))
+-		return 0;
++	if (is_sync && !rq_is_sync(rq))
++		return false;
+ 
+ 	/*
+ 	 * Lookup the bfqq that this bio will be queued with. Allow
+@@ -1619,9 +2116,9 @@ static int bfq_allow_merge(struct request_queue *q, struct request *rq,
+ 	 */
+ 	bic = bfq_bic_lookup(bfqd, current->io_context);
+ 	if (!bic)
+-		return 0;
++		return false;
+ 
+-	bfqq = bic_to_bfqq(bic, bfq_bio_sync(bio));
++	bfqq = bic_to_bfqq(bic, is_sync);
+ 	/*
+ 	 * We take advantage of this function to perform an early merge
+ 	 * of the queues of possible cooperating processes.
+@@ -1636,30 +2133,111 @@ static int bfq_allow_merge(struct request_queue *q, struct request *rq,
  			 * to decide whether bio and rq can be merged.
  			 */
  			bfqq = new_bfqq;
@@ -2397,6 +3291,12 @@ index d1f648d..3bc1f8b 100644
  	return bfqq == RQ_BFQQ(rq);
  }
  
++static int bfq_allow_rq_merge(struct request_queue *q, struct request *rq,
++			      struct request *next)
++{
++	return RQ_BFQQ(rq) == RQ_BFQQ(next);
++}
++
 +/*
 + * Set the maximum time for the in-service queue to consume its
 + * budget. This prevents seeky processes from lowering the throughput.
@@ -2407,6 +3307,7 @@ index d1f648d..3bc1f8b 100644
 +				   struct bfq_queue *bfqq)
 +{
 +	unsigned int timeout_coeff;
++
 +	if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time)
 +		timeout_coeff = 1;
 +	else
@@ -2437,9 +3338,10 @@ index d1f648d..3bc1f8b 100644
 +		BUG_ON(bfqq == bfqd->in_service_queue);
 +		BUG_ON(RB_EMPTY_ROOT(&bfqq->sort_list));
 +
-+		if (bfqq->wr_coeff > 1 &&
++		if (time_is_before_jiffies(bfqq->last_wr_start_finish) &&
++		    bfqq->wr_coeff > 1 &&
 +		    bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time &&
-+			time_is_before_jiffies(bfqq->budget_timeout)) {
++		    time_is_before_jiffies(bfqq->budget_timeout)) {
 +			/*
 +			 * For soft real-time queues, move the start
 +			 * of the weight-raising period forward by the
@@ -2464,8 +3366,25 @@ index d1f648d..3bc1f8b 100644
 +			 * not only expires, but also remains with no
 +			 * request.
 +			 */
-+			bfqq->last_wr_start_finish += jiffies -
-+				bfqq->budget_timeout;
++			if (time_after(bfqq->budget_timeout,
++				       bfqq->last_wr_start_finish))
++				bfqq->last_wr_start_finish +=
++					jiffies - bfqq->budget_timeout;
++			else
++				bfqq->last_wr_start_finish = jiffies;
++
++			if (time_is_after_jiffies(bfqq->last_wr_start_finish)) {
++			       pr_crit(
++			       "BFQ WARNING:last %lu budget %lu jiffies %lu",
++			       bfqq->last_wr_start_finish,
++			       bfqq->budget_timeout,
++			       jiffies);
++			       pr_crit("diff %lu", jiffies -
++				       max_t(unsigned long,
++					     bfqq->last_wr_start_finish,
++					     bfqq->budget_timeout));
++			       bfqq->last_wr_start_finish = jiffies;
++			}
 +		}
 +
 +		bfq_set_budget_timeout(bfqd, bfqq);
@@ -2478,7 +3397,7 @@ index d1f648d..3bc1f8b 100644
  
  	bfqd->in_service_queue = bfqq;
  }
-@@ -1676,31 +2160,6 @@ static struct bfq_queue *bfq_set_in_service_queue(struct bfq_data *bfqd)
+@@ -1675,36 +2253,11 @@ static struct bfq_queue *bfq_set_in_service_queue(struct bfq_data *bfqd)
  	return bfqq;
  }
  
@@ -2510,58 +3429,19 @@ index d1f648d..3bc1f8b 100644
  static void bfq_arm_slice_timer(struct bfq_data *bfqd)
  {
  	struct bfq_queue *bfqq = bfqd->in_service_queue;
-@@ -1725,62 +2184,34 @@ static void bfq_arm_slice_timer(struct bfq_data *bfqd)
- 	 * being too ill-treated, grant them a small fraction of the
- 	 * assigned budget before reducing the waiting time to
- 	 * BFQ_MIN_TT. This happened to help reduce latency.
--	 */
--	sl = bfqd->bfq_slice_idle;
--	/*
--	 * Unless the queue is being weight-raised or the scenario is
+ 	struct bfq_io_cq *bic;
+-	unsigned long sl;
++	u32 sl;
+ 
+ 	BUG_ON(!RB_EMPTY_ROOT(&bfqq->sort_list));
+ 
+@@ -1728,119 +2281,366 @@ static void bfq_arm_slice_timer(struct bfq_data *bfqd)
+ 	sl = bfqd->bfq_slice_idle;
+ 	/*
+ 	 * Unless the queue is being weight-raised or the scenario is
 -	 * asymmetric, grant only minimum idle time if the queue either
 -	 * has been seeky for long enough or has already proved to be
 -	 * constantly seeky.
--	 */
--	if (bfq_sample_valid(bfqq->seek_samples) &&
--	    ((BFQQ_SEEKY(bfqq) && bfqq->entity.service >
--				  bfq_max_budget(bfqq->bfqd) / 8) ||
--	      bfq_bfqq_constantly_seeky(bfqq)) && bfqq->wr_coeff == 1 &&
--	    bfq_symmetric_scenario(bfqd))
--		sl = min(sl, msecs_to_jiffies(BFQ_MIN_TT));
--	else if (bfqq->wr_coeff > 1)
--		sl = sl * 3;
--	bfqd->last_idling_start = ktime_get();
--	mod_timer(&bfqd->idle_slice_timer, jiffies + sl);
--#ifdef CONFIG_BFQ_GROUP_IOSCHED
--	bfqg_stats_set_start_idle_time(bfqq_group(bfqq));
--#endif
--	bfq_log(bfqd, "arm idle: %u/%u ms",
--		jiffies_to_msecs(sl), jiffies_to_msecs(bfqd->bfq_slice_idle));
--}
--
--/*
-- * Set the maximum time for the in-service queue to consume its
-- * budget. This prevents seeky processes from lowering the disk
-- * throughput (always guaranteed with a time slice scheme as in CFQ).
-- */
--static void bfq_set_budget_timeout(struct bfq_data *bfqd)
--{
--	struct bfq_queue *bfqq = bfqd->in_service_queue;
--	unsigned int timeout_coeff;
--	if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time)
--		timeout_coeff = 1;
--	else
--		timeout_coeff = bfqq->entity.weight / bfqq->entity.orig_weight;
--
--	bfqd->last_budget_start = ktime_get();
--
--	bfq_clear_bfqq_budget_new(bfqq);
--	bfqq->budget_timeout = jiffies +
--		bfqd->bfq_timeout[bfq_bfqq_sync(bfqq)] * timeout_coeff;
-+	 */
-+	sl = bfqd->bfq_slice_idle;
-+	/*
-+	 * Unless the queue is being weight-raised or the scenario is
 +	 * asymmetric, grant only minimum idle time if the queue
 +	 * is seeky. A long idling is preserved for a weight-raised
 +	 * queue, or, more in general, in an asymemtric scenario,
@@ -2569,69 +3449,432 @@ index d1f648d..3bc1f8b 100644
 +	 * its reserved share of the throughput (in particular, it is
 +	 * needed if the queue has a higher weight than some other
 +	 * queue).
-+	 */
+ 	 */
+-	if (bfq_sample_valid(bfqq->seek_samples) &&
+-	    ((BFQQ_SEEKY(bfqq) && bfqq->entity.service >
+-				  bfq_max_budget(bfqq->bfqd) / 8) ||
+-	      bfq_bfqq_constantly_seeky(bfqq)) && bfqq->wr_coeff == 1 &&
 +	if (BFQQ_SEEKY(bfqq) && bfqq->wr_coeff == 1 &&
-+	    bfq_symmetric_scenario(bfqd))
-+		sl = min(sl, msecs_to_jiffies(BFQ_MIN_TT));
+ 	    bfq_symmetric_scenario(bfqd))
+-		sl = min(sl, msecs_to_jiffies(BFQ_MIN_TT));
+-	else if (bfqq->wr_coeff > 1)
+-		sl = sl * 3;
++		sl = min_t(u32, sl, BFQ_MIN_TT);
++
+ 	bfqd->last_idling_start = ktime_get();
+-	mod_timer(&bfqd->idle_slice_timer, jiffies + sl);
+-#ifdef CONFIG_BFQ_GROUP_IOSCHED
++	hrtimer_start(&bfqd->idle_slice_timer, ns_to_ktime(sl),
++		      HRTIMER_MODE_REL);
+ 	bfqg_stats_set_start_idle_time(bfqq_group(bfqq));
+-#endif
+-	bfq_log(bfqd, "arm idle: %u/%u ms",
+-		jiffies_to_msecs(sl), jiffies_to_msecs(bfqd->bfq_slice_idle));
++	bfq_log(bfqd, "arm idle: %ld/%ld ms",
++		sl / NSEC_PER_MSEC, bfqd->bfq_slice_idle / NSEC_PER_MSEC);
+ }
+ 
+ /*
+- * Set the maximum time for the in-service queue to consume its
+- * budget. This prevents seeky processes from lowering the disk
+- * throughput (always guaranteed with a time slice scheme as in CFQ).
++ * In autotuning mode, max_budget is dynamically recomputed as the
++ * amount of sectors transferred in timeout at the estimated peak
++ * rate. This enables BFQ to utilize a full timeslice with a full
++ * budget, even if the in-service queue is served at peak rate. And
++ * this maximises throughput with sequential workloads.
+  */
+-static void bfq_set_budget_timeout(struct bfq_data *bfqd)
++static unsigned long bfq_calc_max_budget(struct bfq_data *bfqd)
+ {
+-	struct bfq_queue *bfqq = bfqd->in_service_queue;
+-	unsigned int timeout_coeff;
++	return (u64)bfqd->peak_rate * USEC_PER_MSEC *
++		jiffies_to_msecs(bfqd->bfq_timeout)>>BFQ_RATE_SHIFT;
++}
+ 
+-	if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time)
+-		timeout_coeff = 1;
+-	else
+-		timeout_coeff = bfqq->entity.weight / bfqq->entity.orig_weight;
++/*
++ * Update parameters related to throughput and responsiveness, as a
++ * function of the estimated peak rate. See comments on
++ * bfq_calc_max_budget(), and on T_slow and T_fast arrays.
++ */
++static void update_thr_responsiveness_params(struct bfq_data *bfqd)
++{
++	int dev_type = blk_queue_nonrot(bfqd->queue);
++
++	if (bfqd->bfq_user_max_budget == 0) {
++		bfqd->bfq_max_budget =
++			bfq_calc_max_budget(bfqd);
++		BUG_ON(bfqd->bfq_max_budget < 0);
++		bfq_log(bfqd, "new max_budget = %d",
++			bfqd->bfq_max_budget);
++	}
+ 
+-	bfqd->last_budget_start = ktime_get();
++	if (bfqd->device_speed == BFQ_BFQD_FAST &&
++	    bfqd->peak_rate < device_speed_thresh[dev_type]) {
++		bfqd->device_speed = BFQ_BFQD_SLOW;
++		bfqd->RT_prod = R_slow[dev_type] *
++			T_slow[dev_type];
++	} else if (bfqd->device_speed == BFQ_BFQD_SLOW &&
++		   bfqd->peak_rate > device_speed_thresh[dev_type]) {
++		bfqd->device_speed = BFQ_BFQD_FAST;
++		bfqd->RT_prod = R_fast[dev_type] *
++			T_fast[dev_type];
++	}
+ 
+-	bfq_clear_bfqq_budget_new(bfqq);
+-	bfqq->budget_timeout = jiffies +
+-		bfqd->bfq_timeout[bfq_bfqq_sync(bfqq)] * timeout_coeff;
++	bfq_log(bfqd,
++"dev_type %s dev_speed_class = %s (%llu sects/sec), thresh %llu setcs/sec",
++		dev_type == 0 ? "ROT" : "NONROT",
++		bfqd->device_speed == BFQ_BFQD_FAST ? "FAST" : "SLOW",
++		bfqd->device_speed == BFQ_BFQD_FAST ?
++		(USEC_PER_SEC*(u64)R_fast[dev_type])>>BFQ_RATE_SHIFT :
++		(USEC_PER_SEC*(u64)R_slow[dev_type])>>BFQ_RATE_SHIFT,
++		(USEC_PER_SEC*(u64)device_speed_thresh[dev_type])>>
++		BFQ_RATE_SHIFT);
++}
  
 -	bfq_log_bfqq(bfqd, bfqq, "set budget_timeout %u",
 -		jiffies_to_msecs(bfqd->bfq_timeout[bfq_bfqq_sync(bfqq)] *
 -		timeout_coeff));
-+	bfqd->last_idling_start = ktime_get();
-+	mod_timer(&bfqd->idle_slice_timer, jiffies + sl);
-+	bfqg_stats_set_start_idle_time(bfqq_group(bfqq));
-+	bfq_log(bfqd, "arm idle: %u/%u ms",
-+		jiffies_to_msecs(sl), jiffies_to_msecs(bfqd->bfq_slice_idle));
++static void bfq_reset_rate_computation(struct bfq_data *bfqd, struct request *rq)
++{
++	if (rq != NULL) { /* new rq dispatch now, reset accordingly */
++		bfqd->last_dispatch = bfqd->first_dispatch = ktime_get_ns() ;
++		bfqd->peak_rate_samples = 1;
++		bfqd->sequential_samples = 0;
++		bfqd->tot_sectors_dispatched = bfqd->last_rq_max_size =
++			blk_rq_sectors(rq);
++	} else /* no new rq dispatched, just reset the number of samples */
++		bfqd->peak_rate_samples = 0; /* full re-init on next disp. */
++
++	bfq_log(bfqd,
++		"reset_rate_computation at end, sample %u/%u tot_sects %llu",
++		bfqd->peak_rate_samples, bfqd->sequential_samples,
++		bfqd->tot_sectors_dispatched);
  }
  
- /*
+-/*
 - * Move request from internal lists to the request queue dispatch list.
-+ * Move request from internal lists to the dispatch list of the request queue
-  */
- static void bfq_dispatch_insert(struct request_queue *q, struct request *rq)
+- */
+-static void bfq_dispatch_insert(struct request_queue *q, struct request *rq)
++static void bfq_update_rate_reset(struct bfq_data *bfqd, struct request *rq)
  {
 -	struct bfq_data *bfqd = q->elevator->elevator_data;
- 	struct bfq_queue *bfqq = RQ_BFQQ(rq);
+-	struct bfq_queue *bfqq = RQ_BFQQ(rq);
++	u32 rate, weight, divisor;
  
  	/*
-@@ -1794,15 +2225,9 @@ static void bfq_dispatch_insert(struct request_queue *q, struct request *rq)
- 	 * incrementing bfqq->dispatched.
+-	 * For consistency, the next instruction should have been executed
+-	 * after removing the request from the queue and dispatching it.
+-	 * We execute instead this instruction before bfq_remove_request()
+-	 * (and hence introduce a temporary inconsistency), for efficiency.
+-	 * In fact, in a forced_dispatch, this prevents two counters related
+-	 * to bfqq->dispatched to risk to be uselessly decremented if bfqq
+-	 * is not in service, and then to be incremented again after
+-	 * incrementing bfqq->dispatched.
++	 * For the convergence property to hold (see comments on
++	 * bfq_update_peak_rate()) and for the assessment to be
++	 * reliable, a minimum number of samples must be present, and
++	 * a minimum amount of time must have elapsed. If not so, do
++	 * not compute new rate. Just reset parameters, to get ready
++	 * for a new evaluation attempt.
  	 */
- 	bfqq->dispatched++;
-+
- 	bfq_remove_request(rq);
- 	elv_dispatch_sort(q, rq);
--
+-	bfqq->dispatched++;
+-	bfq_remove_request(rq);
+-	elv_dispatch_sort(q, rq);
++	if (bfqd->peak_rate_samples < BFQ_RATE_MIN_SAMPLES ||
++	    bfqd->delta_from_first < BFQ_RATE_MIN_INTERVAL) {
++		bfq_log(bfqd,
++	"update_rate_reset: only resetting, delta_first %lluus samples %d",
++			bfqd->delta_from_first>>10, bfqd->peak_rate_samples);
++		goto reset_computation;
++	}
+ 
 -	if (bfq_bfqq_sync(bfqq))
 -		bfqd->sync_flight++;
 -#ifdef CONFIG_BFQ_GROUP_IOSCHED
 -	bfqg_stats_update_dispatch(bfqq_group(bfqq), blk_rq_bytes(rq),
 -				   rq->cmd_flags);
 -#endif
++	/*
++	 * If a new request completion has occurred after last
++	 * dispatch, then, to approximate the rate at which requests
++	 * have been served by the device, it is more precise to
++	 * extend the observation interval to the last completion.
++	 */
++	bfqd->delta_from_first =
++		max_t(u64, bfqd->delta_from_first,
++		      bfqd->last_completion - bfqd->first_dispatch);
++
++	BUG_ON(bfqd->delta_from_first == 0);
++	/*
++	 * Rate computed in sects/usec, and not sects/nsec, for
++	 * precision issues.
++	 */
++	rate = div64_ul(bfqd->tot_sectors_dispatched<<BFQ_RATE_SHIFT,
++			div_u64(bfqd->delta_from_first, NSEC_PER_USEC));
++
++	bfq_log(bfqd,
++"update_rate_reset: tot_sects %llu delta_first %lluus rate %llu sects/s (%d)",
++		bfqd->tot_sectors_dispatched, bfqd->delta_from_first>>10,
++		((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT),
++		rate > 20<<BFQ_RATE_SHIFT);
++
++	/*
++	 * Peak rate not updated if:
++	 * - the percentage of sequential dispatches is below 3/4 of the
++	 *   total, and rate is below the current estimated peak rate
++	 * - rate is unreasonably high (> 20M sectors/sec)
++	 */
++	if ((bfqd->sequential_samples < (3 * bfqd->peak_rate_samples)>>2 &&
++	     rate <= bfqd->peak_rate) ||
++		rate > 20<<BFQ_RATE_SHIFT) {
++		bfq_log(bfqd,
++		"update_rate_reset: goto reset, samples %u/%u rate/peak %llu/%llu",
++		bfqd->peak_rate_samples, bfqd->sequential_samples,
++		((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT),
++		((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT));
++		goto reset_computation;
++	} else {
++		bfq_log(bfqd,
++		"update_rate_reset: do update, samples %u/%u rate/peak %llu/%llu",
++		bfqd->peak_rate_samples, bfqd->sequential_samples,
++		((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT),
++		((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT));
++	}
++
++	/*
++	 * We have to update the peak rate, at last! To this purpose,
++	 * we use a low-pass filter. We compute the smoothing constant
++	 * of the filter as a function of the 'weight' of the new
++	 * measured rate.
++	 *
++	 * As can be seen in next formulas, we define this weight as a
++	 * quantity proportional to how sequential the workload is,
++	 * and to how long the observation time interval is.
++	 *
++	 * The weight runs from 0 to 8. The maximum value of the
++	 * weight, 8, yields the minimum value for the smoothing
++	 * constant. At this minimum value for the smoothing constant,
++	 * the measured rate contributes for half of the next value of
++	 * the estimated peak rate.
++	 *
++	 * So, the first step is to compute the weight as a function
++	 * of how sequential the workload is. Note that the weight
++	 * cannot reach 9, because bfqd->sequential_samples cannot
++	 * become equal to bfqd->peak_rate_samples, which, in its
++	 * turn, holds true because bfqd->sequential_samples is not
++	 * incremented for the first sample.
++	 */
++	weight = (9 * bfqd->sequential_samples) / bfqd->peak_rate_samples;
++
++	/*
++	 * Second step: further refine the weight as a function of the
++	 * duration of the observation interval.
++	 */
++	weight = min_t(u32, 8,
++		       div_u64(weight * bfqd->delta_from_first,
++			       BFQ_RATE_REF_INTERVAL));
++
++	/*
++	 * Divisor ranging from 10, for minimum weight, to 2, for
++	 * maximum weight.
++	 */
++	divisor = 10 - weight;
++	BUG_ON(divisor == 0);
++
++	/*
++	 * Finally, update peak rate:
++	 *
++	 * peak_rate = peak_rate * (divisor-1) / divisor  +  rate / divisor
++	 */
++	bfqd->peak_rate *= divisor-1;
++	bfqd->peak_rate /= divisor;
++	rate /= divisor; /* smoothing constant alpha = 1/divisor */
++
++	bfq_log(bfqd,
++		"update_rate_reset: divisor %d tmp_peak_rate %llu tmp_rate %u",
++		divisor,
++		((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT),
++		(u32)((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT));
++
++	BUG_ON(bfqd->peak_rate == 0);
++	BUG_ON(bfqd->peak_rate > 20<<BFQ_RATE_SHIFT);
++
++	bfqd->peak_rate += rate;
++	update_thr_responsiveness_params(bfqd);
++	BUG_ON(bfqd->peak_rate > 20<<BFQ_RATE_SHIFT);
++
++reset_computation:
++	bfq_reset_rate_computation(bfqd, rq);
  }
  
  /*
-@@ -1822,18 +2247,12 @@ static struct request *bfq_check_fifo(struct bfq_queue *bfqq)
+- * Return expired entry, or NULL to just start from scratch in rbtree.
++ * Update the read/write peak rate (the main quantity used for
++ * auto-tuning, see update_thr_responsiveness_params()).
++ *
++ * It is not trivial to estimate the peak rate (correctly): because of
++ * the presence of sw and hw queues between the scheduler and the
++ * device components that finally serve I/O requests, it is hard to
++ * say exactly when a given dispatched request is served inside the
++ * device, and for how long. As a consequence, it is hard to know
++ * precisely at what rate a given set of requests is actually served
++ * by the device.
++ *
++ * On the opposite end, the dispatch time of any request is trivially
++ * available, and, from this piece of information, the "dispatch rate"
++ * of requests can be immediately computed. So, the idea in the next
++ * function is to use what is known, namely request dispatch times
++ * (plus, when useful, request completion times), to estimate what is
++ * unknown, namely in-device request service rate.
++ *
++ * The main issue is that, because of the above facts, the rate at
++ * which a certain set of requests is dispatched over a certain time
++ * interval can vary greatly with respect to the rate at which the
++ * same requests are then served. But, since the size of any
++ * intermediate queue is limited, and the service scheme is lossless
++ * (no request is silently dropped), the following obvious convergence
++ * property holds: the number of requests dispatched MUST become
++ * closer and closer to the number of requests completed as the
++ * observation interval grows. This is the key property used in
++ * the next function to estimate the peak service rate as a function
++ * of the observed dispatch rate. The function assumes to be invoked
++ * on every request dispatch.
+  */
+-static struct request *bfq_check_fifo(struct bfq_queue *bfqq)
++static void bfq_update_peak_rate(struct bfq_data *bfqd, struct request *rq)
+ {
+-	struct request *rq = NULL;
++	u64 now_ns = ktime_get_ns();
++
++	if (bfqd->peak_rate_samples == 0) { /* first dispatch */
++		bfq_log(bfqd,
++		"update_peak_rate: goto reset, samples %d",
++				bfqd->peak_rate_samples) ;
++		bfq_reset_rate_computation(bfqd, rq);
++		goto update_last_values; /* will add one sample */
++	}
  
- 	rq = rq_entry_fifo(bfqq->fifo.next);
+-	if (bfq_bfqq_fifo_expire(bfqq))
+-		return NULL;
++	/*
++	 * Device idle for very long: the observation interval lasting
++	 * up to this dispatch cannot be a valid observation interval
++	 * for computing a new peak rate (similarly to the late-
++	 * completion event in bfq_completed_request()). Go to
++	 * update_rate_and_reset to have the following three steps
++	 * taken:
++	 * - close the observation interval at the last (previous)
++	 *   request dispatch or completion
++	 * - compute rate, if possible, for that observation interval
++	 * - start a new observation interval with this dispatch
++	 */
++	if (now_ns - bfqd->last_dispatch > 100*NSEC_PER_MSEC &&
++	    bfqd->rq_in_driver == 0) {
++		bfq_log(bfqd,
++"update_peak_rate: jumping to updating&resetting delta_last %lluus samples %d",
++			(now_ns - bfqd->last_dispatch)>>10,
++			bfqd->peak_rate_samples) ;
++		goto update_rate_and_reset;
++	}
+ 
+-	bfq_mark_bfqq_fifo_expire(bfqq);
++	/* Update sampling information */
++	bfqd->peak_rate_samples++;
+ 
+-	if (list_empty(&bfqq->fifo))
+-		return NULL;
++	if ((bfqd->rq_in_driver > 0 ||
++		now_ns - bfqd->last_completion < BFQ_MIN_TT)
++	     && get_sdist(bfqd->last_position, rq) < BFQQ_SEEK_THR)
++		bfqd->sequential_samples++;
+ 
+-	rq = rq_entry_fifo(bfqq->fifo.next);
++	bfqd->tot_sectors_dispatched += blk_rq_sectors(rq);
  
 -	if (time_before(jiffies, rq->fifo_time))
-+	if (time_is_after_jiffies(rq->fifo_time))
- 		return NULL;
+-		return NULL;
++	/* Reset max observed rq size every 32 dispatches */
++	if (likely(bfqd->peak_rate_samples % 32))
++		bfqd->last_rq_max_size =
++			max_t(u32, blk_rq_sectors(rq), bfqd->last_rq_max_size);
++	else
++		bfqd->last_rq_max_size = blk_rq_sectors(rq);
  
- 	return rq;
+-	return rq;
++	bfqd->delta_from_first = now_ns - bfqd->first_dispatch;
++
++	bfq_log(bfqd,
++	"update_peak_rate: added samples %u/%u tot_sects %llu delta_first %lluus",
++		bfqd->peak_rate_samples, bfqd->sequential_samples,
++		bfqd->tot_sectors_dispatched,
++		bfqd->delta_from_first>>10);
++
++	/* Target observation interval not yet reached, go on sampling */
++	if (bfqd->delta_from_first < BFQ_RATE_REF_INTERVAL)
++		goto update_last_values;
++
++update_rate_and_reset:
++	bfq_update_rate_reset(bfqd, rq);
++update_last_values:
++	bfqd->last_position = blk_rq_pos(rq) + blk_rq_sectors(rq);
++	bfqd->last_dispatch = now_ns;
++
++	bfq_log(bfqd,
++	"update_peak_rate: delta_first %lluus last_pos %llu peak_rate %llu",
++		(now_ns - bfqd->first_dispatch)>>10,
++		(unsigned long long) bfqd->last_position,
++		((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT));
++	bfq_log(bfqd,
++	"update_peak_rate: samples at end %d", bfqd->peak_rate_samples);
  }
  
 -static int bfq_bfqq_budget_left(struct bfq_queue *bfqq)
--{
++/*
++ * Move request from internal lists to the dispatch list of the request queue
++ */
++static void bfq_dispatch_insert(struct request_queue *q, struct request *rq)
+ {
 -	struct bfq_entity *entity = &bfqq->entity;
++	struct bfq_queue *bfqq = RQ_BFQQ(rq);
+ 
 -	return entity->budget - entity->service;
--}
--
++	/*
++	 * For consistency, the next instruction should have been executed
++	 * after removing the request from the queue and dispatching it.
++	 * We execute instead this instruction before bfq_remove_request()
++	 * (and hence introduce a temporary inconsistency), for efficiency.
++	 * In fact, in a forced_dispatch, this prevents two counters related
++	 * to bfqq->dispatched to risk to be uselessly decremented if bfqq
++	 * is not in service, and then to be incremented again after
++	 * incrementing bfqq->dispatched.
++	 */
++	bfqq->dispatched++;
++	bfq_update_peak_rate(q->elevator->elevator_data, rq);
++
++	bfq_remove_request(rq);
++	elv_dispatch_sort(q, rq);
+ }
+ 
  static void __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  {
  	BUG_ON(bfqq != bfqd->in_service_queue);
-@@ -1850,12 +2269,15 @@ static void __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+ 
+-	__bfq_bfqd_reset_in_service(bfqd);
+-
+ 	/*
+ 	 * If this bfqq is shared between multiple processes, check
+ 	 * to make sure that those processes are still issuing I/Os
+@@ -1851,20 +2651,30 @@ static void __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  		bfq_mark_bfqq_split_coop(bfqq);
  
  	if (RB_EMPTY_ROOT(&bfqq->sort_list)) {
@@ -2641,6 +3884,7 @@ index d1f648d..3bc1f8b 100644
 -		 * the weight-raising mechanism.
 -		 */
 -		bfqq->budget_timeout = jiffies;
+-		bfq_del_bfqq_busy(bfqd, bfqq, 1);
 +		if (bfqq->dispatched == 0)
 +			/*
 +			 * Overloading budget_timeout field to store
@@ -2650,10 +3894,26 @@ index d1f648d..3bc1f8b 100644
 +			 */
 +			bfqq->budget_timeout = jiffies;
 +
- 		bfq_del_bfqq_busy(bfqd, bfqq, 1);
++		bfq_del_bfqq_busy(bfqd, bfqq, true);
  	} else {
- 		bfq_activate_bfqq(bfqd, bfqq);
-@@ -1882,10 +2304,19 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
+-		bfq_activate_bfqq(bfqd, bfqq);
++		bfq_requeue_bfqq(bfqd, bfqq);
+ 		/*
+ 		 * Resort priority tree of potential close cooperators.
+ 		 */
+ 		bfq_pos_tree_add_move(bfqd, bfqq);
+ 	}
++
++	/*
++	 * All in-service entities must have been properly deactivated
++	 * or requeued before executing the next function, which
++	 * resets all in-service entites as no more in service.
++	 */
++	__bfq_bfqd_reset_in_service(bfqd);
+ }
+ 
+ /**
+@@ -1883,10 +2693,19 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
  	struct request *next_rq;
  	int budget, min_budget;
  
@@ -2675,7 +3935,7 @@ index d1f648d..3bc1f8b 100644
  
  	bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last budg %d, budg left %d",
  		bfqq->entity.budget, bfq_bfqq_budget_left(bfqq));
-@@ -1894,7 +2325,7 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
+@@ -1895,7 +2714,7 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
  	bfq_log_bfqq(bfqd, bfqq, "recalc_budg: sync %d, seeky %d",
  		bfq_bfqq_sync(bfqq), BFQQ_SEEKY(bfqd->in_service_queue));
  
@@ -2684,7 +3944,7 @@ index d1f648d..3bc1f8b 100644
  		switch (reason) {
  		/*
  		 * Caveat: in all the following cases we trade latency
-@@ -1936,14 +2367,10 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
+@@ -1937,14 +2756,10 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
  			break;
  		case BFQ_BFQQ_BUDGET_TIMEOUT:
  			/*
@@ -2703,7 +3963,7 @@ index d1f648d..3bc1f8b 100644
  			 */
  			budget = min(budget * 2, bfqd->bfq_max_budget);
  			break;
-@@ -1960,17 +2387,49 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
+@@ -1961,17 +2776,49 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
  			budget = min(budget * 4, bfqd->bfq_max_budget);
  			break;
  		case BFQ_BFQQ_NO_MORE_REQUESTS:
@@ -2760,7 +4020,7 @@ index d1f648d..3bc1f8b 100644
  		 */
  		budget = bfqd->bfq_max_budget;
  
-@@ -1981,65 +2440,105 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
+@@ -1982,160 +2829,120 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
  		bfqq->max_budget = min(bfqq->max_budget, bfqd->bfq_max_budget);
  
  	/*
@@ -2795,39 +4055,34 @@ index d1f648d..3bc1f8b 100644
  }
  
 -static unsigned long bfq_calc_max_budget(u64 peak_rate, u64 timeout)
-+static unsigned long bfq_calc_max_budget(struct bfq_data *bfqd)
- {
+-{
 -	unsigned long max_budget;
 -
- 	/*
- 	 * The max_budget calculated when autotuning is equal to the
+-	/*
+-	 * The max_budget calculated when autotuning is equal to the
 -	 * amount of sectors transfered in timeout_sync at the
-+	 * amount of sectors transfered in timeout at the
- 	 * estimated peak rate.
- 	 */
+-	 * estimated peak rate.
+-	 */
 -	max_budget = (unsigned long)(peak_rate * 1000 *
 -				     timeout >> BFQ_RATE_SHIFT);
 -
 -	return max_budget;
-+	return bfqd->peak_rate * 1000 * jiffies_to_msecs(bfqd->bfq_timeout) >>
-+		BFQ_RATE_SHIFT;
- }
- 
+-}
+-
  /*
 - * In addition to updating the peak rate, checks whether the process
 - * is "slow", and returns 1 if so. This slow flag is used, in addition
 - * to the budget timeout, to reduce the amount of service provided to
 - * seeky processes, and hence reduce their chances to lower the
 - * throughput. See the code for more details.
-+ * Update the read peak rate (quantity used for auto-tuning) as a
-+ * function of the rate at which bfqq has been served, and check
-+ * whether the process associated with bfqq is "slow". Return true if
-+ * the process is slow. The slow flag is used, in addition to the
-+ * budget timeout, to reduce the amount of service provided to seeky
-+ * processes, and hence reduce their chances to lower the
-+ * throughput. More details in the body of the function.
++ * Return true if the process associated with bfqq is "slow". The slow
++ * flag is used, in addition to the budget timeout, to reduce the
++ * amount of service provided to seeky processes, and thus reduce
++ * their chances to lower the throughput. More details in the comments
++ * on the function bfq_bfqq_expire().
 + *
-+ * An important observation is in order: with devices with internal
++ * An important observation is in order: as discussed in the comments
++ * on the function bfq_update_peak_rate(), with devices with internal
 + * queues, it is hard if ever possible to know when and for how long
 + * an I/O request is processed by the device (apart from the trivial
 + * I/O pattern where a new request is dispatched only after the
@@ -2835,29 +4090,32 @@ index d1f648d..3bc1f8b 100644
 + * the real rate at which the I/O requests of each bfq_queue are
 + * served.  In fact, for an I/O scheduler like BFQ, serving a
 + * bfq_queue means just dispatching its requests during its service
-+ * slot, i.e., until the budget of the queue is exhausted, or the
-+ * queue remains idle, or, finally, a timeout fires. But, during the
-+ * service slot of a bfq_queue, the device may be still processing
-+ * requests of bfq_queues served in previous service slots. On the
-+ * opposite end, the requests of the in-service bfq_queue may be
-+ * completed after the service slot of the queue finishes. Anyway,
-+ * unless more sophisticated solutions are used (where possible), the
-+ * sum of the sizes of the requests dispatched during the service slot
-+ * of a bfq_queue is probably the only approximation available for
-+ * the service received by the bfq_queue during its service slot. And,
-+ * as written above, this sum is the quantity used in this function to
-+ * evaluate the peak rate.
++ * slot (i.e., until the budget of the queue is exhausted, or the
++ * queue remains idle, or, finally, a timeout fires). But, during the
++ * service slot of a bfq_queue, around 100 ms at most, the device may
++ * be even still processing requests of bfq_queues served in previous
++ * service slots. On the opposite end, the requests of the in-service
++ * bfq_queue may be completed after the service slot of the queue
++ * finishes.
++ *
++ * Anyway, unless more sophisticated solutions are used
++ * (where possible), the sum of the sizes of the requests dispatched
++ * during the service slot of a bfq_queue is probably the only
++ * approximation available for the service received by the bfq_queue
++ * during its service slot. And this sum is the quantity used in this
++ * function to evaluate the I/O speed of a process.
   */
- static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+-static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq,
 -				 bool compensate, enum bfqq_expiration reason)
++static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq,
 +				 bool compensate, enum bfqq_expiration reason,
 +				 unsigned long *delta_ms)
  {
 -	u64 bw, usecs, expected, timeout;
 -	ktime_t delta;
-+	u64 bw, bwdiv10, delta_usecs, delta_ms_tmp;
+-	int update = 0;
 +	ktime_t delta_ktime;
- 	int update = 0;
++	u32 delta_usecs;
 +	bool slow = BFQQ_SEEKY(bfqq); /* if delta too short, use seekyness */
  
 -	if (!bfq_bfqq_sync(bfqq) || bfq_bfqq_budget_new(bfqq))
@@ -2871,133 +4129,110 @@ index d1f648d..3bc1f8b 100644
 -		delta = ktime_get();
 -	delta = ktime_sub(delta, bfqd->last_budget_start);
 -	usecs = ktime_to_us(delta);
+-
+-	/* Don't trust short/unrealistic values. */
+-	if (usecs < 100 || usecs >= LONG_MAX)
+-		return false;
+-
+-	/*
+-	 * Calculate the bandwidth for the last slice.  We use a 64 bit
+-	 * value to store the peak rate, in sectors per usec in fixed
+-	 * point math.  We do so to have enough precision in the estimate
+-	 * and to avoid overflows.
+-	 */
+-	bw = (u64)bfqq->entity.service << BFQ_RATE_SHIFT;
+-	do_div(bw, (unsigned long)usecs);
 +		delta_ktime = ktime_get();
 +	delta_ktime = ktime_sub(delta_ktime, bfqd->last_budget_start);
 +	delta_usecs = ktime_to_us(delta_ktime);
- 
- 	/* Don't trust short/unrealistic values. */
--	if (usecs < 100 || usecs >= LONG_MAX)
--		return false;
++
++	/* don't trust short/unrealistic values. */
 +	if (delta_usecs < 1000 || delta_usecs >= LONG_MAX) {
 +		if (blk_queue_nonrot(bfqd->queue))
-+			*delta_ms = BFQ_MIN_TT; /* give same worst-case
-+						   guarantees as
-+						   idling for seeky
-+						*/
-+		else /* Charge at least one seek */
-+			*delta_ms = jiffies_to_msecs(bfq_slice_idle);
++			 /*
++			  * give same worst-case guarantees as idling
++			  * for seeky
++			  */
++			*delta_ms = BFQ_MIN_TT / NSEC_PER_MSEC;
++		else /* charge at least one seek */
++			*delta_ms = bfq_slice_idle / NSEC_PER_MSEC;
++
++		bfq_log(bfqd, "bfq_bfqq_is_slow: unrealistic %u", delta_usecs);
++
 +		return slow;
 +	}
-+
-+	delta_ms_tmp = delta_usecs;
-+	do_div(delta_ms_tmp, 1000);
-+	*delta_ms = delta_ms_tmp;
  
- 	/*
- 	 * Calculate the bandwidth for the last slice.  We use a 64 bit
-@@ -2048,32 +2547,51 @@ static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq,
- 	 * and to avoid overflows.
- 	 */
- 	bw = (u64)bfqq->entity.service << BFQ_RATE_SHIFT;
--	do_div(bw, (unsigned long)usecs);
--
 -	timeout = jiffies_to_msecs(bfqd->bfq_timeout[BLK_RW_SYNC]);
-+	do_div(bw, (unsigned long)delta_usecs);
++	*delta_ms = delta_usecs / USEC_PER_MSEC;
  
-+	bfq_log(bfqd, "measured bw = %llu sects/sec",
-+		(1000000*bw)>>BFQ_RATE_SHIFT);
  	/*
- 	 * Use only long (> 20ms) intervals to filter out spikes for
- 	 * the peak rate estimation.
+-	 * Use only long (> 20ms) intervals to filter out spikes for
+-	 * the peak rate estimation.
++	 * Use only long (> 20ms) intervals to filter out excessive
++	 * spikes in service rate estimation.
  	 */
 -	if (usecs > 20000) {
-+	if (delta_usecs > 20000) {
-+		bool fully_sequential = bfqq->seek_history == 0;
-+		/*
-+		 * Soft real-time queues are not good candidates for
-+		 * evaluating bw, as they are likely to be slow even
-+		 * if sequential.
-+		 */
-+		bool non_soft_rt = bfqq->wr_coeff == 1 ||
-+			bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time;
-+		bool consumed_large_budget =
-+			reason == BFQ_BFQQ_BUDGET_EXHAUSTED &&
-+			bfqq->entity.budget >= bfqd->bfq_max_budget * 2 / 3;
-+		bool served_for_long_time =
-+			reason == BFQ_BFQQ_BUDGET_TIMEOUT ||
-+			consumed_large_budget;
-+
-+		BUG_ON(bfqq->seek_history == 0 &&
-+		       hweight32(bfqq->seek_history) != 0);
-+
- 		if (bw > bfqd->peak_rate ||
+-		if (bw > bfqd->peak_rate ||
 -		   (!BFQQ_SEEKY(bfqq) &&
 -		    reason == BFQ_BFQQ_BUDGET_TIMEOUT)) {
 -			bfq_log(bfqd, "measured bw =%llu", bw);
-+		    (bfq_bfqq_sync(bfqq) && fully_sequential && non_soft_rt &&
-+		     served_for_long_time)) {
- 			/*
- 			 * To smooth oscillations use a low-pass filter with
+-			/*
+-			 * To smooth oscillations use a low-pass filter with
 -			 * alpha=7/8, i.e.,
 -			 * new_rate = (7/8) * old_rate + (1/8) * bw
-+			 * alpha=9/10, i.e.,
-+			 * new_rate = (9/10) * old_rate + (1/10) * bw
- 			 */
+-			 */
 -			do_div(bw, 8);
 -			if (bw == 0)
 -				return 0;
 -			bfqd->peak_rate *= 7;
 -			do_div(bfqd->peak_rate, 8);
 -			bfqd->peak_rate += bw;
-+			bwdiv10 = bw;
-+			do_div(bwdiv10, 10);
-+			if (bwdiv10 == 0)
-+				return false; /* bw too low to be used */
-+			bfqd->peak_rate *= 9;
-+			do_div(bfqd->peak_rate, 10);
-+			bfqd->peak_rate += bwdiv10;
- 			update = 1;
+-			update = 1;
 -			bfq_log(bfqd, "new peak_rate=%llu", bfqd->peak_rate);
-+			bfq_log(bfqd, "new peak_rate = %llu sects/sec",
-+				(1000000*bfqd->peak_rate)>>BFQ_RATE_SHIFT);
- 		}
- 
- 		update |= bfqd->peak_rate_samples == BFQ_PEAK_RATE_SAMPLES - 1;
-@@ -2086,9 +2604,8 @@ static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq,
- 			int dev_type = blk_queue_nonrot(bfqd->queue);
- 			if (bfqd->bfq_user_max_budget == 0) {
- 				bfqd->bfq_max_budget =
+-		}
+-
+-		update |= bfqd->peak_rate_samples == BFQ_PEAK_RATE_SAMPLES - 1;
+-
+-		if (bfqd->peak_rate_samples < BFQ_PEAK_RATE_SAMPLES)
+-			bfqd->peak_rate_samples++;
+-
+-		if (bfqd->peak_rate_samples == BFQ_PEAK_RATE_SAMPLES &&
+-		    update) {
+-			int dev_type = blk_queue_nonrot(bfqd->queue);
+-
+-			if (bfqd->bfq_user_max_budget == 0) {
+-				bfqd->bfq_max_budget =
 -					bfq_calc_max_budget(bfqd->peak_rate,
 -							    timeout);
 -				bfq_log(bfqd, "new max_budget=%d",
-+					bfq_calc_max_budget(bfqd);
-+				bfq_log(bfqd, "new max_budget = %d",
- 					bfqd->bfq_max_budget);
- 			}
- 			if (bfqd->device_speed == BFQ_BFQD_FAST &&
-@@ -2102,38 +2619,35 @@ static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq,
- 				bfqd->RT_prod = R_fast[dev_type] *
- 						T_fast[dev_type];
- 			}
-+			bfq_log(bfqd, "dev_speed_class = %d (%d sects/sec), "
-+				"thresh %d setcs/sec",
-+				bfqd->device_speed,
-+				bfqd->device_speed == BFQ_BFQD_FAST ?
-+				(1000000*R_fast[dev_type])>>BFQ_RATE_SHIFT :
-+				(1000000*R_slow[dev_type])>>BFQ_RATE_SHIFT,
-+				(1000000*device_speed_thresh[dev_type])>>
-+				BFQ_RATE_SHIFT);
- 		}
+-					bfqd->bfq_max_budget);
+-			}
+-			if (bfqd->device_speed == BFQ_BFQD_FAST &&
+-			    bfqd->peak_rate < device_speed_thresh[dev_type]) {
+-				bfqd->device_speed = BFQ_BFQD_SLOW;
+-				bfqd->RT_prod = R_slow[dev_type] *
+-						T_slow[dev_type];
+-			} else if (bfqd->device_speed == BFQ_BFQD_SLOW &&
+-			    bfqd->peak_rate > device_speed_thresh[dev_type]) {
+-				bfqd->device_speed = BFQ_BFQD_FAST;
+-				bfqd->RT_prod = R_fast[dev_type] *
+-						T_fast[dev_type];
+-			}
+-		}
++	if (delta_usecs > 20000) {
 +		/*
-+		 * Caveat: processes doing IO in the slower disk zones
-+		 * tend to be slow(er) even if not seeky. In this
-+		 * respect, the estimated peak rate is likely to be an
-+		 * average over the disk surface. Accordingly, to not
-+		 * be too harsh with unlucky processes, a process is
-+		 * deemed slow only if its bw has been lower than half
-+		 * of the estimated peak rate.
++		 * Caveat for rotational devices: processes doing I/O
++		 * in the slower disk zones tend to be slow(er) even
++		 * if not seeky. In this respect, the estimated peak
++		 * rate is likely to be an average over the disk
++		 * surface. Accordingly, to not be too harsh with
++		 * unlucky processes, a process is deemed slow only if
++		 * its rate has been lower than half of the estimated
++		 * peak rate.
 +		 */
-+		slow = bw < bfqd->peak_rate / 2;
++		slow = bfqq->entity.service < bfqd->bfq_max_budget / 2;
++		bfq_log(bfqd, "bfq_bfqq_is_slow: relative rate %d/%d",
++			bfqq->entity.service, bfqd->bfq_max_budget);
  	}
  
 -	/*
@@ -3019,12 +4254,7 @@ index d1f648d..3bc1f8b 100644
 -	 * before the budget timeout expiration.
 -	 */
 -	expected = bw * 1000 * timeout >> BFQ_RATE_SHIFT;
-+	bfq_log_bfqq(bfqd, bfqq,
-+		     "update_peak_rate: bw %llu sect/s, peak rate %llu, "
-+		     "slow %d",
-+		     (1000000*bw)>>BFQ_RATE_SHIFT,
-+		     (1000000*bfqd->peak_rate)>>BFQ_RATE_SHIFT,
-+		     bw < bfqd->peak_rate / 2);
++	bfq_log_bfqq(bfqd, bfqq, "bfq_bfqq_is_slow: slow %d", slow);
  
 -	/*
 -	 * Caveat: processes doing IO in the slower disk zones will
@@ -3039,14 +4269,12 @@ index d1f648d..3bc1f8b 100644
  }
  
  /*
-@@ -2191,6 +2705,15 @@ static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -2193,20 +3000,35 @@ static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd,
  						struct bfq_queue *bfqq)
  {
 +	bfq_log_bfqq(bfqd, bfqq,
-+		     "softrt_next_start: service_blkg %lu "
-+		     "soft_rate %u sects/sec"
-+		     "interval %u",
++"softrt_next_start: service_blkg %lu soft_rate %u sects/sec interval %u",
 +		     bfqq->service_from_backlogged,
 +		     bfqd->bfq_wr_max_softrt_rate,
 +		     jiffies_to_msecs(HZ * bfqq->service_from_backlogged /
@@ -3055,7 +4283,8 @@ index d1f648d..3bc1f8b 100644
  	return max(bfqq->last_idle_bklogged +
  		   HZ * bfqq->service_from_backlogged /
  		   bfqd->bfq_wr_max_softrt_rate,
-@@ -2198,13 +2721,21 @@ static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd,
+-		   jiffies + bfqq->bfqd->bfq_slice_idle + 4);
++		   jiffies + nsecs_to_jiffies(bfqq->bfqd->bfq_slice_idle) + 4);
  }
  
  /*
@@ -3064,25 +4293,25 @@ index d1f648d..3bc1f8b 100644
 - * time_is_before_jiffies().
 + * Return the farthest future time instant according to jiffies
 + * macros.
-+ */
+  */
+-static unsigned long bfq_infinity_from_now(unsigned long now)
 +static unsigned long bfq_greatest_from_now(void)
-+{
+ {
+-	return now + ULONG_MAX / 2;
 +	return jiffies + MAX_JIFFY_OFFSET;
 +}
 +
 +/*
 + * Return the farthest past time instant according to jiffies
 + * macros.
-  */
--static unsigned long bfq_infinity_from_now(unsigned long now)
++ */
 +static unsigned long bfq_smallest_from_now(void)
- {
--	return now + ULONG_MAX / 2;
++{
 +	return jiffies - MAX_JIFFY_OFFSET;
  }
  
  /**
-@@ -2214,28 +2745,24 @@ static unsigned long bfq_infinity_from_now(unsigned long now)
+@@ -2216,28 +3038,24 @@ static unsigned long bfq_infinity_from_now(unsigned long now)
   * @compensate: if true, compensate for the time spent idling.
   * @reason: the reason causing the expiration.
   *
@@ -3128,33 +4357,23 @@ index d1f648d..3bc1f8b 100644
   */
  static void bfq_bfqq_expire(struct bfq_data *bfqd,
  			    struct bfq_queue *bfqq,
-@@ -2243,40 +2770,53 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
+@@ -2245,41 +3063,52 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
  			    enum bfqq_expiration reason)
  {
  	bool slow;
 +	unsigned long delta = 0;
 +	struct bfq_entity *entity = &bfqq->entity;
-+
+ 
  	BUG_ON(bfqq != bfqd->in_service_queue);
  
  	/*
 -	 * Update disk peak rate for autotuning and check whether the
-+	 * Update device peak rate for autotuning and check whether the
- 	 * process is slow (see bfq_update_peak_rate).
- 	 */
--	slow = bfq_update_peak_rate(bfqd, bfqq, compensate, reason);
-+	slow = bfq_update_peak_rate(bfqd, bfqq, compensate, reason, &delta);
- 
- 	/*
--	 * As above explained, 'punish' slow (i.e., seeky), timed-out
--	 * and async queues, to favor sequential sync workloads.
--	 *
--	 * Processes doing I/O in the slower disk zones will tend to be
--	 * slow(er) even if not seeky. Hence, since the estimated peak
--	 * rate is actually an average over the disk surface, these
--	 * processes may timeout just for bad luck. To avoid punishing
--	 * them we do not charge a full budget to a process that
--	 * succeeded in consuming at least 2/3 of its budget.
+-	 * process is slow (see bfq_update_peak_rate).
++	 * Check whether the process is slow (see bfq_bfqq_is_slow).
++	 */
++	slow = bfq_bfqq_is_slow(bfqd, bfqq, compensate, reason, &delta);
++
++	/*
 +	 * Increase service_from_backlogged before next statement,
 +	 * because the possible next invocation of
 +	 * bfq_bfqq_charge_time would likely inflate
@@ -3163,17 +4382,22 @@ index d1f648d..3bc1f8b 100644
 +	 * heuristic to correctly compute the bandwidth consumed by
 +	 * bfqq.
  	 */
--	if (slow || (reason == BFQ_BFQQ_BUDGET_TIMEOUT &&
--		     bfq_bfqq_budget_left(bfqq) >=  bfqq->entity.budget / 3))
--		bfq_bfqq_charge_full_budget(bfqq);
+-	slow = bfq_update_peak_rate(bfqd, bfqq, compensate, reason);
 +	bfqq->service_from_backlogged += entity->service;
  
--	bfqq->service_from_backlogged += bfqq->entity.service;
-+	/*
+ 	/*
+-	 * As above explained, 'punish' slow (i.e., seeky), timed-out
+-	 * and async queues, to favor sequential sync workloads.
 +	 * As above explained, charge slow (typically seeky) and
 +	 * timed-out queues with the time and not the service
 +	 * received, to favor sequential workloads.
-+	 *
+ 	 *
+-	 * Processes doing I/O in the slower disk zones will tend to be
+-	 * slow(er) even if not seeky. Hence, since the estimated peak
+-	 * rate is actually an average over the disk surface, these
+-	 * processes may timeout just for bad luck. To avoid punishing
+-	 * them we do not charge a full budget to a process that
+-	 * succeeded in consuming at least 2/3 of its budget.
 +	 * Processes doing I/O in the slower disk zones will tend to
 +	 * be slow(er) even if not seeky. Therefore, since the
 +	 * estimated peak rate is actually an average over the disk
@@ -3183,7 +4407,12 @@ index d1f648d..3bc1f8b 100644
 +	 * allows BFQ to preserve enough elasticity to still perform
 +	 * bandwidth, and not time, distribution with little unlucky
 +	 * or quasi-sequential processes.
-+	 */
+ 	 */
+-	if (slow || (reason == BFQ_BFQQ_BUDGET_TIMEOUT &&
+-		     bfq_bfqq_budget_left(bfqq) >=  bfqq->entity.budget / 3))
+-		bfq_bfqq_charge_full_budget(bfqq);
+-
+-	bfqq->service_from_backlogged += bfqq->entity.service;
 +	if (bfqq->wr_coeff == 1 &&
 +	    (slow ||
 +	     (reason == BFQ_BFQQ_BUDGET_TIMEOUT &&
@@ -3199,12 +4428,12 @@ index d1f648d..3bc1f8b 100644
 +	BUG_ON(bfqq->entity.budget < bfqq->entity.service);
  
  	if (reason == BFQ_BFQQ_TOO_IDLE &&
--	    bfqq->entity.service <= 2 * bfqq->entity.budget / 10 )
-+	    entity->service <= 2 * entity->budget / 10 )
+-	    bfqq->entity.service <= 2 * bfqq->entity.budget / 10)
++	    entity->service <= 2 * entity->budget / 10)
  		bfq_clear_bfqq_IO_bound(bfqq);
  
  	if (bfqd->low_latency && bfqq->wr_coeff == 1)
-@@ -2285,19 +2825,23 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
+@@ -2288,19 +3117,23 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
  	if (bfqd->low_latency && bfqd->bfq_wr_max_softrt_rate > 0 &&
  	    RB_EMPTY_ROOT(&bfqq->sort_list)) {
  		/*
@@ -3236,7 +4465,7 @@ index d1f648d..3bc1f8b 100644
  			/*
  			 * The application is still waiting for the
  			 * completion of one or more requests:
-@@ -2314,7 +2858,7 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
+@@ -2317,7 +3150,7 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
  			 *    happened to be in the past.
  			 */
  			bfqq->soft_rt_next_start =
@@ -3245,7 +4474,7 @@ index d1f648d..3bc1f8b 100644
  			/*
  			 * Schedule an update of soft_rt_next_start to when
  			 * the task may be discovered to be isochronous.
-@@ -2324,15 +2868,27 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
+@@ -2327,15 +3160,27 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
  	}
  
  	bfq_log_bfqq(bfqd, bfqq,
@@ -3275,7 +4504,7 @@ index d1f648d..3bc1f8b 100644
  }
  
  /*
-@@ -2342,20 +2898,17 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
+@@ -2345,20 +3190,17 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
   */
  static bool bfq_bfqq_budget_timeout(struct bfq_queue *bfqq)
  {
@@ -3304,7 +4533,7 @@ index d1f648d..3bc1f8b 100644
  static bool bfq_may_expire_for_budg_timeout(struct bfq_queue *bfqq)
  {
  	bfq_log_bfqq(bfqq->bfqd, bfqq,
-@@ -2397,10 +2950,12 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
+@@ -2400,10 +3242,12 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
  {
  	struct bfq_data *bfqd = bfqq->bfqd;
  	bool idling_boosts_thr, idling_boosts_thr_without_issues,
@@ -3318,16 +4547,7 @@ index d1f648d..3bc1f8b 100644
  	/*
  	 * The next variable takes into account the cases where idling
  	 * boosts the throughput.
-@@ -2422,7 +2977,7 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
- 	 */
- 	idling_boosts_thr = !bfqd->hw_tag ||
- 		(!blk_queue_nonrot(bfqd->queue) && bfq_bfqq_IO_bound(bfqq) &&
--		 bfq_bfqq_idle_window(bfqq)) ;
-+		 bfq_bfqq_idle_window(bfqq));
- 
- 	/*
- 	 * The value of the next variable,
-@@ -2463,74 +3018,27 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
+@@ -2466,74 +3310,27 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
  		bfqd->wr_busy_queues == 0;
  
  	/*
@@ -3421,7 +4641,7 @@ index d1f648d..3bc1f8b 100644
  	 * (i)  each of these processes must get the same throughput as
  	 *      the others;
  	 * (ii) all these processes have the same I/O pattern
-@@ -2552,26 +3060,53 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
+@@ -2555,26 +3352,53 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
  	 * words, only if sub-condition (i) holds, then idling is
  	 * allowed, and the device tends to be prevented from queueing
  	 * many requests, possibly of several processes. The reason
@@ -3495,7 +4715,7 @@ index d1f648d..3bc1f8b 100644
  	 *
  	 * According to the above considerations, the next variable is
  	 * true (only) if sub-condition (i) holds. To compute the
-@@ -2579,7 +3114,7 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
+@@ -2582,7 +3406,7 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
  	 * the function bfq_symmetric_scenario(), but also check
  	 * whether bfqq is being weight-raised, because
  	 * bfq_symmetric_scenario() does not take into account also
@@ -3504,7 +4724,7 @@ index d1f648d..3bc1f8b 100644
  	 * bfq_weights_tree_add()).
  	 *
  	 * As a side note, it is worth considering that the above
-@@ -2601,17 +3136,16 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
+@@ -2604,17 +3428,16 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
  	 * bfqq. Such a case is when bfqq became active in a burst of
  	 * queue activations. Queues that became active during a large
  	 * burst benefit only from throughput, as discussed in the
@@ -3527,13 +4747,15 @@ index d1f648d..3bc1f8b 100644
  
  	/*
  	 * We have now all the components we need to compute the return
-@@ -2621,6 +3155,14 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
+@@ -2624,6 +3447,16 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
  	 * 2) idling either boosts the throughput (without issues), or
  	 *    is necessary to preserve service guarantees.
  	 */
-+	bfq_log_bfqq(bfqd, bfqq, "may_idle: sync %d idling_boosts_thr %d "
-+		     "wr_busy %d boosts %d IO-bound %d guar %d",
-+		     bfq_bfqq_sync(bfqq), idling_boosts_thr,
++	bfq_log_bfqq(bfqd, bfqq, "may_idle: sync %d idling_boosts_thr %d",
++		     bfq_bfqq_sync(bfqq), idling_boosts_thr);
++
++	bfq_log_bfqq(bfqd, bfqq,
++		     "may_idle: wr_busy %d boosts %d IO-bound %d guar %d",
 +		     bfqd->wr_busy_queues,
 +		     idling_boosts_thr_without_issues,
 +		     bfq_bfqq_IO_bound(bfqq),
@@ -3542,7 +4764,7 @@ index d1f648d..3bc1f8b 100644
  	return bfq_bfqq_sync(bfqq) &&
  		(idling_boosts_thr_without_issues ||
  		 idling_needed_for_service_guarantees);
-@@ -2632,7 +3174,7 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
+@@ -2635,7 +3468,7 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
   * 1) the queue must remain in service and cannot be expired, and
   * 2) the device must be idled to wait for the possible arrival of a new
   *    request for the queue.
@@ -3551,17 +4773,102 @@ index d1f648d..3bc1f8b 100644
   * why performing device idling is the best choice to boost the throughput
   * and preserve service guarantees when bfq_bfqq_may_idle itself
   * returns true.
-@@ -2698,9 +3240,7 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd)
+@@ -2665,18 +3498,33 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd)
+ 	bfq_log_bfqq(bfqd, bfqq, "select_queue: already in-service queue");
+ 
+ 	if (bfq_may_expire_for_budg_timeout(bfqq) &&
+-	    !timer_pending(&bfqd->idle_slice_timer) &&
++	    !hrtimer_active(&bfqd->idle_slice_timer) &&
+ 	    !bfq_bfqq_must_idle(bfqq))
+ 		goto expire;
+ 
++check_queue:
++	/*
++	 * This loop is rarely executed more than once. Even when it
++	 * happens, it is much more convenient to re-execute this loop
++	 * than to return NULL and trigger a new dispatch to get a
++	 * request served.
++	 */
+ 	next_rq = bfqq->next_rq;
+ 	/*
+ 	 * If bfqq has requests queued and it has enough budget left to
+ 	 * serve them, keep the queue, otherwise expire it.
+ 	 */
+ 	if (next_rq) {
++		BUG_ON(RB_EMPTY_ROOT(&bfqq->sort_list));
++
+ 		if (bfq_serv_to_charge(next_rq, bfqq) >
+ 			bfq_bfqq_budget_left(bfqq)) {
++			/*
++			 * Expire the queue for budget exhaustion,
++			 * which makes sure that the next budget is
++			 * enough to serve the next request, even if
++			 * it comes from the fifo expired path.
++			 */
+ 			reason = BFQ_BFQQ_BUDGET_EXHAUSTED;
+ 			goto expire;
+ 		} else {
+@@ -2685,7 +3533,8 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd)
+ 			 * not disable disk idling even when a new request
+ 			 * arrives.
+ 			 */
+-			if (timer_pending(&bfqd->idle_slice_timer)) {
++			if (bfq_bfqq_wait_request(bfqq)) {
++				BUG_ON(!hrtimer_active(&bfqd->idle_slice_timer));
+ 				/*
+ 				 * If we get here: 1) at least a new request
+ 				 * has arrived but we have not disabled the
+@@ -2700,10 +3549,8 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd)
+ 				 * So we disable idling.
  				 */
  				bfq_clear_bfqq_wait_request(bfqq);
- 				del_timer(&bfqd->idle_slice_timer);
+-				del_timer(&bfqd->idle_slice_timer);
 -#ifdef CONFIG_BFQ_GROUP_IOSCHED
++				hrtimer_try_to_cancel(&bfqd->idle_slice_timer);
  				bfqg_stats_update_idle_time(bfqq_group(bfqq));
 -#endif
  			}
  			goto keep_queue;
  		}
-@@ -2745,14 +3285,11 @@ static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+@@ -2714,7 +3561,7 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd)
+ 	 * for a new request, or has requests waiting for a completion and
+ 	 * may idle after their completion, then keep it anyway.
+ 	 */
+-	if (timer_pending(&bfqd->idle_slice_timer) ||
++	if (hrtimer_active(&bfqd->idle_slice_timer) ||
+ 	    (bfqq->dispatched != 0 && bfq_bfqq_may_idle(bfqq))) {
+ 		bfqq = NULL;
+ 		goto keep_queue;
+@@ -2725,9 +3572,16 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd)
+ 	bfq_bfqq_expire(bfqd, bfqq, false, reason);
+ new_queue:
+ 	bfqq = bfq_set_in_service_queue(bfqd);
+-	bfq_log(bfqd, "select_queue: new queue %d returned",
+-		bfqq ? bfqq->pid : 0);
++	if (bfqq) {
++		bfq_log_bfqq(bfqd, bfqq, "select_queue: checking new queue");
++		goto check_queue;
++	}
+ keep_queue:
++	if (bfqq)
++		bfq_log_bfqq(bfqd, bfqq, "select_queue: returned this queue");
++	else
++		bfq_log(bfqd, "select_queue: no queue returned");
++
+ 	return bfqq;
+ }
+ 
+@@ -2736,6 +3590,9 @@ static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+ 	struct bfq_entity *entity = &bfqq->entity;
+ 
+ 	if (bfqq->wr_coeff > 1) { /* queue is being weight-raised */
++		BUG_ON(bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time &&
++		       time_is_after_jiffies(bfqq->last_wr_start_finish));
++
+ 		bfq_log_bfqq(bfqd, bfqq,
+ 			"raising period dur %u/%u msec, old coeff %u, w %d(%d)",
+ 			jiffies_to_msecs(jiffies - bfqq->last_wr_start_finish),
+@@ -2749,22 +3606,30 @@ static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  			bfq_log_bfqq(bfqd, bfqq, "WARN: pending prio change");
  
  		/*
@@ -3574,21 +4881,84 @@ index d1f648d..3bc1f8b 100644
 +		 * time has elapsed from the beginning of this
 +		 * weight-raising period, then end weight raising.
  		 */
- 		if (bfq_bfqq_in_large_burst(bfqq) ||
+-		if (bfq_bfqq_in_large_burst(bfqq) ||
 -		    bfq_bfqq_cooperations(bfqq) >= bfqd->bfq_coop_thresh ||
- 		    time_is_before_jiffies(bfqq->last_wr_start_finish +
- 					   bfqq->wr_cur_max_time)) {
- 			bfqq->last_wr_start_finish = jiffies;
-@@ -2811,13 +3348,29 @@ static int bfq_dispatch_request(struct bfq_data *bfqd,
- 		 */
- 		if (!bfqd->rq_in_driver)
- 			bfq_schedule_dispatch(bfqd);
-+		BUG_ON(bfqq->entity.budget < bfqq->entity.service);
- 		goto expire;
+-		    time_is_before_jiffies(bfqq->last_wr_start_finish +
+-					   bfqq->wr_cur_max_time)) {
+-			bfqq->last_wr_start_finish = jiffies;
+-			bfq_log_bfqq(bfqd, bfqq,
+-				     "wrais ending at %lu, rais_max_time %u",
+-				     bfqq->last_wr_start_finish,
+-				     jiffies_to_msecs(bfqq->wr_cur_max_time));
++		if (bfq_bfqq_in_large_burst(bfqq))
+ 			bfq_bfqq_end_wr(bfqq);
++		else if (time_is_before_jiffies(bfqq->last_wr_start_finish +
++					   bfqq->wr_cur_max_time)) {
++			if (bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time ||
++			time_is_before_jiffies(bfqq->wr_start_at_switch_to_srt +
++					bfq_wr_duration(bfqd)))
++				bfq_bfqq_end_wr(bfqq);
++			else {
++				/* switch back to interactive wr */
++				bfqq->wr_coeff = bfqd->bfq_wr_coeff;
++				bfqq->wr_cur_max_time = bfq_wr_duration(bfqd);
++				bfqq->last_wr_start_finish =
++					bfqq->wr_start_at_switch_to_srt;
++				BUG_ON(time_is_after_jiffies(
++					       bfqq->last_wr_start_finish));
++				bfqq->entity.prio_changed = 1;
++				bfq_log_bfqq(bfqd, bfqq,
++					"back to interactive wr");
++			}
+ 		}
  	}
+ 	/* Update weight both if it must be raised and if it must be lowered */
+@@ -2782,46 +3647,34 @@ static int bfq_dispatch_request(struct bfq_data *bfqd,
+ 				struct bfq_queue *bfqq)
+ {
+ 	int dispatched = 0;
+-	struct request *rq;
++	struct request *rq = bfqq->next_rq;
+ 	unsigned long service_to_charge;
  
+ 	BUG_ON(RB_EMPTY_ROOT(&bfqq->sort_list));
+-
+-	/* Follow expired path, else get first next available. */
+-	rq = bfq_check_fifo(bfqq);
+-	if (!rq)
+-		rq = bfqq->next_rq;
++	BUG_ON(!rq);
+ 	service_to_charge = bfq_serv_to_charge(rq, bfqq);
+ 
+-	if (service_to_charge > bfq_bfqq_budget_left(bfqq)) {
+-		/*
+-		 * This may happen if the next rq is chosen in fifo order
+-		 * instead of sector order. The budget is properly
+-		 * dimensioned to be always sufficient to serve the next
+-		 * request only if it is chosen in sector order. The reason
+-		 * is that it would be quite inefficient and little useful
+-		 * to always make sure that the budget is large enough to
+-		 * serve even the possible next rq in fifo order.
+-		 * In fact, requests are seldom served in fifo order.
+-		 *
+-		 * Expire the queue for budget exhaustion, and make sure
+-		 * that the next act_budget is enough to serve the next
+-		 * request, even if it comes from the fifo expired path.
+-		 */
+-		bfqq->next_rq = rq;
+-		/*
+-		 * Since this dispatch is failed, make sure that
+-		 * a new one will be performed
+-		 */
+-		if (!bfqd->rq_in_driver)
+-			bfq_schedule_dispatch(bfqd);
+-		goto expire;
+-	}
++	BUG_ON(service_to_charge > bfq_bfqq_budget_left(bfqq));
++
 +	BUG_ON(bfqq->entity.budget < bfqq->entity.service);
- 	/* Finally, insert request into driver dispatch list. */
+ 
+-	/* Finally, insert request into driver dispatch list. */
  	bfq_bfqq_served(bfqq, service_to_charge);
 +
 +	BUG_ON(bfqq->entity.budget < bfqq->entity.service);
@@ -3609,7 +4979,7 @@ index d1f648d..3bc1f8b 100644
  	bfq_update_wr_data(bfqd, bfqq);
  
  	bfq_log_bfqq(bfqd, bfqq,
-@@ -2833,9 +3386,7 @@ static int bfq_dispatch_request(struct bfq_data *bfqd,
+@@ -2837,9 +3690,7 @@ static int bfq_dispatch_request(struct bfq_data *bfqd,
  		bfqd->in_service_bic = RQ_BIC(rq);
  	}
  
@@ -3620,7 +4990,7 @@ index d1f648d..3bc1f8b 100644
  		goto expire;
  
  	return dispatched;
-@@ -2881,8 +3432,8 @@ static int bfq_forced_dispatch(struct bfq_data *bfqd)
+@@ -2885,8 +3736,8 @@ static int bfq_forced_dispatch(struct bfq_data *bfqd)
  		st = bfq_entity_service_tree(&bfqq->entity);
  
  		dispatched += __bfq_forced_dispatch_bfqq(bfqq);
@@ -3630,7 +5000,7 @@ index d1f648d..3bc1f8b 100644
  		bfq_forget_idle(st);
  	}
  
-@@ -2895,9 +3446,9 @@ static int bfq_dispatch_requests(struct request_queue *q, int force)
+@@ -2899,37 +3750,37 @@ static int bfq_dispatch_requests(struct request_queue *q, int force)
  {
  	struct bfq_data *bfqd = q->elevator->elevator_data;
  	struct bfq_queue *bfqq;
@@ -3641,7 +5011,25 @@ index d1f648d..3bc1f8b 100644
  	if (bfqd->busy_queues == 0)
  		return 0;
  
-@@ -2908,21 +3459,7 @@ static int bfq_dispatch_requests(struct request_queue *q, int force)
+ 	if (unlikely(force))
+ 		return bfq_forced_dispatch(bfqd);
+ 
++	/*
++	 * Force device to serve one request at a time if
++	 * strict_guarantees is true. Forcing this service scheme is
++	 * currently the ONLY way to guarantee that the request
++	 * service order enforced by the scheduler is respected by a
++	 * queueing device. Otherwise the device is free even to make
++	 * some unlucky request wait for as long as the device
++	 * wishes.
++	 *
++	 * Of course, serving one request at at time may cause loss of
++	 * throughput.
++	 */
++	if (bfqd->strict_guarantees && bfqd->rq_in_driver > 0)
++		return 0;
++
+ 	bfqq = bfq_select_queue(bfqd);
  	if (!bfqq)
  		return 0;
  
@@ -3662,9 +5050,13 @@ index d1f648d..3bc1f8b 100644
 -		return 0;
 +	BUG_ON(bfqq->entity.budget < bfqq->entity.service);
  
- 	bfq_clear_bfqq_wait_request(bfqq);
- 	BUG_ON(timer_pending(&bfqd->idle_slice_timer));
-@@ -2933,6 +3470,8 @@ static int bfq_dispatch_requests(struct request_queue *q, int force)
+-	bfq_clear_bfqq_wait_request(bfqq);
+-	BUG_ON(timer_pending(&bfqd->idle_slice_timer));
++	BUG_ON(bfq_bfqq_wait_request(bfqq));
+ 
+ 	if (!bfq_dispatch_request(bfqd, bfqq))
+ 		return 0;
+@@ -2937,6 +3788,8 @@ static int bfq_dispatch_requests(struct request_queue *q, int force)
  	bfq_log_bfqq(bfqd, bfqq, "dispatched %s request",
  			bfq_bfqq_sync(bfqq) ? "sync" : "async");
  
@@ -3673,7 +5065,7 @@ index d1f648d..3bc1f8b 100644
  	return 1;
  }
  
-@@ -2944,23 +3483,22 @@ static int bfq_dispatch_requests(struct request_queue *q, int force)
+@@ -2948,23 +3801,21 @@ static int bfq_dispatch_requests(struct request_queue *q, int force)
   */
  static void bfq_put_queue(struct bfq_queue *bfqq)
  {
@@ -3698,11 +5090,10 @@ index d1f648d..3bc1f8b 100644
  	BUG_ON(bfqq->entity.tree);
  	BUG_ON(bfq_bfqq_busy(bfqq));
 -	BUG_ON(bfqd->in_service_queue == bfqq);
-+	BUG_ON(bfqq->bfqd->in_service_queue == bfqq);
  
  	if (bfq_bfqq_sync(bfqq))
  		/*
-@@ -2973,7 +3511,7 @@ static void bfq_put_queue(struct bfq_queue *bfqq)
+@@ -2977,7 +3828,7 @@ static void bfq_put_queue(struct bfq_queue *bfqq)
  		 */
  		hlist_del_init(&bfqq->burst_list_node);
  
@@ -3711,7 +5102,7 @@ index d1f648d..3bc1f8b 100644
  
  	kmem_cache_free(bfq_pool, bfqq);
  #ifdef CONFIG_BFQ_GROUP_IOSCHED
-@@ -3007,8 +3545,7 @@ static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+@@ -3011,8 +3862,7 @@ static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  		bfq_schedule_dispatch(bfqd);
  	}
  
@@ -3721,10 +5112,12 @@ index d1f648d..3bc1f8b 100644
  
  	bfq_put_cooperator(bfqq);
  
-@@ -3019,26 +3556,7 @@ static void bfq_init_icq(struct io_cq *icq)
- {
- 	struct bfq_io_cq *bic = icq_to_bic(icq);
+@@ -3021,28 +3871,7 @@ static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  
+ static void bfq_init_icq(struct io_cq *icq)
+ {
+-	struct bfq_io_cq *bic = icq_to_bic(icq);
+-
 -	bic->ttime.last_end_request = jiffies;
 -	/*
 -	 * A newly created bic indicates that the process has just
@@ -3745,11 +5138,11 @@ index d1f648d..3bc1f8b 100644
 -	 * as needing weight raising.
 -	 */
 -	bic->wr_time_left = 1;
-+	bic->ttime.last_end_request = bfq_smallest_from_now();
++	icq_to_bic(icq)->ttime.last_end_request = ktime_get_ns() - (1ULL<<32);
  }
  
  static void bfq_exit_icq(struct io_cq *icq)
-@@ -3046,21 +3564,21 @@ static void bfq_exit_icq(struct io_cq *icq)
+@@ -3050,21 +3879,21 @@ static void bfq_exit_icq(struct io_cq *icq)
  	struct bfq_io_cq *bic = icq_to_bic(icq);
  	struct bfq_data *bfqd = bic_to_bfqd(bic);
  
@@ -3778,26 +5171,27 @@ index d1f648d..3bc1f8b 100644
  	}
  }
  
-@@ -3068,7 +3586,8 @@ static void bfq_exit_icq(struct io_cq *icq)
+@@ -3072,8 +3901,8 @@ static void bfq_exit_icq(struct io_cq *icq)
   * Update the entity prio values; note that the new values will not
   * be used until the next (re)activation.
   */
--static void bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic)
+-static void
+-bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic)
 +static void bfq_set_next_ioprio_data(struct bfq_queue *bfqq,
 +				     struct bfq_io_cq *bic)
  {
  	struct task_struct *tsk = current;
  	int ioprio_class;
-@@ -3100,7 +3619,7 @@ static void bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *b
+@@ -3105,7 +3934,7 @@ bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic)
  		break;
  	}
  
 -	if (bfqq->new_ioprio < 0 || bfqq->new_ioprio >= IOPRIO_BE_NR) {
 +	if (bfqq->new_ioprio >= IOPRIO_BE_NR) {
- 		printk(KERN_CRIT "bfq_set_next_ioprio_data: new_ioprio %d\n",
- 				 bfqq->new_ioprio);
+ 		pr_crit("bfq_set_next_ioprio_data: new_ioprio %d\n",
+ 			bfqq->new_ioprio);
  		BUG();
-@@ -3108,45 +3627,40 @@ static void bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *b
+@@ -3113,45 +3942,40 @@ bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic)
  
  	bfqq->entity.new_weight = bfq_ioprio_to_weight(bfqq->new_ioprio);
  	bfqq->entity.prio_changed = 1;
@@ -3857,7 +5251,7 @@ index d1f648d..3bc1f8b 100644
  }
  
  static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
-@@ -3155,8 +3669,9 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -3160,8 +3984,9 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  	RB_CLEAR_NODE(&bfqq->entity.rb_node);
  	INIT_LIST_HEAD(&bfqq->fifo);
  	INIT_HLIST_NODE(&bfqq->burst_list_node);
@@ -3868,7 +5262,7 @@ index d1f648d..3bc1f8b 100644
  	bfqq->bfqd = bfqd;
  
  	if (bic)
-@@ -3166,6 +3681,7 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -3171,6 +3996,7 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  		if (!bfq_class_idle(bfqq))
  			bfq_mark_bfqq_idle_window(bfqq);
  		bfq_mark_bfqq_sync(bfqq);
@@ -3876,14 +5270,16 @@ index d1f648d..3bc1f8b 100644
  	} else
  		bfq_clear_bfqq_sync(bfqq);
  	bfq_mark_bfqq_IO_bound(bfqq);
-@@ -3175,72 +3691,17 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -3180,72 +4006,19 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  	bfqq->pid = pid;
  
  	bfqq->wr_coeff = 1;
 -	bfqq->last_wr_start_finish = 0;
-+	bfqq->last_wr_start_finish = bfq_smallest_from_now();
++	bfqq->last_wr_start_finish = jiffies;
++	bfqq->wr_start_at_switch_to_srt = bfq_smallest_from_now();
 +	bfqq->budget_timeout = bfq_smallest_from_now();
 +	bfqq->split_time = bfq_smallest_from_now();
++
  	/*
  	 * Set to the value for which bfqq will not be deemed as
  	 * soft rt when it becomes backlogged.
@@ -3934,7 +5330,7 @@ index d1f648d..3bc1f8b 100644
 -
 -		if (bfqq) {
 -			bfq_init_bfqq(bfqd, bfqq, bic, current->pid,
--                                      is_sync);
+-				      is_sync);
 -			bfq_init_entity(&bfqq->entity, bfqg);
 -			bfq_log_bfqq(bfqd, bfqq, "allocated");
 -		} else {
@@ -3955,7 +5351,7 @@ index d1f648d..3bc1f8b 100644
  }
  
  static struct bfq_queue **bfq_async_queue_prio(struct bfq_data *bfqd,
-@@ -3263,44 +3724,60 @@ static struct bfq_queue **bfq_async_queue_prio(struct bfq_data *bfqd,
+@@ -3268,90 +4041,93 @@ static struct bfq_queue **bfq_async_queue_prio(struct bfq_data *bfqd,
  }
  
  static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd,
@@ -3976,7 +5372,7 @@ index d1f648d..3bc1f8b 100644
 -		struct bfq_group *bfqg;
 +	rcu_read_lock();
 +
-+	bfqg = bfq_find_set_group(bfqd,bio_blkcg(bio));
++	bfqg = bfq_find_set_group(bfqd, bio_blkcg(bio));
 +	if (!bfqg) {
 +		bfqq = &bfqd->oom_bfqq;
 +		goto out;
@@ -3996,7 +5392,8 @@ index d1f648d..3bc1f8b 100644
  
 -	if (!bfqq)
 -		bfqq = bfq_find_alloc_queue(bfqd, bio, is_sync, bic, gfp_mask);
-+	bfqq = kmem_cache_alloc_node(bfq_pool, GFP_NOWAIT | __GFP_ZERO,
++	bfqq = kmem_cache_alloc_node(bfq_pool,
++				     GFP_NOWAIT | __GFP_ZERO | __GFP_NOWARN,
 +				     bfqd->queue->node);
 +
 +	if (bfqq) {
@@ -4017,7 +5414,13 @@ index d1f648d..3bc1f8b 100644
 -	if (!is_sync && !(*async_bfqq)) {
 -		atomic_inc(&bfqq->ref);
 +	if (async_bfqq) {
-+		bfqq->ref++;
++		bfqq->ref++; /*
++			      * Extra group reference, w.r.t. sync
++			      * queue. This extra reference is removed
++			      * only if bfqq->bfqg disappears, to
++			      * guarantee that this queue is not freed
++			      * until its group goes away.
++			      */
  		bfq_log_bfqq(bfqd, bfqq, "get_queue, bfqq not in async: %p, %d",
 -			     bfqq, atomic_read(&bfqq->ref));
 +			     bfqq, bfqq->ref);
@@ -4034,14 +5437,33 @@ index d1f648d..3bc1f8b 100644
  	return bfqq;
  }
  
-@@ -3316,37 +3793,21 @@ static void bfq_update_io_thinktime(struct bfq_data *bfqd,
- 				bic->ttime.ttime_samples;
+ static void bfq_update_io_thinktime(struct bfq_data *bfqd,
+ 				    struct bfq_io_cq *bic)
+ {
+-	unsigned long elapsed = jiffies - bic->ttime.last_end_request;
+-	unsigned long ttime = min(elapsed, 2UL * bfqd->bfq_slice_idle);
++	struct bfq_ttime *ttime = &bic->ttime;
++	u64 elapsed = ktime_get_ns() - bic->ttime.last_end_request;
+ 
+-	bic->ttime.ttime_samples = (7*bic->ttime.ttime_samples + 256) / 8;
+-	bic->ttime.ttime_total = (7*bic->ttime.ttime_total + 256*ttime) / 8;
+-	bic->ttime.ttime_mean = (bic->ttime.ttime_total + 128) /
+-				bic->ttime.ttime_samples;
++	elapsed = min_t(u64, elapsed, 2 * bfqd->bfq_slice_idle);
++
++	ttime->ttime_samples = (7*bic->ttime.ttime_samples + 256) / 8;
++	ttime->ttime_total = div_u64(7*ttime->ttime_total + 256*elapsed,  8);
++	ttime->ttime_mean = div64_ul(ttime->ttime_total + 128,
++				     ttime->ttime_samples);
  }
  
 -static void bfq_update_io_seektime(struct bfq_data *bfqd,
 -				   struct bfq_queue *bfqq,
 -				   struct request *rq)
--{
++static void
++bfq_update_io_seektime(struct bfq_data *bfqd, struct bfq_queue *bfqq,
++		       struct request *rq)
+ {
 -	sector_t sdist;
 -	u64 total;
 -
@@ -4060,32 +5482,24 @@ index d1f648d..3bc1f8b 100644
 -		sdist = min(sdist, (bfqq->seek_mean * 4) + 2*1024*1024);
 -	else
 -		sdist = min(sdist, (bfqq->seek_mean * 4) + 2*1024*64);
- 
+-
 -	bfqq->seek_samples = (7*bfqq->seek_samples + 256) / 8;
 -	bfqq->seek_total = (7*bfqq->seek_total + (u64)256*sdist) / 8;
 -	total = bfqq->seek_total + (bfqq->seek_samples/2);
 -	do_div(total, bfqq->seek_samples);
 -	bfqq->seek_mean = (sector_t)total;
-+static void
-+bfq_update_io_seektime(struct bfq_data *bfqd, struct bfq_queue *bfqq,
-+		       struct request *rq)
-+{
-+	sector_t sdist = 0;
-+	if (bfqq->last_request_pos) {
-+		if (bfqq->last_request_pos < blk_rq_pos(rq))
-+			sdist = blk_rq_pos(rq) - bfqq->last_request_pos;
-+		else
-+			sdist = bfqq->last_request_pos - blk_rq_pos(rq);
-+	}
- 
+-
 -	bfq_log_bfqq(bfqd, bfqq, "dist=%llu mean=%llu", (u64)sdist,
 -			(u64)bfqq->seek_mean);
 +	bfqq->seek_history <<= 1;
-+	bfqq->seek_history |= (sdist > BFQQ_SEEK_THR);
++	bfqq->seek_history |=
++		get_sdist(bfqq->last_request_pos, rq) > BFQQ_SEEK_THR &&
++		(!blk_queue_nonrot(bfqd->queue) ||
++		 blk_rq_sectors(rq) < BFQQ_SECT_THR_NONROT);
  }
  
  /*
-@@ -3364,7 +3825,8 @@ static void bfq_update_idle_window(struct bfq_data *bfqd,
+@@ -3369,7 +4145,8 @@ static void bfq_update_idle_window(struct bfq_data *bfqd,
  		return;
  
  	/* Idle window just restored, statistics are meaningless. */
@@ -4095,7 +5509,7 @@ index d1f648d..3bc1f8b 100644
  		return;
  
  	enable_idle = bfq_bfqq_idle_window(bfqq);
-@@ -3404,22 +3866,13 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -3409,22 +4186,13 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  
  	bfq_update_io_thinktime(bfqd, bic);
  	bfq_update_io_seektime(bfqd, bfqq, rq);
@@ -4114,13 +5528,13 @@ index d1f648d..3bc1f8b 100644
  	bfq_log_bfqq(bfqd, bfqq,
 -		     "rq_enqueued: idle_window=%d (seeky %d, mean %llu)",
 -		     bfq_bfqq_idle_window(bfqq), BFQQ_SEEKY(bfqq),
--		     (long long unsigned)bfqq->seek_mean);
+-		     (unsigned long long) bfqq->seek_mean);
 +		     "rq_enqueued: idle_window=%d (seeky %d)",
 +		     bfq_bfqq_idle_window(bfqq), BFQQ_SEEKY(bfqq));
  
  	bfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq);
  
-@@ -3433,14 +3886,15 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -3438,14 +4206,15 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  		 * is small and the queue is not to be expired, then
  		 * just exit.
  		 *
@@ -4144,17 +5558,19 @@ index d1f648d..3bc1f8b 100644
  		 */
  		if (small_req && !budget_timeout)
  			return;
-@@ -3453,9 +3907,7 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -3457,10 +4226,8 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+ 		 * timer.
  		 */
  		bfq_clear_bfqq_wait_request(bfqq);
- 		del_timer(&bfqd->idle_slice_timer);
+-		del_timer(&bfqd->idle_slice_timer);
 -#ifdef CONFIG_BFQ_GROUP_IOSCHED
++		hrtimer_try_to_cancel(&bfqd->idle_slice_timer);
  		bfqg_stats_update_idle_time(bfqq_group(bfqq));
 -#endif
  
  		/*
  		 * The queue is not empty, because a new request just
-@@ -3499,27 +3951,19 @@ static void bfq_insert_request(struct request_queue *q, struct request *rq)
+@@ -3504,28 +4271,20 @@ static void bfq_insert_request(struct request_queue *q, struct request *rq)
  			 */
  			new_bfqq->allocated[rq_data_dir(rq)]++;
  			bfqq->allocated[rq_data_dir(rq)]--;
@@ -4182,10 +5598,12 @@ index d1f648d..3bc1f8b 100644
 -	 */
 -	if (bfqq->bic)
 -		bfqq->bic->wr_time_left = 0;
- 	rq->fifo_time = jiffies + bfqd->bfq_fifo_expire[rq_is_sync(rq)];
+-	rq->fifo_time = jiffies + bfqd->bfq_fifo_expire[rq_is_sync(rq)];
++	rq->fifo_time = ktime_get_ns() + bfqd->bfq_fifo_expire[rq_is_sync(rq)];
  	list_add_tail(&rq->queuelist, &bfqq->fifo);
  
-@@ -3528,8 +3972,8 @@ static void bfq_insert_request(struct request_queue *q, struct request *rq)
+ 	bfq_rq_enqueued(bfqd, bfqq, rq);
+@@ -3533,8 +4292,8 @@ static void bfq_insert_request(struct request_queue *q, struct request *rq)
  
  static void bfq_update_hw_tag(struct bfq_data *bfqd)
  {
@@ -4196,11 +5614,13 @@ index d1f648d..3bc1f8b 100644
  
  	if (bfqd->hw_tag == 1)
  		return;
-@@ -3555,48 +3999,45 @@ static void bfq_completed_request(struct request_queue *q, struct request *rq)
+@@ -3560,48 +4319,85 @@ static void bfq_completed_request(struct request_queue *q, struct request *rq)
  {
  	struct bfq_queue *bfqq = RQ_BFQQ(rq);
  	struct bfq_data *bfqd = bfqq->bfqd;
 -	bool sync = bfq_bfqq_sync(bfqq);
++	u64 now_ns;
++	u32 delta_us;
  
 -	bfq_log_bfqq(bfqd, bfqq, "completed one req with %u sects left (%d)",
 -		     blk_rq_sectors(rq), sync);
@@ -4217,8 +5637,10 @@ index d1f648d..3bc1f8b 100644
 -#ifdef CONFIG_BFQ_GROUP_IOSCHED
  	bfqg_stats_update_completion(bfqq_group(bfqq),
  				     rq_start_time_ns(rq),
- 				     rq_io_start_time_ns(rq), rq->cmd_flags);
+-				     rq_io_start_time_ns(rq), rq->cmd_flags);
 -#endif
++				     rq_io_start_time_ns(rq),
++				     rq->cmd_flags);
  
  	if (!bfqq->dispatched && !bfq_bfqq_busy(bfqq)) {
 +		BUG_ON(!RB_EMPTY_ROOT(&bfqq->sort_list));
@@ -4247,7 +5669,44 @@ index d1f648d..3bc1f8b 100644
 -		bfqd->sync_flight--;
 -		RQ_BIC(rq)->ttime.last_end_request = jiffies;
 -	}
-+	RQ_BIC(rq)->ttime.last_end_request = jiffies;
++	now_ns = ktime_get_ns();
++
++	RQ_BIC(rq)->ttime.last_end_request = now_ns;
++
++	/*
++	 * Using us instead of ns, to get a reasonable precision in
++	 * computing rate in next check.
++	 */
++	delta_us = div_u64(now_ns - bfqd->last_completion, NSEC_PER_USEC);
++
++	bfq_log(bfqd, "rq_completed: delta %uus/%luus max_size %u rate %llu/%llu",
++		delta_us, BFQ_MIN_TT/NSEC_PER_USEC, bfqd->last_rq_max_size,
++		(USEC_PER_SEC*
++		(u64)((bfqd->last_rq_max_size<<BFQ_RATE_SHIFT)/delta_us))
++			>>BFQ_RATE_SHIFT,
++		(USEC_PER_SEC*(u64)(1UL<<(BFQ_RATE_SHIFT-10)))>>BFQ_RATE_SHIFT);
++
++	/*
++	 * If the request took rather long to complete, and, according
++	 * to the maximum request size recorded, this completion latency
++	 * implies that the request was certainly served at a very low
++	 * rate (less than 1M sectors/sec), then the whole observation
++	 * interval that lasts up to this time instant cannot be a
++	 * valid time interval for computing a new peak rate.  Invoke
++	 * bfq_update_rate_reset to have the following three steps
++	 * taken:
++	 * - close the observation interval at the last (previous)
++	 *   request dispatch or completion
++	 * - compute rate, if possible, for that observation interval
++	 * - reset to zero samples, which will trigger a proper
++	 *   re-initialization of the observation interval on next
++	 *   dispatch
++	 */
++	if (delta_us > BFQ_MIN_TT/NSEC_PER_USEC &&
++	   (bfqd->last_rq_max_size<<BFQ_RATE_SHIFT)/delta_us <
++			1UL<<(BFQ_RATE_SHIFT - 10))
++		bfq_update_rate_reset(bfqd, NULL);
++	bfqd->last_completion = now_ns;
  
  	/*
 -	 * If we are waiting to discover whether the request pattern of the
@@ -4265,7 +5724,7 @@ index d1f648d..3bc1f8b 100644
  	 */
  	if (bfq_bfqq_softrt_update(bfqq) && bfqq->dispatched == 0 &&
  	    RB_EMPTY_ROOT(&bfqq->sort_list))
-@@ -3608,10 +4049,7 @@ static void bfq_completed_request(struct request_queue *q, struct request *rq)
+@@ -3613,10 +4409,7 @@ static void bfq_completed_request(struct request_queue *q, struct request *rq)
  	 * or if we want to idle in case it has no pending requests.
  	 */
  	if (bfqd->in_service_queue == bfqq) {
@@ -4277,7 +5736,25 @@ index d1f648d..3bc1f8b 100644
  			bfq_arm_slice_timer(bfqd);
  			goto out;
  		} else if (bfq_may_expire_for_budg_timeout(bfqq))
-@@ -3682,14 +4120,14 @@ static void bfq_put_request(struct request *rq)
+@@ -3646,7 +4439,7 @@ static int __bfq_may_queue(struct bfq_queue *bfqq)
+ 	return ELV_MQUEUE_MAY;
+ }
+ 
+-static int bfq_may_queue(struct request_queue *q, int rw)
++static int bfq_may_queue(struct request_queue *q, unsigned int op)
+ {
+ 	struct bfq_data *bfqd = q->elevator->elevator_data;
+ 	struct task_struct *tsk = current;
+@@ -3663,7 +4456,7 @@ static int bfq_may_queue(struct request_queue *q, int rw)
+ 	if (!bic)
+ 		return ELV_MQUEUE_MAY;
+ 
+-	bfqq = bic_to_bfqq(bic, rw_is_sync(rw));
++	bfqq = bic_to_bfqq(bic, op_is_sync(op));
+ 	if (bfqq)
+ 		return __bfq_may_queue(bfqq);
+ 
+@@ -3687,14 +4480,14 @@ static void bfq_put_request(struct request *rq)
  		rq->elv.priv[1] = NULL;
  
  		bfq_log_bfqq(bfqq->bfqd, bfqq, "put_request %p, %d",
@@ -4294,7 +5771,7 @@ index d1f648d..3bc1f8b 100644
   */
  static struct bfq_queue *
  bfq_split_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq)
-@@ -3727,11 +4165,8 @@ static int bfq_set_request(struct request_queue *q, struct request *rq,
+@@ -3732,11 +4525,8 @@ static int bfq_set_request(struct request_queue *q, struct request *rq,
  	unsigned long flags;
  	bool split = false;
  
@@ -4307,7 +5784,7 @@ index d1f648d..3bc1f8b 100644
  
  	if (!bic)
  		goto queue_fail;
-@@ -3741,23 +4176,47 @@ static int bfq_set_request(struct request_queue *q, struct request *rq,
+@@ -3746,23 +4536,47 @@ static int bfq_set_request(struct request_queue *q, struct request *rq,
  new_queue:
  	bfqq = bic_to_bfqq(bic, is_sync);
  	if (!bfqq || bfqq == &bfqd->oom_bfqq) {
@@ -4335,18 +5812,14 @@ index d1f648d..3bc1f8b 100644
 +					     "large burst");
  				bfq_mark_bfqq_in_large_burst(bfqq);
 -			else {
--			    bfq_clear_bfqq_in_large_burst(bfqq);
--			    if (bic->was_in_burst_list)
--			       hlist_add_head(&bfqq->burst_list_node,
--				              &bfqd->burst_list);
 +			} else {
 +				bfq_log_bfqq(bfqd, bfqq,
 +					     "set_request: clearing in "
 +					     "large burst");
-+				bfq_clear_bfqq_in_large_burst(bfqq);
-+				if (bic->was_in_burst_list)
-+					hlist_add_head(&bfqq->burst_list_node,
-+						       &bfqd->burst_list);
+ 				bfq_clear_bfqq_in_large_burst(bfqq);
+ 				if (bic->was_in_burst_list)
+ 					hlist_add_head(&bfqq->burst_list_node,
+ 						       &bfqd->burst_list);
  			}
 +			bfqq->split_time = jiffies;
  		}
@@ -4362,7 +5835,7 @@ index d1f648d..3bc1f8b 100644
  			bfqq = bfq_split_bfqq(bic, bfqq);
  			split = true;
  			if (!bfqq)
-@@ -3766,9 +4225,8 @@ new_queue:
+@@ -3771,9 +4585,8 @@ static int bfq_set_request(struct request_queue *q, struct request *rq,
  	}
  
  	bfqq->allocated[rw]++;
@@ -4374,7 +5847,7 @@ index d1f648d..3bc1f8b 100644
  
  	rq->elv.priv[0] = bic;
  	rq->elv.priv[1] = bfqq;
-@@ -3783,7 +4241,6 @@ new_queue:
+@@ -3788,7 +4601,6 @@ static int bfq_set_request(struct request_queue *q, struct request *rq,
  	if (likely(bfqq != &bfqd->oom_bfqq) && bfqq_process_refs(bfqq) == 1) {
  		bfqq->bic = bic;
  		if (split) {
@@ -4382,7 +5855,7 @@ index d1f648d..3bc1f8b 100644
  			/*
  			 * If the queue has just been split from a shared
  			 * queue, restore the idle window and the possible
-@@ -3793,6 +4250,9 @@ new_queue:
+@@ -3798,6 +4610,9 @@ static int bfq_set_request(struct request_queue *q, struct request *rq,
  		}
  	}
  
@@ -4392,15 +5865,43 @@ index d1f648d..3bc1f8b 100644
  	spin_unlock_irqrestore(q->queue_lock, flags);
  
  	return 0;
-@@ -3872,6 +4332,7 @@ static void bfq_shutdown_timer_wq(struct bfq_data *bfqd)
+@@ -3824,9 +4639,10 @@ static void bfq_kick_queue(struct work_struct *work)
+  * Handler of the expiration of the timer running if the in-service queue
+  * is idling inside its time slice.
+  */
+-static void bfq_idle_slice_timer(unsigned long data)
++static enum hrtimer_restart bfq_idle_slice_timer(struct hrtimer *timer)
+ {
+-	struct bfq_data *bfqd = (struct bfq_data *)data;
++	struct bfq_data *bfqd = container_of(timer, struct bfq_data,
++					     idle_slice_timer);
+ 	struct bfq_queue *bfqq;
+ 	unsigned long flags;
+ 	enum bfqq_expiration reason;
+@@ -3844,6 +4660,8 @@ static void bfq_idle_slice_timer(unsigned long data)
+ 	 */
+ 	if (bfqq) {
+ 		bfq_log_bfqq(bfqd, bfqq, "slice_timer expired");
++		bfq_clear_bfqq_wait_request(bfqq);
++
+ 		if (bfq_bfqq_budget_timeout(bfqq))
+ 			/*
+ 			 * Also here the queue can be safely expired
+@@ -3869,11 +4687,12 @@ static void bfq_idle_slice_timer(unsigned long data)
+ 	bfq_schedule_dispatch(bfqd);
+ 
+ 	spin_unlock_irqrestore(bfqd->queue->queue_lock, flags);
++	return HRTIMER_NORESTART;
+ }
+ 
+ static void bfq_shutdown_timer_wq(struct bfq_data *bfqd)
+ {
+-	del_timer_sync(&bfqd->idle_slice_timer);
++	hrtimer_cancel(&bfqd->idle_slice_timer);
  	cancel_work_sync(&bfqd->unplug_work);
  }
  
-+#ifdef CONFIG_BFQ_GROUP_IOSCHED
- static void __bfq_put_async_bfqq(struct bfq_data *bfqd,
- 					struct bfq_queue **bfqq_ptr)
- {
-@@ -3880,9 +4341,9 @@ static void __bfq_put_async_bfqq(struct bfq_data *bfqd,
+@@ -3885,9 +4704,9 @@ static void __bfq_put_async_bfqq(struct bfq_data *bfqd,
  
  	bfq_log(bfqd, "put_async_bfqq: %p", bfqq);
  	if (bfqq) {
@@ -4412,24 +5913,38 @@ index d1f648d..3bc1f8b 100644
  		bfq_put_queue(bfqq);
  		*bfqq_ptr = NULL;
  	}
-@@ -3904,6 +4365,7 @@ static void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg)
+@@ -3922,19 +4741,18 @@ static void bfq_exit_queue(struct elevator_queue *e)
  
- 	__bfq_put_async_bfqq(bfqd, &bfqg->async_idle_bfqq);
- }
-+#endif
+ 	BUG_ON(bfqd->in_service_queue);
+ 	list_for_each_entry_safe(bfqq, n, &bfqd->idle_list, bfqq_list)
+-		bfq_deactivate_bfqq(bfqd, bfqq, 0);
++		bfq_deactivate_bfqq(bfqd, bfqq, false, false);
  
- static void bfq_exit_queue(struct elevator_queue *e)
- {
-@@ -3923,8 +4385,6 @@ static void bfq_exit_queue(struct elevator_queue *e)
+ 	spin_unlock_irq(q->queue_lock);
  
  	bfq_shutdown_timer_wq(bfqd);
  
 -	synchronize_rcu();
 -
- 	BUG_ON(timer_pending(&bfqd->idle_slice_timer));
+-	BUG_ON(timer_pending(&bfqd->idle_slice_timer));
++	BUG_ON(hrtimer_active(&bfqd->idle_slice_timer));
  
  #ifdef CONFIG_BFQ_GROUP_IOSCHED
-@@ -3973,11 +4433,14 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
+ 	blkcg_deactivate_policy(q, &blkcg_policy_bfq);
+ #else
++	bfq_put_async_queues(bfqd, bfqd->root_group);
+ 	kfree(bfqd->root_group);
+ #endif
+ 
+@@ -3954,6 +4772,7 @@ static void bfq_init_root_group(struct bfq_group *root_group,
+ 	root_group->rq_pos_tree = RB_ROOT;
+ 	for (i = 0; i < BFQ_IOPRIO_CLASSES; i++)
+ 		root_group->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT;
++	root_group->sched_data.bfq_class_idle_last_service = jiffies;
+ }
+ 
+ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
+@@ -3978,11 +4797,14 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
  	 * will not attempt to free it.
  	 */
  	bfq_init_bfqq(bfqd, &bfqd->oom_bfqq, NULL, 1, 0);
@@ -4445,7 +5960,7 @@ index d1f648d..3bc1f8b 100644
  	/*
  	 * Trigger weight initialization, according to ioprio, at the
  	 * oom_bfqq's first activation. The oom_bfqq's ioprio and ioprio
-@@ -3996,9 +4459,6 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
+@@ -4001,13 +4823,10 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
  		goto out_free;
  	bfq_init_root_group(bfqd->root_group, bfqd);
  	bfq_init_entity(&bfqd->oom_bfqq.entity, bfqd->root_group);
@@ -4453,12 +5968,19 @@ index d1f648d..3bc1f8b 100644
 -	bfqd->active_numerous_groups = 0;
 -#endif
  
- 	init_timer(&bfqd->idle_slice_timer);
+-	init_timer(&bfqd->idle_slice_timer);
++	hrtimer_init(&bfqd->idle_slice_timer, CLOCK_MONOTONIC,
++		     HRTIMER_MODE_REL);
  	bfqd->idle_slice_timer.function = bfq_idle_slice_timer;
-@@ -4023,20 +4483,19 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
+-	bfqd->idle_slice_timer.data = (unsigned long)bfqd;
+ 
+ 	bfqd->queue_weights_tree = RB_ROOT;
+ 	bfqd->group_weights_tree = RB_ROOT;
+@@ -4027,21 +4846,19 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
+ 	bfqd->bfq_back_max = bfq_back_max;
  	bfqd->bfq_back_penalty = bfq_back_penalty;
  	bfqd->bfq_slice_idle = bfq_slice_idle;
- 	bfqd->bfq_class_idle_last_service = 0;
+-	bfqd->bfq_class_idle_last_service = 0;
 -	bfqd->bfq_max_budget_async_rq = bfq_max_budget_async_rq;
 -	bfqd->bfq_timeout[BLK_RW_ASYNC] = bfq_timeout_async;
 -	bfqd->bfq_timeout[BLK_RW_SYNC] = bfq_timeout_sync;
@@ -4483,7 +6005,7 @@ index d1f648d..3bc1f8b 100644
  	bfqd->bfq_wr_rt_max_time = msecs_to_jiffies(300);
  	bfqd->bfq_wr_max_time = 0;
  	bfqd->bfq_wr_min_idle_time = msecs_to_jiffies(2000);
-@@ -4048,16 +4507,15 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
+@@ -4053,16 +4870,15 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
  					      * video.
  					      */
  	bfqd->wr_busy_queues = 0;
@@ -4504,9 +6026,36 @@ index d1f648d..3bc1f8b 100644
  	bfqd->device_speed = BFQ_BFQD_FAST;
  
  	return 0;
-@@ -4161,10 +4619,8 @@ SHOW_FUNCTION(bfq_back_seek_max_show, bfqd->bfq_back_max, 0);
+@@ -4088,7 +4904,7 @@ static int __init bfq_slab_setup(void)
+ 
+ static ssize_t bfq_var_show(unsigned int var, char *page)
+ {
+-	return sprintf(page, "%d\n", var);
++	return sprintf(page, "%u\n", var);
+ }
+ 
+ static ssize_t bfq_var_store(unsigned long *var, const char *page,
+@@ -4159,21 +4975,21 @@ static ssize_t bfq_weights_show(struct elevator_queue *e, char *page)
+ static ssize_t __FUNC(struct elevator_queue *e, char *page)		\
+ {									\
+ 	struct bfq_data *bfqd = e->elevator_data;			\
+-	unsigned int __data = __VAR;					\
+-	if (__CONV)							\
++	u64 __data = __VAR;						\
++	if (__CONV == 1)						\
+ 		__data = jiffies_to_msecs(__data);			\
++	else if (__CONV == 2)						\
++		__data = div_u64(__data, NSEC_PER_MSEC);		\
+ 	return bfq_var_show(__data, (page));				\
+ }
+-SHOW_FUNCTION(bfq_fifo_expire_sync_show, bfqd->bfq_fifo_expire[1], 1);
+-SHOW_FUNCTION(bfq_fifo_expire_async_show, bfqd->bfq_fifo_expire[0], 1);
++SHOW_FUNCTION(bfq_fifo_expire_sync_show, bfqd->bfq_fifo_expire[1], 2);
++SHOW_FUNCTION(bfq_fifo_expire_async_show, bfqd->bfq_fifo_expire[0], 2);
+ SHOW_FUNCTION(bfq_back_seek_max_show, bfqd->bfq_back_max, 0);
  SHOW_FUNCTION(bfq_back_seek_penalty_show, bfqd->bfq_back_penalty, 0);
- SHOW_FUNCTION(bfq_slice_idle_show, bfqd->bfq_slice_idle, 1);
+-SHOW_FUNCTION(bfq_slice_idle_show, bfqd->bfq_slice_idle, 1);
++SHOW_FUNCTION(bfq_slice_idle_show, bfqd->bfq_slice_idle, 2);
  SHOW_FUNCTION(bfq_max_budget_show, bfqd->bfq_user_max_budget, 0);
 -SHOW_FUNCTION(bfq_max_budget_async_rq_show,
 -	      bfqd->bfq_max_budget_async_rq, 0);
@@ -4517,52 +6066,129 @@ index d1f648d..3bc1f8b 100644
  SHOW_FUNCTION(bfq_low_latency_show, bfqd->low_latency, 0);
  SHOW_FUNCTION(bfq_wr_coeff_show, bfqd->bfq_wr_coeff, 0);
  SHOW_FUNCTION(bfq_wr_rt_max_time_show, bfqd->bfq_wr_rt_max_time, 1);
-@@ -4199,10 +4655,6 @@ STORE_FUNCTION(bfq_back_seek_max_store, &bfqd->bfq_back_max, 0, INT_MAX, 0);
+@@ -4183,6 +4999,17 @@ SHOW_FUNCTION(bfq_wr_min_inter_arr_async_show, bfqd->bfq_wr_min_inter_arr_async,
+ SHOW_FUNCTION(bfq_wr_max_softrt_rate_show, bfqd->bfq_wr_max_softrt_rate, 0);
+ #undef SHOW_FUNCTION
+ 
++#define USEC_SHOW_FUNCTION(__FUNC, __VAR)				\
++static ssize_t __FUNC(struct elevator_queue *e, char *page)		\
++{									\
++	struct bfq_data *bfqd = e->elevator_data;			\
++	u64 __data = __VAR;						\
++	__data = div_u64(__data, NSEC_PER_USEC);			\
++	return bfq_var_show(__data, (page));				\
++}
++USEC_SHOW_FUNCTION(bfq_slice_idle_us_show, bfqd->bfq_slice_idle);
++#undef USEC_SHOW_FUNCTION
++
+ #define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV)			\
+ static ssize_t								\
+ __FUNC(struct elevator_queue *e, const char *page, size_t count)	\
+@@ -4194,24 +5021,22 @@ __FUNC(struct elevator_queue *e, const char *page, size_t count)	\
+ 		__data = (MIN);						\
+ 	else if (__data > (MAX))					\
+ 		__data = (MAX);						\
+-	if (__CONV)							\
++	if (__CONV == 1)						\
+ 		*(__PTR) = msecs_to_jiffies(__data);			\
++	else if (__CONV == 2)						\
++		*(__PTR) = (u64)__data * NSEC_PER_MSEC;			\
+ 	else								\
+ 		*(__PTR) = __data;					\
+ 	return ret;							\
+ }
+ STORE_FUNCTION(bfq_fifo_expire_sync_store, &bfqd->bfq_fifo_expire[1], 1,
+-		INT_MAX, 1);
++		INT_MAX, 2);
+ STORE_FUNCTION(bfq_fifo_expire_async_store, &bfqd->bfq_fifo_expire[0], 1,
+-		INT_MAX, 1);
++		INT_MAX, 2);
+ STORE_FUNCTION(bfq_back_seek_max_store, &bfqd->bfq_back_max, 0, INT_MAX, 0);
  STORE_FUNCTION(bfq_back_seek_penalty_store, &bfqd->bfq_back_penalty, 1,
  		INT_MAX, 0);
- STORE_FUNCTION(bfq_slice_idle_store, &bfqd->bfq_slice_idle, 0, INT_MAX, 1);
+-STORE_FUNCTION(bfq_slice_idle_store, &bfqd->bfq_slice_idle, 0, INT_MAX, 1);
 -STORE_FUNCTION(bfq_max_budget_async_rq_store, &bfqd->bfq_max_budget_async_rq,
 -		1, INT_MAX, 0);
 -STORE_FUNCTION(bfq_timeout_async_store, &bfqd->bfq_timeout[BLK_RW_ASYNC], 0,
 -		INT_MAX, 1);
++STORE_FUNCTION(bfq_slice_idle_store, &bfqd->bfq_slice_idle, 0, INT_MAX, 2);
  STORE_FUNCTION(bfq_wr_coeff_store, &bfqd->bfq_wr_coeff, 1, INT_MAX, 0);
  STORE_FUNCTION(bfq_wr_max_time_store, &bfqd->bfq_wr_max_time, 0, INT_MAX, 1);
  STORE_FUNCTION(bfq_wr_rt_max_time_store, &bfqd->bfq_wr_rt_max_time, 0, INT_MAX,
-@@ -4224,10 +4676,8 @@ static ssize_t bfq_weights_store(struct elevator_queue *e,
+@@ -4224,6 +5049,23 @@ STORE_FUNCTION(bfq_wr_max_softrt_rate_store, &bfqd->bfq_wr_max_softrt_rate, 0,
+ 		INT_MAX, 0);
+ #undef STORE_FUNCTION
  
- static unsigned long bfq_estimated_max_budget(struct bfq_data *bfqd)
- {
++#define USEC_STORE_FUNCTION(__FUNC, __PTR, MIN, MAX)			\
++static ssize_t __FUNC(struct elevator_queue *e, const char *page, size_t count)\
++{									\
++	struct bfq_data *bfqd = e->elevator_data;			\
++	unsigned long uninitialized_var(__data);			\
++	int ret = bfq_var_store(&__data, (page), count);		\
++	if (__data < (MIN))						\
++		__data = (MIN);						\
++	else if (__data > (MAX))					\
++		__data = (MAX);						\
++	*(__PTR) = (u64)__data * NSEC_PER_USEC;				\
++	return ret;							\
++}
++USEC_STORE_FUNCTION(bfq_slice_idle_us_store, &bfqd->bfq_slice_idle, 0,
++		    UINT_MAX);
++#undef USEC_STORE_FUNCTION
++
+ /* do nothing for the moment */
+ static ssize_t bfq_weights_store(struct elevator_queue *e,
+ 				    const char *page, size_t count)
+@@ -4231,16 +5073,6 @@ static ssize_t bfq_weights_store(struct elevator_queue *e,
+ 	return count;
+ }
+ 
+-static unsigned long bfq_estimated_max_budget(struct bfq_data *bfqd)
+-{
 -	u64 timeout = jiffies_to_msecs(bfqd->bfq_timeout[BLK_RW_SYNC]);
 -
- 	if (bfqd->peak_rate_samples >= BFQ_PEAK_RATE_SAMPLES)
+-	if (bfqd->peak_rate_samples >= BFQ_PEAK_RATE_SAMPLES)
 -		return bfq_calc_max_budget(bfqd->peak_rate, timeout);
-+		return bfq_calc_max_budget(bfqd);
- 	else
- 		return bfq_default_max_budget;
- }
-@@ -4252,6 +4702,10 @@ static ssize_t bfq_max_budget_store(struct elevator_queue *e,
+-	else
+-		return bfq_default_max_budget;
+-}
+-
+ static ssize_t bfq_max_budget_store(struct elevator_queue *e,
+ 				    const char *page, size_t count)
+ {
+@@ -4249,7 +5081,7 @@ static ssize_t bfq_max_budget_store(struct elevator_queue *e,
+ 	int ret = bfq_var_store(&__data, (page), count);
+ 
+ 	if (__data == 0)
+-		bfqd->bfq_max_budget = bfq_estimated_max_budget(bfqd);
++		bfqd->bfq_max_budget = bfq_calc_max_budget(bfqd);
+ 	else {
+ 		if (__data > INT_MAX)
+ 			__data = INT_MAX;
+@@ -4261,6 +5093,10 @@ static ssize_t bfq_max_budget_store(struct elevator_queue *e,
  	return ret;
  }
  
-+/* 
++/*
 + * Leaving this name to preserve name compatibility with cfq
 + * parameters, but this timeout is used for both sync and async.
 + */
  static ssize_t bfq_timeout_sync_store(struct elevator_queue *e,
  				      const char *page, size_t count)
  {
-@@ -4264,13 +4718,31 @@ static ssize_t bfq_timeout_sync_store(struct elevator_queue *e,
+@@ -4273,9 +5109,27 @@ static ssize_t bfq_timeout_sync_store(struct elevator_queue *e,
  	else if (__data > INT_MAX)
  		__data = INT_MAX;
  
 -	bfqd->bfq_timeout[BLK_RW_SYNC] = msecs_to_jiffies(__data);
 +	bfqd->bfq_timeout = msecs_to_jiffies(__data);
  	if (bfqd->bfq_user_max_budget == 0)
- 		bfqd->bfq_max_budget = bfq_estimated_max_budget(bfqd);
- 
- 	return ret;
- }
- 
+-		bfqd->bfq_max_budget = bfq_estimated_max_budget(bfqd);
++		bfqd->bfq_max_budget = bfq_calc_max_budget(bfqd);
++
++	return ret;
++}
++
 +static ssize_t bfq_strict_guarantees_store(struct elevator_queue *e,
 +				     const char *page, size_t count)
 +{
@@ -4573,20 +6199,18 @@ index d1f648d..3bc1f8b 100644
 +	if (__data > 1)
 +		__data = 1;
 +	if (!bfqd->strict_guarantees && __data == 1
-+	    && bfqd->bfq_slice_idle < msecs_to_jiffies(8))
-+		bfqd->bfq_slice_idle = msecs_to_jiffies(8);
++	    && bfqd->bfq_slice_idle < 8 * NSEC_PER_MSEC)
++		bfqd->bfq_slice_idle = 8 * NSEC_PER_MSEC;
 +
 +	bfqd->strict_guarantees = __data;
-+
-+	return ret;
-+}
-+
- static ssize_t bfq_low_latency_store(struct elevator_queue *e,
- 				     const char *page, size_t count)
- {
-@@ -4297,9 +4769,8 @@ static struct elv_fs_entry bfq_attrs[] = {
+ 
+ 	return ret;
+ }
+@@ -4305,10 +5159,10 @@ static struct elv_fs_entry bfq_attrs[] = {
+ 	BFQ_ATTR(back_seek_max),
  	BFQ_ATTR(back_seek_penalty),
  	BFQ_ATTR(slice_idle),
++	BFQ_ATTR(slice_idle_us),
  	BFQ_ATTR(max_budget),
 -	BFQ_ATTR(max_budget_async_rq),
  	BFQ_ATTR(timeout_sync),
@@ -4595,7 +6219,17 @@ index d1f648d..3bc1f8b 100644
  	BFQ_ATTR(low_latency),
  	BFQ_ATTR(wr_coeff),
  	BFQ_ATTR(wr_max_time),
-@@ -4342,9 +4813,28 @@ static struct elevator_type iosched_bfq = {
+@@ -4328,7 +5182,8 @@ static struct elevator_type iosched_bfq = {
+ #ifdef CONFIG_BFQ_GROUP_IOSCHED
+ 		.elevator_bio_merged_fn =	bfq_bio_merged,
+ #endif
+-		.elevator_allow_merge_fn =	bfq_allow_merge,
++		.elevator_allow_bio_merge_fn =	bfq_allow_bio_merge,
++		.elevator_allow_rq_merge_fn =	bfq_allow_rq_merge,
+ 		.elevator_dispatch_fn =		bfq_dispatch_requests,
+ 		.elevator_add_req_fn =		bfq_insert_request,
+ 		.elevator_activate_req_fn =	bfq_activate_request,
+@@ -4351,18 +5206,28 @@ static struct elevator_type iosched_bfq = {
  	.elevator_owner =	THIS_MODULE,
  };
  
@@ -4620,32 +6254,46 @@ index d1f648d..3bc1f8b 100644
  static int __init bfq_init(void)
  {
  	int ret;
-+	char msg[50] = "BFQ I/O-scheduler: v8r3";
- 
- 	/*
- 	 * Can be 0 on HZ < 1000 setups.
-@@ -4352,9 +4842,6 @@ static int __init bfq_init(void)
- 	if (bfq_slice_idle == 0)
- 		bfq_slice_idle = 1;
- 
+-
+-	/*
+-	 * Can be 0 on HZ < 1000 setups.
+-	 */
+-	if (bfq_slice_idle == 0)
+-		bfq_slice_idle = 1;
+-
 -	if (bfq_timeout_async == 0)
 -		bfq_timeout_async = 1;
--
++	char msg[60] = "BFQ I/O-scheduler: v8r8";
+ 
  #ifdef CONFIG_BFQ_GROUP_IOSCHED
  	ret = blkcg_policy_register(&blkcg_policy_bfq);
- 	if (ret)
-@@ -4370,23 +4857,34 @@ static int __init bfq_init(void)
- 	 * installed on the reference devices (see the comments before the
- 	 * definitions of the two arrays).
+@@ -4375,27 +5240,46 @@ static int __init bfq_init(void)
+ 		goto err_pol_unreg;
+ 
+ 	/*
+-	 * Times to load large popular applications for the typical systems
+-	 * installed on the reference devices (see the comments before the
+-	 * definitions of the two arrays).
++	 * Times to load large popular applications for the typical
++	 * systems installed on the reference devices (see the
++	 * comments before the definitions of the next two
++	 * arrays). Actually, we use slightly slower values, as the
++	 * estimated peak rate tends to be smaller than the actual
++	 * peak rate.  The reason for this last fact is that estimates
++	 * are computed over much shorter time intervals than the long
++	 * intervals typically used for benchmarking. Why? First, to
++	 * adapt more quickly to variations. Second, because an I/O
++	 * scheduler cannot rely on a peak-rate-evaluation workload to
++	 * be run for a long time.
  	 */
 -	T_slow[0] = msecs_to_jiffies(2600);
 -	T_slow[1] = msecs_to_jiffies(1000);
 -	T_fast[0] = msecs_to_jiffies(5500);
 -	T_fast[1] = msecs_to_jiffies(2000);
-+	T_slow[0] = msecs_to_jiffies(3500);
-+	T_slow[1] = msecs_to_jiffies(1500);
-+	T_fast[0] = msecs_to_jiffies(8000);
-+	T_fast[1] = msecs_to_jiffies(3000);
++	T_slow[0] = msecs_to_jiffies(3500); /* actually 4 sec */
++	T_slow[1] = msecs_to_jiffies(6000); /* actually 6.5 sec */
++	T_fast[0] = msecs_to_jiffies(7000); /* actually 8 sec */
++	T_fast[1] = msecs_to_jiffies(2500); /* actually 3 sec */
  
  	/*
 -	 * Thresholds that determine the switch between speed classes (see
@@ -4679,50 +6327,145 @@ index d1f648d..3bc1f8b 100644
  	return 0;
  
 diff --git a/block/bfq-sched.c b/block/bfq-sched.c
-index a64fec1..7d73b9d 100644
+index a5ed694..2e9dc59 100644
 --- a/block/bfq-sched.c
 +++ b/block/bfq-sched.c
-@@ -7,9 +7,11 @@
+@@ -7,28 +7,166 @@
   * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
   *		      Paolo Valente <paolo.valente@unimore.it>
   *
 - * Copyright (C) 2010 Paolo Valente <paolo.valente@unimore.it>
-+ * Copyright (C) 2016 Paolo Valente <paolo.valente@unimore.it>
-  */
- 
++ * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
++ *
++ * Copyright (C) 2016 Paolo Valente <paolo.valente@linaro.org>
++ */
++
 +static struct bfq_group *bfqq_group(struct bfq_queue *bfqq);
 +
- #ifdef CONFIG_BFQ_GROUP_IOSCHED
- #define for_each_entity(entity)	\
- 	for (; entity ; entity = entity->parent)
-@@ -22,8 +24,6 @@ static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd,
- 						 int extract,
- 						 struct bfq_data *bfqd);
- 
--static struct bfq_group *bfqq_group(struct bfq_queue *bfqq);
--
- static void bfq_update_budget(struct bfq_entity *next_in_service)
- {
- 	struct bfq_entity *bfqg_entity;
-@@ -48,6 +48,7 @@ static void bfq_update_budget(struct bfq_entity *next_in_service)
- static int bfq_update_next_in_service(struct bfq_sched_data *sd)
- {
- 	struct bfq_entity *next_in_service;
++/**
++ * bfq_gt - compare two timestamps.
++ * @a: first ts.
++ * @b: second ts.
++ *
++ * Return @a > @b, dealing with wrapping correctly.
++ */
++static int bfq_gt(u64 a, u64 b)
++{
++	return (s64)(a - b) > 0;
++}
++
++static struct bfq_entity *bfq_root_active_entity(struct rb_root *tree)
++{
++	struct rb_node *node = tree->rb_node;
++
++	return rb_entry(node, struct bfq_entity, rb_node);
++}
++
++static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd);
++
++static bool bfq_update_parent_budget(struct bfq_entity *next_in_service);
++
++/**
++ * bfq_update_next_in_service - update sd->next_in_service
++ * @sd: sched_data for which to perform the update.
++ * @new_entity: if not NULL, pointer to the entity whose activation,
++ *		requeueing or repositionig triggered the invocation of
++ *		this function.
++ *
++ * This function is called to update sd->next_in_service, which, in
++ * its turn, may change as a consequence of the insertion or
++ * extraction of an entity into/from one of the active trees of
++ * sd. These insertions/extractions occur as a consequence of
++ * activations/deactivations of entities, with some activations being
++ * 'true' activations, and other activations being requeueings (i.e.,
++ * implementing the second, requeueing phase of the mechanism used to
++ * reposition an entity in its active tree; see comments on
++ * __bfq_activate_entity and __bfq_requeue_entity for details). In
++ * both the last two activation sub-cases, new_entity points to the
++ * just activated or requeued entity.
++ *
++ * Returns true if sd->next_in_service changes in such a way that
++ * entity->parent may become the next_in_service for its parent
++ * entity.
+  */
++static bool bfq_update_next_in_service(struct bfq_sched_data *sd,
++				       struct bfq_entity *new_entity)
++{
++	struct bfq_entity *next_in_service = sd->next_in_service;
 +	struct bfq_queue *bfqq;
- 
- 	if (sd->in_service_entity)
- 		/* will update/requeue at the end of service */
-@@ -65,14 +66,29 @@ static int bfq_update_next_in_service(struct bfq_sched_data *sd)
- 
- 	if (next_in_service)
- 		bfq_update_budget(next_in_service);
-+	else
-+		goto exit;
++	bool parent_sched_may_change = false;
++
++	/*
++	 * If this update is triggered by the activation, requeueing
++	 * or repositiong of an entity that does not coincide with
++	 * sd->next_in_service, then a full lookup in the active tree
++	 * can be avoided. In fact, it is enough to check whether the
++	 * just-modified entity has a higher priority than
++	 * sd->next_in_service, or, even if it has the same priority
++	 * as sd->next_in_service, is eligible and has a lower virtual
++	 * finish time than sd->next_in_service. If this compound
++	 * condition holds, then the new entity becomes the new
++	 * next_in_service. Otherwise no change is needed.
++	 */
++	if (new_entity && new_entity != sd->next_in_service) {
++		/*
++		 * Flag used to decide whether to replace
++		 * sd->next_in_service with new_entity. Tentatively
++		 * set to true, and left as true if
++		 * sd->next_in_service is NULL.
++		 */
++		bool replace_next = true;
++
++		/*
++		 * If there is already a next_in_service candidate
++		 * entity, then compare class priorities or timestamps
++		 * to decide whether to replace sd->service_tree with
++		 * new_entity.
++		 */
++		if (next_in_service) {
++			unsigned int new_entity_class_idx =
++				bfq_class_idx(new_entity);
++			struct bfq_service_tree *st =
++				sd->service_tree + new_entity_class_idx;
++
++			/*
++			 * For efficiency, evaluate the most likely
++			 * sub-condition first.
++			 */
++			replace_next =
++				(new_entity_class_idx ==
++				 bfq_class_idx(next_in_service)
++				 &&
++				 !bfq_gt(new_entity->start, st->vtime)
++				 &&
++				 bfq_gt(next_in_service->finish,
++					new_entity->finish))
++				||
++				new_entity_class_idx <
++				bfq_class_idx(next_in_service);
++		}
++
++		if (replace_next)
++			next_in_service = new_entity;
++	} else /* invoked because of a deactivation: lookup needed */
++		next_in_service = bfq_lookup_next_entity(sd);
++
++	if (next_in_service) {
++		parent_sched_may_change = !sd->next_in_service ||
++			bfq_update_parent_budget(next_in_service);
++	}
++
++	sd->next_in_service = next_in_service;
++
++	if (!next_in_service)
++		return parent_sched_may_change;
  
 +	bfqq = bfq_entity_to_bfqq(next_in_service);
 +	if (bfqq)
 +		bfq_log_bfqq(bfqq->bfqd, bfqq,
 +			     "update_next_in_service: chosen this queue");
+ #ifdef CONFIG_BFQ_GROUP_IOSCHED
+-#define for_each_entity(entity)	\
 +	else {
 +		struct bfq_group *bfqg =
 +			container_of(next_in_service,
@@ -4731,24 +6474,165 @@ index a64fec1..7d73b9d 100644
 +		bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg,
 +			     "update_next_in_service: chosen this entity");
 +	}
-+exit:
- 	return 1;
++#endif
++	return parent_sched_may_change;
++}
++
++#ifdef CONFIG_BFQ_GROUP_IOSCHED
++/* both next loops stop at one of the child entities of the root group */
++#define for_each_entity(entity)				\
+ 	for (; entity ; entity = entity->parent)
+ 
+ #define for_each_entity_safe(entity, parent) \
+ 	for (; entity && ({ parent = entity->parent; 1; }); entity = parent)
+ 
+-
+-static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd,
+-						 int extract,
+-						 struct bfq_data *bfqd);
+-
+-static struct bfq_group *bfqq_group(struct bfq_queue *bfqq);
+-
+-static void bfq_update_budget(struct bfq_entity *next_in_service)
++/*
++ * Returns true if this budget changes may let next_in_service->parent
++ * become the next_in_service entity for its parent entity.
++ */
++static bool bfq_update_parent_budget(struct bfq_entity *next_in_service)
+ {
+ 	struct bfq_entity *bfqg_entity;
+ 	struct bfq_group *bfqg;
+ 	struct bfq_sched_data *group_sd;
++	bool ret = false;
+ 
+ 	BUG_ON(!next_in_service);
+ 
+@@ -41,60 +179,68 @@ static void bfq_update_budget(struct bfq_entity *next_in_service)
+ 	 * as it must never become an in-service entity.
+ 	 */
+ 	bfqg_entity = bfqg->my_entity;
+-	if (bfqg_entity)
++	if (bfqg_entity) {
++		if (bfqg_entity->budget > next_in_service->budget)
++			ret = true;
+ 		bfqg_entity->budget = next_in_service->budget;
++	}
++
++	return ret;
  }
  
- static void bfq_check_next_in_service(struct bfq_sched_data *sd,
- 				      struct bfq_entity *entity)
+-static int bfq_update_next_in_service(struct bfq_sched_data *sd)
++/*
++ * This function tells whether entity stops being a candidate for next
++ * service, according to the following logic.
++ *
++ * This function is invoked for an entity that is about to be set in
++ * service. If such an entity is a queue, then the entity is no longer
++ * a candidate for next service (i.e, a candidate entity to serve
++ * after the in-service entity is expired). The function then returns
++ * true.
++ *
++ * In contrast, the entity could stil be a candidate for next service
++ * if it is not a queue, and has more than one child. In fact, even if
++ * one of its children is about to be set in service, other children
++ * may still be the next to serve. As a consequence, a non-queue
++ * entity is not a candidate for next-service only if it has only one
++ * child. And only if this condition holds, then the function returns
++ * true for a non-queue entity.
++ */
++static bool bfq_no_longer_next_in_service(struct bfq_entity *entity)
  {
--	BUG_ON(sd->next_in_service != entity);
-+	WARN_ON(sd->next_in_service != entity);
+-	struct bfq_entity *next_in_service;
++	struct bfq_group *bfqg;
+ 
+-	if (sd->in_service_entity)
+-		/* will update/requeue at the end of service */
+-		return 0;
++	if (bfq_entity_to_bfqq(entity))
++		return true;
+ 
+-	/*
+-	 * NOTE: this can be improved in many ways, such as returning
+-	 * 1 (and thus propagating upwards the update) only when the
+-	 * budget changes, or caching the bfqq that will be scheduled
+-	 * next from this subtree.  By now we worry more about
+-	 * correctness than about performance...
+-	 */
+-	next_in_service = bfq_lookup_next_entity(sd, 0, NULL);
+-	sd->next_in_service = next_in_service;
++	bfqg = container_of(entity, struct bfq_group, entity);
+ 
+-	if (next_in_service)
+-		bfq_update_budget(next_in_service);
++	BUG_ON(bfqg == ((struct bfq_data *)(bfqg->bfqd))->root_group);
++	BUG_ON(bfqg->active_entities == 0);
++	if (bfqg->active_entities == 1)
++		return true;
+ 
+-	return 1;
++	return false;
  }
- #else
+ 
+-static void bfq_check_next_in_service(struct bfq_sched_data *sd,
+-				      struct bfq_entity *entity)
+-{
+-	BUG_ON(sd->next_in_service != entity);
+-}
+-#else
++#else /* CONFIG_BFQ_GROUP_IOSCHED */
  #define for_each_entity(entity)	\
-@@ -151,20 +167,35 @@ static u64 bfq_delta(unsigned long service, unsigned long weight)
+ 	for (; entity ; entity = NULL)
+ 
+ #define for_each_entity_safe(entity, parent) \
+ 	for (parent = NULL; entity ; entity = parent)
+ 
+-static int bfq_update_next_in_service(struct bfq_sched_data *sd)
++static bool bfq_update_parent_budget(struct bfq_entity *next_in_service)
+ {
+-	return 0;
++	return false;
+ }
+ 
+-static void bfq_check_next_in_service(struct bfq_sched_data *sd,
+-				      struct bfq_entity *entity)
++static bool bfq_no_longer_next_in_service(struct bfq_entity *entity)
+ {
++	return true;
+ }
+ 
+-static void bfq_update_budget(struct bfq_entity *next_in_service)
+-{
+-}
+-#endif
++#endif /* CONFIG_BFQ_GROUP_IOSCHED */
+ 
+ /*
+  * Shift for timestamp calculations.  This actually limits the maximum
+@@ -105,18 +251,6 @@ static void bfq_update_budget(struct bfq_entity *next_in_service)
+  */
+ #define WFQ_SERVICE_SHIFT	22
+ 
+-/**
+- * bfq_gt - compare two timestamps.
+- * @a: first ts.
+- * @b: second ts.
+- *
+- * Return @a > @b, dealing with wrapping correctly.
+- */
+-static int bfq_gt(u64 a, u64 b)
+-{
+-	return (s64)(a - b) > 0;
+-}
+-
+ static struct bfq_queue *bfq_entity_to_bfqq(struct bfq_entity *entity)
+ {
+ 	struct bfq_queue *bfqq = NULL;
+@@ -151,20 +285,36 @@ static u64 bfq_delta(unsigned long service, unsigned long weight)
  static void bfq_calc_finish(struct bfq_entity *entity, unsigned long service)
  {
  	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
--
-+	unsigned long long start, finish, delta ;
++	unsigned long long start, finish, delta;
+ 
  	BUG_ON(entity->weight == 0);
  
  	entity->finish = entity->start +
@@ -4782,7 +6666,34 @@ index a64fec1..7d73b9d 100644
  	}
  }
  
-@@ -386,8 +417,6 @@ static void bfq_active_insert(struct bfq_service_tree *st,
+@@ -293,10 +443,26 @@ static void bfq_update_min(struct bfq_entity *entity, struct rb_node *node)
+ static void bfq_update_active_node(struct rb_node *node)
+ {
+ 	struct bfq_entity *entity = rb_entry(node, struct bfq_entity, rb_node);
++	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
+ 
+ 	entity->min_start = entity->start;
+ 	bfq_update_min(entity, node->rb_right);
+ 	bfq_update_min(entity, node->rb_left);
++
++	if (bfqq) {
++		bfq_log_bfqq(bfqq->bfqd, bfqq,
++			     "update_active_node: new min_start %llu",
++			     ((entity->min_start>>10)*1000)>>12);
++#ifdef CONFIG_BFQ_GROUP_IOSCHED
++	} else {
++		struct bfq_group *bfqg =
++			container_of(entity, struct bfq_group, entity);
++
++		bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg,
++			     "update_active_node: new min_start %llu",
++			     ((entity->min_start>>10)*1000)>>12);
++#endif
++	}
+ }
+ 
+ /**
+@@ -386,8 +552,6 @@ static void bfq_active_insert(struct bfq_service_tree *st,
  		BUG_ON(!bfqg);
  		BUG_ON(!bfqd);
  		bfqg->active_entities++;
@@ -4791,16 +6702,16 @@ index a64fec1..7d73b9d 100644
  	}
  #endif
  }
-@@ -399,7 +428,7 @@ static void bfq_active_insert(struct bfq_service_tree *st,
+@@ -399,7 +563,7 @@ static void bfq_active_insert(struct bfq_service_tree *st,
  static unsigned short bfq_ioprio_to_weight(int ioprio)
  {
  	BUG_ON(ioprio < 0 || ioprio >= IOPRIO_BE_NR);
 -	return IOPRIO_BE_NR * BFQ_WEIGHT_CONVERSION_COEFF - ioprio;
-+	return (IOPRIO_BE_NR - ioprio) * BFQ_WEIGHT_CONVERSION_COEFF ;
++	return (IOPRIO_BE_NR - ioprio) * BFQ_WEIGHT_CONVERSION_COEFF;
  }
  
  /**
-@@ -422,9 +451,9 @@ static void bfq_get_entity(struct bfq_entity *entity)
+@@ -422,9 +586,9 @@ static void bfq_get_entity(struct bfq_entity *entity)
  	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
  
  	if (bfqq) {
@@ -4812,7 +6723,7 @@ index a64fec1..7d73b9d 100644
  	}
  }
  
-@@ -499,10 +528,6 @@ static void bfq_active_extract(struct bfq_service_tree *st,
+@@ -499,10 +663,6 @@ static void bfq_active_extract(struct bfq_service_tree *st,
  		BUG_ON(!bfqd);
  		BUG_ON(!bfqg->active_entities);
  		bfqg->active_entities--;
@@ -4823,7 +6734,13 @@ index a64fec1..7d73b9d 100644
  	}
  #endif
  }
-@@ -552,7 +577,7 @@ static void bfq_forget_entity(struct bfq_service_tree *st,
+@@ -547,12 +707,12 @@ static void bfq_forget_entity(struct bfq_service_tree *st,
+ 
+ 	BUG_ON(!entity->on_st);
+ 
+-	entity->on_st = 0;
++	entity->on_st = false;
+ 	st->wsum -= entity->weight;
  	if (bfqq) {
  		sd = entity->sched_data;
  		bfq_log_bfqq(bfqq->bfqd, bfqq, "forget_entity: %p %d",
@@ -4832,7 +6749,7 @@ index a64fec1..7d73b9d 100644
  		bfq_put_queue(bfqq);
  	}
  }
-@@ -602,7 +627,7 @@ __bfq_entity_update_weight_prio(struct bfq_service_tree *old_st,
+@@ -602,7 +762,7 @@ __bfq_entity_update_weight_prio(struct bfq_service_tree *old_st,
  
  	if (entity->prio_changed) {
  		struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
@@ -4841,13 +6758,9 @@ index a64fec1..7d73b9d 100644
  		struct bfq_data *bfqd = NULL;
  		struct rb_root *root;
  #ifdef CONFIG_BFQ_GROUP_IOSCHED
-@@ -628,12 +653,14 @@ __bfq_entity_update_weight_prio(struct bfq_service_tree *old_st,
- 		if (entity->new_weight != entity->orig_weight) {
- 			if (entity->new_weight < BFQ_MIN_WEIGHT ||
+@@ -630,7 +790,10 @@ __bfq_entity_update_weight_prio(struct bfq_service_tree *old_st,
  			    entity->new_weight > BFQ_MAX_WEIGHT) {
--				printk(KERN_CRIT "update_weight_prio: "
--						 "new_weight %d\n",
-+				pr_crit("update_weight_prio: new_weight %d\n",
+ 				pr_crit("update_weight_prio: new_weight %d\n",
  					entity->new_weight);
 -				BUG();
 +				if (entity->new_weight < BFQ_MIN_WEIGHT)
@@ -4855,12 +6768,9 @@ index a64fec1..7d73b9d 100644
 +				else
 +					entity->new_weight = BFQ_MAX_WEIGHT;
  			}
--			entity->orig_weight = entity->new_weight;
-+		       	entity->orig_weight = entity->new_weight;
+ 			entity->orig_weight = entity->new_weight;
  			if (bfqq)
- 				bfqq->ioprio =
- 				  bfq_weight_to_ioprio(entity->orig_weight);
-@@ -662,6 +689,13 @@ __bfq_entity_update_weight_prio(struct bfq_service_tree *old_st,
+@@ -661,6 +824,13 @@ __bfq_entity_update_weight_prio(struct bfq_service_tree *old_st,
  		 * associated with its new weight.
  		 */
  		if (prev_weight != new_weight) {
@@ -4874,7 +6784,7 @@ index a64fec1..7d73b9d 100644
  			root = bfqq ? &bfqd->queue_weights_tree :
  				      &bfqd->group_weights_tree;
  			bfq_weights_tree_remove(bfqd, entity, root);
-@@ -708,7 +742,7 @@ static void bfq_bfqq_served(struct bfq_queue *bfqq, int served)
+@@ -707,7 +877,7 @@ static void bfq_bfqq_served(struct bfq_queue *bfqq, int served)
  		st = bfq_entity_service_tree(entity);
  
  		entity->service += served;
@@ -4883,7 +6793,7 @@ index a64fec1..7d73b9d 100644
  		BUG_ON(st->wsum == 0);
  
  		st->vtime += bfq_delta(served, st->wsum);
-@@ -717,31 +751,69 @@ static void bfq_bfqq_served(struct bfq_queue *bfqq, int served)
+@@ -716,234 +886,574 @@ static void bfq_bfqq_served(struct bfq_queue *bfqq, int served)
  #ifdef CONFIG_BFQ_GROUP_IOSCHED
  	bfqg_stats_set_start_empty_time(bfqq_group(bfqq));
  #endif
@@ -4937,107 +6847,34 @@ index a64fec1..7d73b9d 100644
 +	if (time_ms > 0 && time_ms < timeout_ms)
 +		tot_serv_to_charge =
 +			(bfqd->bfq_max_budget * time_ms) / timeout_ms;
-+
+ 
+-	bfq_log_bfqq(bfqq->bfqd, bfqq, "charge_full_budget");
 +	if (tot_serv_to_charge < entity->service)
 +		tot_serv_to_charge = entity->service;
  
--	bfq_log_bfqq(bfqq->bfqd, bfqq, "charge_full_budget");
+-	bfq_bfqq_served(bfqq, entity->budget - entity->service);
 +	bfq_log_bfqq(bfqq->bfqd, bfqq,
 +		     "charge_time: %lu/%u ms, %d/%d/%d sectors",
 +		     time_ms, timeout_ms, entity->service,
 +		     tot_serv_to_charge, entity->budget);
- 
--	bfq_bfqq_served(bfqq, entity->budget - entity->service);
++
 +	/* Increase budget to avoid inconsistencies */
 +	if (tot_serv_to_charge > entity->budget)
 +		entity->budget = tot_serv_to_charge;
 +
 +	bfq_bfqq_served(bfqq,
 +			max_t(int, 0, tot_serv_to_charge - entity->service));
- }
- 
- /**
-  * __bfq_activate_entity - activate an entity.
-  * @entity: the entity being activated.
-+ * @non_blocking_wait_rq: true if this entity was waiting for a request
-  *
-  * Called whenever an entity is activated, i.e., it is not active and one
-  * of its children receives a new request, or has to be reactivated due to
-@@ -749,11 +821,16 @@ static void bfq_bfqq_charge_full_budget(struct bfq_queue *bfqq)
-  * service received if @entity is active) of the queue to calculate its
-  * timestamps.
-  */
--static void __bfq_activate_entity(struct bfq_entity *entity)
-+static void __bfq_activate_entity(struct bfq_entity *entity,
-+				  bool non_blocking_wait_rq)
- {
- 	struct bfq_sched_data *sd = entity->sched_data;
- 	struct bfq_service_tree *st = bfq_entity_service_tree(entity);
++}
++
++static void bfq_update_fin_time_enqueue(struct bfq_entity *entity,
++					struct bfq_service_tree *st,
++					bool backshifted)
++{
 +	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
-+	bool backshifted = false;
- 
-+	BUG_ON(!sd);
-+	BUG_ON(!st);
- 	if (entity == sd->in_service_entity) {
- 		BUG_ON(entity->tree);
- 		/*
-@@ -771,45 +848,133 @@ static void __bfq_activate_entity(struct bfq_entity *entity)
- 		 * old start time.
- 		 */
- 		bfq_active_extract(st, entity);
--	} else if (entity->tree == &st->idle) {
--		/*
--		 * Must be on the idle tree, bfq_idle_extract() will
--		 * check for that.
--		 */
--		bfq_idle_extract(st, entity);
--		entity->start = bfq_gt(st->vtime, entity->finish) ?
--				       st->vtime : entity->finish;
- 	} else {
--		/*
--		 * The finish time of the entity may be invalid, and
--		 * it is in the past for sure, otherwise the queue
--		 * would have been on the idle tree.
--		 */
--		entity->start = st->vtime;
--		st->wsum += entity->weight;
--		bfq_get_entity(entity);
-+		unsigned long long min_vstart;
- 
--		BUG_ON(entity->on_st);
--		entity->on_st = 1;
-+		/* See comments on bfq_fqq_update_budg_for_activation */
-+		if (non_blocking_wait_rq && bfq_gt(st->vtime, entity->finish)) {
-+			backshifted = true;
-+			min_vstart = entity->finish;
-+		} else
-+			min_vstart = st->vtime;
++	struct bfq_sched_data *sd = entity->sched_data;
 +
-+		if (entity->tree == &st->idle) {
-+			/*
-+			 * Must be on the idle tree, bfq_idle_extract() will
-+			 * check for that.
-+			 */
-+			bfq_idle_extract(st, entity);
-+			entity->start = bfq_gt(min_vstart, entity->finish) ?
-+				min_vstart : entity->finish;
-+		} else {
-+			/*
-+			 * The finish time of the entity may be invalid, and
-+			 * it is in the past for sure, otherwise the queue
-+			 * would have been on the idle tree.
-+			 */
-+			entity->start = min_vstart;
-+			st->wsum += entity->weight;
-+			bfq_get_entity(entity);
-+
-+			BUG_ON(entity->on_st);
-+			entity->on_st = 1;
-+		}
- 	}
- 
- 	st = __bfq_entity_update_weight_prio(st, entity);
- 	bfq_calc_finish(entity, entity->budget);
++	st = __bfq_entity_update_weight_prio(st, entity);
++	bfq_calc_finish(entity, entity->budget);
 +
 +	/*
 +	 * If some queues enjoy backshifting for a while, then their
@@ -5097,7 +6934,7 @@ index a64fec1..7d73b9d 100644
 +		}
 +	}
 +
- 	bfq_active_insert(st, entity);
++	bfq_active_insert(st, entity);
 +
 +	if (bfqq) {
 +		bfq_log_bfqq(bfqq->bfqd, bfqq,
@@ -5113,70 +6950,466 @@ index a64fec1..7d73b9d 100644
 +			     entity->start <= st->vtime ? "" : "non ", st);
 +#endif
 +	}
++	BUG_ON(RB_EMPTY_ROOT(&st->active));
++	BUG_ON(&st->active != &sd->service_tree->active &&
++	       &st->active != &(sd->service_tree+1)->active &&
++	       &st->active != &(sd->service_tree+2)->active);
  }
  
  /**
-  * bfq_activate_entity - activate an entity and its ancestors if necessary.
-  * @entity: the entity to activate.
-+ * @non_blocking_wait_rq: true if this entity was waiting for a request
+- * __bfq_activate_entity - activate an entity.
++ * __bfq_activate_entity - handle activation of entity.
+  * @entity: the entity being activated.
++ * @non_blocking_wait_rq: true if entity was waiting for a request
++ *
++ * Called for a 'true' activation, i.e., if entity is not active and
++ * one of its children receives a new request.
   *
-  * Activate @entity and all the entities on the path from it to the root.
+- * Called whenever an entity is activated, i.e., it is not active and one
+- * of its children receives a new request, or has to be reactivated due to
+- * budget exhaustion.  It uses the current budget of the entity (and the
+- * service received if @entity is active) of the queue to calculate its
+- * timestamps.
++ * Basically, this function updates the timestamps of entity and
++ * inserts entity into its active tree, ater possible extracting it
++ * from its idle tree.
+  */
+-static void __bfq_activate_entity(struct bfq_entity *entity)
++static void __bfq_activate_entity(struct bfq_entity *entity,
++				  bool non_blocking_wait_rq)
+ {
+ 	struct bfq_sched_data *sd = entity->sched_data;
+ 	struct bfq_service_tree *st = bfq_entity_service_tree(entity);
++	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
++	bool backshifted = false;
++	unsigned long long min_vstart;
+ 
+-	if (entity == sd->in_service_entity) {
+-		BUG_ON(entity->tree);
+-		/*
+-		 * If we are requeueing the current entity we have
+-		 * to take care of not charging to it service it has
+-		 * not received.
+-		 */
+-		bfq_calc_finish(entity, entity->service);
+-		entity->start = entity->finish;
+-		sd->in_service_entity = NULL;
+-	} else if (entity->tree == &st->active) {
+-		/*
+-		 * Requeueing an entity due to a change of some
+-		 * next_in_service entity below it.  We reuse the
+-		 * old start time.
+-		 */
+-		bfq_active_extract(st, entity);
+-	} else if (entity->tree == &st->idle) {
++	BUG_ON(!sd);
++	BUG_ON(!st);
++
++	/* See comments on bfq_fqq_update_budg_for_activation */
++	if (non_blocking_wait_rq && bfq_gt(st->vtime, entity->finish)) {
++		backshifted = true;
++		min_vstart = entity->finish;
++	} else
++		min_vstart = st->vtime;
++
++	if (entity->tree == &st->idle) {
+ 		/*
+ 		 * Must be on the idle tree, bfq_idle_extract() will
+ 		 * check for that.
+ 		 */
+ 		bfq_idle_extract(st, entity);
+-		entity->start = bfq_gt(st->vtime, entity->finish) ?
+-				       st->vtime : entity->finish;
++		entity->start = bfq_gt(min_vstart, entity->finish) ?
++			min_vstart : entity->finish;
+ 	} else {
+ 		/*
+ 		 * The finish time of the entity may be invalid, and
+ 		 * it is in the past for sure, otherwise the queue
+ 		 * would have been on the idle tree.
+ 		 */
+-		entity->start = st->vtime;
++		entity->start = min_vstart;
+ 		st->wsum += entity->weight;
+ 		bfq_get_entity(entity);
+ 
+-		BUG_ON(entity->on_st);
+-		entity->on_st = 1;
++		BUG_ON(entity->on_st && bfqq);
++
++#ifdef CONFIG_BFQ_GROUP_IOSCHED
++		if (entity->on_st && !bfqq) {
++			struct bfq_group *bfqg =
++				container_of(entity, struct bfq_group,
++					     entity);
++
++			bfq_log_bfqg((struct bfq_data *)bfqg->bfqd,
++				     bfqg,
++				     "activate bug, class %d in_service %p",
++				     bfq_class_idx(entity), sd->in_service_entity);
++		}
++#endif
++		BUG_ON(entity->on_st && !bfqq);
++		entity->on_st = true;
+ 	}
+ 
+-	st = __bfq_entity_update_weight_prio(st, entity);
+-	bfq_calc_finish(entity, entity->budget);
+-	bfq_active_insert(st, entity);
++	bfq_update_fin_time_enqueue(entity, st, backshifted);
+ }
+ 
+ /**
+- * bfq_activate_entity - activate an entity and its ancestors if necessary.
+- * @entity: the entity to activate.
++ * __bfq_requeue_entity - handle requeueing or repositioning of an entity.
++ * @entity: the entity being requeued or repositioned.
++ *
++ * Requeueing is needed if this entity stops being served, which
++ * happens if a leaf descendant entity has expired. On the other hand,
++ * repositioning is needed if the next_inservice_entity for the child
++ * entity has changed. See the comments inside the function for
++ * details.
+  *
+- * Activate @entity and all the entities on the path from it to the root.
++ * Basically, this function: 1) removes entity from its active tree if
++ * present there, 2) updates the timestamps of entity and 3) inserts
++ * entity back into its active tree (in the new, right position for
++ * the new values of the timestamps).
   */
 -static void bfq_activate_entity(struct bfq_entity *entity)
-+static void bfq_activate_entity(struct bfq_entity *entity,
-+				bool non_blocking_wait_rq)
++static void __bfq_requeue_entity(struct bfq_entity *entity)
++{
++	struct bfq_sched_data *sd = entity->sched_data;
++	struct bfq_service_tree *st = bfq_entity_service_tree(entity);
++
++	BUG_ON(!sd);
++	BUG_ON(!st);
++
++	BUG_ON(entity != sd->in_service_entity &&
++	       entity->tree != &st->active);
++
++	if (entity == sd->in_service_entity) {
++		/*
++		 * We are requeueing the current in-service entity,
++		 * which may have to be done for one of the following
++		 * reasons:
++		 * - entity represents the in-service queue, and the
++		 *   in-service queue is being requeued after an
++		 *   expiration;
++		 * - entity represents a group, and its budget has
++		 *   changed because one of its child entities has
++		 *   just been either activated or requeued for some
++		 *   reason; the timestamps of the entity need then to
++		 *   be updated, and the entity needs to be enqueued
++		 *   or repositioned accordingly.
++		 *
++		 * In particular, before requeueing, the start time of
++		 * the entity must be moved forward to account for the
++		 * service that the entity has received while in
++		 * service. This is done by the next instructions. The
++		 * finish time will then be updated according to this
++		 * new value of the start time, and to the budget of
++		 * the entity.
++		 */
++		bfq_calc_finish(entity, entity->service);
++		entity->start = entity->finish;
++		BUG_ON(entity->tree && entity->tree != &st->active);
++		/*
++		 * In addition, if the entity had more than one child
++		 * when set in service, then was not extracted from
++		 * the active tree. This implies that the position of
++		 * the entity in the active tree may need to be
++		 * changed now, because we have just updated the start
++		 * time of the entity, and we will update its finish
++		 * time in a moment (the requeueing is then, more
++		 * precisely, a repositioning in this case). To
++		 * implement this repositioning, we: 1) dequeue the
++		 * entity here, 2) update the finish time and
++		 * requeue the entity according to the new
++		 * timestamps below.
++		 */
++		if (entity->tree)
++			bfq_active_extract(st, entity);
++	} else { /* The entity is already active, and not in service */
++		/*
++		 * In this case, this function gets called only if the
++		 * next_in_service entity below this entity has
++		 * changed, and this change has caused the budget of
++		 * this entity to change, which, finally implies that
++		 * the finish time of this entity must be
++		 * updated. Such an update may cause the scheduling,
++		 * i.e., the position in the active tree, of this
++		 * entity to change. We handle this change by: 1)
++		 * dequeueing the entity here, 2) updating the finish
++		 * time and requeueing the entity according to the new
++		 * timestamps below. This is the same approach as the
++		 * non-extracted-entity sub-case above.
++		 */
++		bfq_active_extract(st, entity);
++	}
++
++	bfq_update_fin_time_enqueue(entity, st, false);
++}
++
++static void __bfq_activate_requeue_entity(struct bfq_entity *entity,
++					  struct bfq_sched_data *sd,
++					  bool non_blocking_wait_rq)
++{
++	struct bfq_service_tree *st = bfq_entity_service_tree(entity);
++
++	if (sd->in_service_entity == entity || entity->tree == &st->active)
++		 /*
++		  * in service or already queued on the active tree,
++		  * requeue or reposition
++		  */
++		__bfq_requeue_entity(entity);
++	else
++		/*
++		 * Not in service and not queued on its active tree:
++		 * the activity is idle and this is a true activation.
++		 */
++		__bfq_activate_entity(entity, non_blocking_wait_rq);
++}
++
++
++/**
++ * bfq_activate_entity - activate or requeue an entity representing a bfq_queue,
++ *			 and activate, requeue or reposition all ancestors
++ *			 for which such an update becomes necessary.
++ * @entity: the entity to activate.
++ * @non_blocking_wait_rq: true if this entity was waiting for a request
++ * @requeue: true if this is a requeue, which implies that bfqq is
++ *	     being expired; thus ALL its ancestors stop being served and must
++ *	     therefore be requeued
++ */
++static void bfq_activate_requeue_entity(struct bfq_entity *entity,
++					bool non_blocking_wait_rq,
++					bool requeue)
  {
  	struct bfq_sched_data *sd;
  
  	for_each_entity(entity) {
 -		__bfq_activate_entity(entity);
+-
 +		BUG_ON(!entity);
-+		__bfq_activate_entity(entity, non_blocking_wait_rq);
- 
  		sd = entity->sched_data;
- 		if (!bfq_update_next_in_service(sd))
-@@ -890,23 +1055,24 @@ static void bfq_deactivate_entity(struct bfq_entity *entity, int requeue)
+-		if (!bfq_update_next_in_service(sd))
+-			/*
+-			 * No need to propagate the activation to the
+-			 * upper entities, as they will be updated when
+-			 * the in-service entity is rescheduled.
+-			 */
++		__bfq_activate_requeue_entity(entity, sd, non_blocking_wait_rq);
++
++		BUG_ON(RB_EMPTY_ROOT(&sd->service_tree->active) &&
++		       RB_EMPTY_ROOT(&(sd->service_tree+1)->active) &&
++		       RB_EMPTY_ROOT(&(sd->service_tree+2)->active));
++
++		if (!bfq_update_next_in_service(sd, entity) && !requeue) {
++			BUG_ON(!sd->next_in_service);
+ 			break;
++		}
++		BUG_ON(!sd->next_in_service);
+ 	}
+ }
  
- 		if (!__bfq_deactivate_entity(entity, requeue))
+ /**
+  * __bfq_deactivate_entity - deactivate an entity from its service tree.
+  * @entity: the entity to deactivate.
+- * @requeue: if false, the entity will not be put into the idle tree.
+- *
+- * Deactivate an entity, independently from its previous state.  If the
+- * entity was not on a service tree just return, otherwise if it is on
+- * any scheduler tree, extract it from that tree, and if necessary
+- * and if the caller did not specify @requeue, put it on the idle tree.
++ * @ins_into_idle_tree: if false, the entity will not be put into the
++ *			idle tree.
+  *
+- * Return %1 if the caller should update the entity hierarchy, i.e.,
+- * if the entity was in service or if it was the next_in_service for
+- * its sched_data; return %0 otherwise.
++ * Deactivates an entity, independently from its previous state.  Must
++ * be invoked only if entity is on a service tree. Extracts the entity
++ * from that tree, and if necessary and allowed, puts it on the idle
++ * tree.
+  */
+-static int __bfq_deactivate_entity(struct bfq_entity *entity, int requeue)
++static bool __bfq_deactivate_entity(struct bfq_entity *entity,
++				    bool ins_into_idle_tree)
+ {
+ 	struct bfq_sched_data *sd = entity->sched_data;
+-	struct bfq_service_tree *st;
+-	int was_in_service;
+-	int ret = 0;
+-
+-	if (sd == NULL || !entity->on_st) /* never activated, or inactive */
+-		return 0;
++	struct bfq_service_tree *st = bfq_entity_service_tree(entity);
++	bool was_in_service = entity == sd->in_service_entity;
+ 
+-	st = bfq_entity_service_tree(entity);
+-	was_in_service = entity == sd->in_service_entity;
++	if (!entity->on_st) { /* entity never activated, or already inactive */
++		BUG_ON(entity == entity->sched_data->in_service_entity);
++		return false;
++	}
+ 
+-	BUG_ON(was_in_service && entity->tree);
++	BUG_ON(was_in_service && entity->tree && entity->tree != &st->active);
+ 
+-	if (was_in_service) {
++	if (was_in_service)
+ 		bfq_calc_finish(entity, entity->service);
+-		sd->in_service_entity = NULL;
+-	} else if (entity->tree == &st->active)
++
++	if (entity->tree == &st->active)
+ 		bfq_active_extract(st, entity);
+-	else if (entity->tree == &st->idle)
++	else if (!was_in_service && entity->tree == &st->idle)
+ 		bfq_idle_extract(st, entity);
+ 	else if (entity->tree)
+ 		BUG();
+ 
+-	if (was_in_service || sd->next_in_service == entity)
+-		ret = bfq_update_next_in_service(sd);
+-
+-	if (!requeue || !bfq_gt(entity->finish, st->vtime))
++	if (!ins_into_idle_tree || !bfq_gt(entity->finish, st->vtime))
+ 		bfq_forget_entity(st, entity);
+ 	else
+ 		bfq_idle_insert(st, entity);
+ 
+-	BUG_ON(sd->in_service_entity == entity);
+-	BUG_ON(sd->next_in_service == entity);
+-
+-	return ret;
++	return true;
+ }
+ 
+ /**
+- * bfq_deactivate_entity - deactivate an entity.
++ * bfq_deactivate_entity - deactivate an entity representing a bfq_queue.
+  * @entity: the entity to deactivate.
+- * @requeue: true if the entity can be put on the idle tree
++ * @ins_into_idle_tree: true if the entity can be put on the idle tree
+  */
+-static void bfq_deactivate_entity(struct bfq_entity *entity, int requeue)
++static void bfq_deactivate_entity(struct bfq_entity *entity,
++				  bool ins_into_idle_tree,
++				  bool expiration)
+ {
+ 	struct bfq_sched_data *sd;
+-	struct bfq_entity *parent;
++	struct bfq_entity *parent = NULL;
+ 
+ 	for_each_entity_safe(entity, parent) {
+ 		sd = entity->sched_data;
+ 
+-		if (!__bfq_deactivate_entity(entity, requeue))
++		BUG_ON(sd == NULL); /*
++				     * It would mean that this is the
++				     * root group.
++				     */
++
++		BUG_ON(expiration && entity != sd->in_service_entity);
++
++		BUG_ON(entity != sd->in_service_entity &&
++		       entity->tree ==
++		       &bfq_entity_service_tree(entity)->active &&
++		       !sd->next_in_service);
++
++		if (!__bfq_deactivate_entity(entity, ins_into_idle_tree)) {
  			/*
 -			 * The parent entity is still backlogged, and
 -			 * we don't need to update it as it is still
 -			 * in service.
-+			 * next_in_service has not been changed, so
-+			 * no upwards update is needed
++			 * Entity is not any tree any more, so, this
++			 * deactivation is a no-op, and there is
++			 * nothing to change for upper-level entities
++			 * (in case of expiration, this can never
++			 * happen).
  			 */
- 			break;
+-			break;
++			BUG_ON(expiration); /*
++					     * entity cannot be already out of
++					     * any tree
++					     */
++			return;
++		}
  
- 		if (sd->next_in_service)
+-		if (sd->next_in_service)
++		if (sd->next_in_service == entity)
  			/*
 -			 * The parent entity is still backlogged and
 -			 * the budgets on the path towards the root
 -			 * need to be updated.
-+			 * The parent entity is still backlogged,
-+			 * because next_in_service is not NULL, and
-+			 * next_in_service has been updated (see
-+			 * comment on the body of the above if):
-+			 * upwards update of the schedule is needed.
++			 * entity was the next_in_service entity,
++			 * then, since entity has just been
++			 * deactivated, a new one must be found.
  			 */
- 			goto update;
+-			goto update;
++			bfq_update_next_in_service(sd, NULL);
++
++		if (sd->next_in_service) {
++			/*
++			 * The parent entity is still backlogged,
++			 * because next_in_service is not NULL. So, no
++			 * further upwards deactivation must be
++			 * performed.  Yet, next_in_service has
++			 * changed.  Then the schedule does need to be
++			 * updated upwards.
++			 */
++			BUG_ON(sd->next_in_service == entity);
++			break;
++		}
  
  		/*
 -		 * If we reach there the parent is no more backlogged and
 -		 * we want to propagate the dequeue upwards.
-+		 * If we get here, then the parent is no more backlogged and
-+		 * we want to propagate the deactivation upwards.
++		 * If we get here, then the parent is no more
++		 * backlogged and we need to propagate the
++		 * deactivation upwards. Thus let the loop go on.
  		 */
- 		requeue = 1;
- 	}
-@@ -916,9 +1082,23 @@ static void bfq_deactivate_entity(struct bfq_entity *entity, int requeue)
- update:
+-		requeue = 1;
+-	}
+ 
+-	return;
++		/*
++		 * Also let parent be queued into the idle tree on
++		 * deactivation, to preserve service guarantees, and
++		 * assuming that who invoked this function does not
++		 * need parent entities too to be removed completely.
++		 */
++		ins_into_idle_tree = true;
++	}
+ 
+-update:
++	/*
++	 * If the deactivation loop is fully executed, then there are
++	 * no more entities to touch and next loop is not executed at
++	 * all. Otherwise, requeue remaining entities if they are
++	 * about to stop receiving service, or reposition them if this
++	 * is not the case.
++	 */
  	entity = parent;
  	for_each_entity(entity) {
 -		__bfq_activate_entity(entity);
 +		struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
-+		__bfq_activate_entity(entity, false);
++
++		/*
++		 * Invoke __bfq_requeue_entity on entity, even if
++		 * already active, to requeue/reposition it in the
++		 * active tree (because sd->next_in_service has
++		 * changed)
++		 */
++		__bfq_requeue_entity(entity);
  
  		sd = entity->sched_data;
+-		if (!bfq_update_next_in_service(sd))
++		BUG_ON(expiration && sd->in_service_entity != entity);
++
 +		if (bfqq)
 +			bfq_log_bfqq(bfqq->bfqd, bfqq,
 +				     "invoking udpdate_next for this queue");
@@ -5190,33 +7423,195 @@ index a64fec1..7d73b9d 100644
 +				     "invoking udpdate_next for this entity");
 +		}
 +#endif
- 		if (!bfq_update_next_in_service(sd))
++		if (!bfq_update_next_in_service(sd, entity) &&
++		    !expiration)
++			/*
++			 * next_in_service unchanged or not causing
++			 * any change in entity->parent->sd, and no
++			 * requeueing needed for expiration: stop
++			 * here.
++			 */
  			break;
  	}
-@@ -997,10 +1177,11 @@ left:
-  * Update the virtual time in @st and return the first eligible entity
-  * it contains.
+ }
+ 
+ /**
+- * bfq_update_vtime - update vtime if necessary.
++ * bfq_calc_vtime_jump - compute the value to which the vtime should jump,
++ *                       if needed, to have at least one entity eligible.
+  * @st: the service tree to act upon.
+  *
+- * If necessary update the service tree vtime to have at least one
+- * eligible entity, skipping to its start time.  Assumes that the
+- * active tree of the device is not empty.
+- *
+- * NOTE: this hierarchical implementation updates vtimes quite often,
+- * we may end up with reactivated processes getting timestamps after a
+- * vtime skip done because we needed a ->first_active entity on some
+- * intermediate node.
++ * Assumes that st is not empty.
+  */
+-static void bfq_update_vtime(struct bfq_service_tree *st)
++static u64 bfq_calc_vtime_jump(struct bfq_service_tree *st)
+ {
+-	struct bfq_entity *entry;
+-	struct rb_node *node = st->active.rb_node;
++	struct bfq_entity *root_entity = bfq_root_active_entity(&st->active);
++
++	if (bfq_gt(root_entity->min_start, st->vtime)) {
++		struct bfq_queue *bfqq = bfq_entity_to_bfqq(root_entity);
+ 
+-	entry = rb_entry(node, struct bfq_entity, rb_node);
+-	if (bfq_gt(entry->min_start, st->vtime)) {
+-		st->vtime = entry->min_start;
++		if (bfqq)
++			bfq_log_bfqq(bfqq->bfqd, bfqq,
++				     "calc_vtime_jump: new value %llu",
++				     root_entity->min_start);
++#ifdef CONFIG_BFQ_GROUP_IOSCHED
++		else {
++			struct bfq_group *bfqg =
++				container_of(root_entity, struct bfq_group,
++					     entity);
++
++			bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg,
++				     "calc_vtime_jump: new value %llu",
++				     root_entity->min_start);
++		}
++#endif
++		return root_entity->min_start;
++	}
++	return st->vtime;
++}
++
++static void bfq_update_vtime(struct bfq_service_tree *st, u64 new_value)
++{
++	if (new_value > st->vtime) {
++		st->vtime = new_value;
+ 		bfq_forget_idle(st);
+ 	}
+ }
+@@ -952,6 +1462,7 @@ static void bfq_update_vtime(struct bfq_service_tree *st)
+  * bfq_first_active_entity - find the eligible entity with
+  *                           the smallest finish time
+  * @st: the service tree to select from.
++ * @vtime: the system virtual to use as a reference for eligibility
+  *
+  * This function searches the first schedulable entity, starting from the
+  * root of the tree and going on the left every time on this side there is
+@@ -959,7 +1470,8 @@ static void bfq_update_vtime(struct bfq_service_tree *st)
+  * the right is followed only if a) the left subtree contains no eligible
+  * entities and b) no eligible entity has been found yet.
+  */
+-static struct bfq_entity *bfq_first_active_entity(struct bfq_service_tree *st)
++static struct bfq_entity *bfq_first_active_entity(struct bfq_service_tree *st,
++						  u64 vtime)
+ {
+ 	struct bfq_entity *entry, *first = NULL;
+ 	struct rb_node *node = st->active.rb_node;
+@@ -967,15 +1479,15 @@ static struct bfq_entity *bfq_first_active_entity(struct bfq_service_tree *st)
+ 	while (node) {
+ 		entry = rb_entry(node, struct bfq_entity, rb_node);
+ left:
+-		if (!bfq_gt(entry->start, st->vtime))
++		if (!bfq_gt(entry->start, vtime))
+ 			first = entry;
+ 
+-		BUG_ON(bfq_gt(entry->min_start, st->vtime));
++		BUG_ON(bfq_gt(entry->min_start, vtime));
+ 
+ 		if (node->rb_left) {
+ 			entry = rb_entry(node->rb_left,
+ 					 struct bfq_entity, rb_node);
+-			if (!bfq_gt(entry->min_start, st->vtime)) {
++			if (!bfq_gt(entry->min_start, vtime)) {
+ 				node = node->rb_left;
+ 				goto left;
+ 			}
+@@ -993,31 +1505,84 @@ static struct bfq_entity *bfq_first_active_entity(struct bfq_service_tree *st)
+  * __bfq_lookup_next_entity - return the first eligible entity in @st.
+  * @st: the service tree.
+  *
+- * Update the virtual time in @st and return the first eligible entity
+- * it contains.
++ * If there is no in-service entity for the sched_data st belongs to,
++ * then return the entity that will be set in service if:
++ * 1) the parent entity this st belongs to is set in service;
++ * 2) no entity belonging to such parent entity undergoes a state change
++ * that would influence the timestamps of the entity (e.g., becomes idle,
++ * becomes backlogged, changes its budget, ...).
++ *
++ * In this first case, update the virtual time in @st too (see the
++ * comments on this update inside the function).
++ *
++ * In constrast, if there is an in-service entity, then return the
++ * entity that would be set in service if not only the above
++ * conditions, but also the next one held true: the currently
++ * in-service entity, on expiration,
++ * 1) gets a finish time equal to the current one, or
++ * 2) is not eligible any more, or
++ * 3) is idle.
   */
 -static struct bfq_entity *__bfq_lookup_next_entity(struct bfq_service_tree *st,
 -						   bool force)
 +static struct bfq_entity *
-+__bfq_lookup_next_entity(struct bfq_service_tree *st, bool force)
++__bfq_lookup_next_entity(struct bfq_service_tree *st, bool in_service
++#if 0
++			 , bool force
++#endif
++	)
  {
- 	struct bfq_entity *entity, *new_next_in_service = NULL;
+-	struct bfq_entity *entity, *new_next_in_service = NULL;
++	struct bfq_entity *entity
++#if 0
++		, *new_next_in_service = NULL
++#endif
++		;
++	u64 new_vtime;
 +	struct bfq_queue *bfqq;
  
  	if (RB_EMPTY_ROOT(&st->active))
  		return NULL;
-@@ -1009,6 +1190,24 @@ static struct bfq_entity *__bfq_lookup_next_entity(struct bfq_service_tree *st,
- 	entity = bfq_first_active_entity(st);
- 	BUG_ON(bfq_gt(entity->start, st->vtime));
  
+-	bfq_update_vtime(st);
+-	entity = bfq_first_active_entity(st);
+-	BUG_ON(bfq_gt(entity->start, st->vtime));
++	/*
++	 * Get the value of the system virtual time for which at
++	 * least one entity is eligible.
++	 */
++	new_vtime = bfq_calc_vtime_jump(st);
+ 
+ 	/*
+-	 * If the chosen entity does not match with the sched_data's
+-	 * next_in_service and we are forcedly serving the IDLE priority
+-	 * class tree, bubble up budget update.
++	 * If there is no in-service entity for the sched_data this
++	 * active tree belongs to, then push the system virtual time
++	 * up to the value that guarantees that at least one entity is
++	 * eligible. If, instead, there is an in-service entity, then
++	 * do not make any such update, because there is already an
++	 * eligible entity, namely the in-service one (even if the
++	 * entity is not on st, because it was extracted when set in
++	 * service).
+ 	 */
+-	if (unlikely(force && entity != entity->sched_data->next_in_service)) {
+-		new_next_in_service = entity;
+-		for_each_entity(new_next_in_service)
+-			bfq_update_budget(new_next_in_service);
++	if (!in_service)
++		bfq_update_vtime(st, new_vtime);
++
++	entity = bfq_first_active_entity(st, new_vtime);
++	BUG_ON(bfq_gt(entity->start, new_vtime));
++
++	/* Log some information */
 +	bfqq = bfq_entity_to_bfqq(entity);
 +	if (bfqq)
 +		bfq_log_bfqq(bfqq->bfqd, bfqq,
 +			     "__lookup_next: start %llu vtime %llu st %p",
 +			     ((entity->start>>10)*1000)>>12,
-+			     ((st->vtime>>10)*1000)>>12, st);
++			     ((new_vtime>>10)*1000)>>12, st);
 +#ifdef CONFIG_BFQ_GROUP_IOSCHED
 +	else {
 +		struct bfq_group *bfqg =
@@ -5225,69 +7620,115 @@ index a64fec1..7d73b9d 100644
 +		bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg,
 +			     "__lookup_next: start %llu vtime %llu st %p",
 +			     ((entity->start>>10)*1000)>>12,
-+			     ((st->vtime>>10)*1000)>>12, st);
++			     ((new_vtime>>10)*1000)>>12, st);
+ 	}
++#endif
++
++	BUG_ON(!entity);
+ 
+ 	return entity;
+ }
+@@ -1025,50 +1590,81 @@ static struct bfq_entity *__bfq_lookup_next_entity(struct bfq_service_tree *st,
+ /**
+  * bfq_lookup_next_entity - return the first eligible entity in @sd.
+  * @sd: the sched_data.
+- * @extract: if true the returned entity will be also extracted from @sd.
+  *
+- * NOTE: since we cache the next_in_service entity at each level of the
+- * hierarchy, the complexity of the lookup can be decreased with
+- * absolutely no effort just returning the cached next_in_service value;
+- * we prefer to do full lookups to test the consistency of * the data
+- * structures.
++ * This function is invoked when there has been a change in the trees
++ * for sd, and we need know what is the new next entity after this
++ * change.
+  */
+-static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd,
+-						 int extract,
+-						 struct bfq_data *bfqd)
++static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd)
+ {
+ 	struct bfq_service_tree *st = sd->service_tree;
+-	struct bfq_entity *entity;
+-	int i = 0;
+-
+-	BUG_ON(sd->in_service_entity);
++	struct bfq_service_tree *idle_class_st = st + (BFQ_IOPRIO_CLASSES - 1);
++	struct bfq_entity *entity = NULL;
++	struct bfq_queue *bfqq;
++	int class_idx = 0;
+ 
+-	if (bfqd &&
+-	    jiffies - bfqd->bfq_class_idle_last_service > BFQ_CL_IDLE_TIMEOUT) {
+-		entity = __bfq_lookup_next_entity(st + BFQ_IOPRIO_CLASSES - 1,
+-						  true);
+-		if (entity) {
+-			i = BFQ_IOPRIO_CLASSES - 1;
+-			bfqd->bfq_class_idle_last_service = jiffies;
+-			sd->next_in_service = entity;
+-		}
++	BUG_ON(!sd);
++	BUG_ON(!st);
++	/*
++	 * Choose from idle class, if needed to guarantee a minimum
++	 * bandwidth to this class (and if there is some active entity
++	 * in idle class). This should also mitigate
++	 * priority-inversion problems in case a low priority task is
++	 * holding file system resources.
++	 */
++	if (time_is_before_jiffies(sd->bfq_class_idle_last_service +
++				   BFQ_CL_IDLE_TIMEOUT)) {
++		if (!RB_EMPTY_ROOT(&idle_class_st->active))
++			class_idx = BFQ_IOPRIO_CLASSES - 1;
++		/* About to be served if backlogged, or not yet backlogged */
++		sd->bfq_class_idle_last_service = jiffies;
+ 	}
+-	for (; i < BFQ_IOPRIO_CLASSES; i++) {
+-		entity = __bfq_lookup_next_entity(st + i, false);
+-		if (entity) {
+-			if (extract) {
+-				bfq_check_next_in_service(sd, entity);
+-				bfq_active_extract(st + i, entity);
+-				sd->in_service_entity = entity;
+-				sd->next_in_service = NULL;
+-			}
++
++	/*
++	 * Find the next entity to serve for the highest-priority
++	 * class, unless the idle class needs to be served.
++	 */
++	for (; class_idx < BFQ_IOPRIO_CLASSES; class_idx++) {
++		entity = __bfq_lookup_next_entity(st + class_idx,
++						  sd->in_service_entity);
++
++		if (entity)
+ 			break;
+-		}
+ 	}
+ 
++	BUG_ON(!entity &&
++	       (!RB_EMPTY_ROOT(&st->active) || !RB_EMPTY_ROOT(&(st+1)->active) ||
++		!RB_EMPTY_ROOT(&(st+2)->active)));
++
++	if (!entity)
++		return NULL;
++
++	/* Log some information */
++	bfqq = bfq_entity_to_bfqq(entity);
++	if (bfqq)
++		bfq_log_bfqq(bfqq->bfqd, bfqq, "chosen from st %p %d",
++			     st + class_idx, class_idx);
++#ifdef CONFIG_BFQ_GROUP_IOSCHED
++	else {
++		struct bfq_group *bfqg =
++			container_of(entity, struct bfq_group, entity);
++
++		bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg,
++			     "chosen from st %p %d",
++			     st + class_idx, class_idx);
 +	}
 +#endif
 +
- 	/*
- 	 * If the chosen entity does not match with the sched_data's
- 	 * next_in_service and we are forcedly serving the IDLE priority
-@@ -1045,10 +1244,28 @@ static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd,
- 	BUG_ON(sd->in_service_entity);
- 
- 	if (bfqd &&
--	    jiffies - bfqd->bfq_class_idle_last_service > BFQ_CL_IDLE_TIMEOUT) {
-+	    jiffies - bfqd->bfq_class_idle_last_service >
-+	    BFQ_CL_IDLE_TIMEOUT) {
- 		entity = __bfq_lookup_next_entity(st + BFQ_IOPRIO_CLASSES - 1,
- 						  true);
- 		if (entity) {
-+			struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
-+			if (bfqq)
-+				bfq_log_bfqq(bfqd, bfqq,
-+					     "idle chosen from st %p %d",
-+					     st + BFQ_IOPRIO_CLASSES - 1,
-+					BFQ_IOPRIO_CLASSES - 1) ;
-+#ifdef CONFIG_BFQ_GROUP_IOSCHED
-+			else {
-+				struct bfq_group *bfqg =
-+				container_of(entity, struct bfq_group, entity);
-+
-+				bfq_log_bfqg(bfqd, bfqg,
-+					     "idle chosen from st %p %d",
-+					     st + BFQ_IOPRIO_CLASSES - 1,
-+					BFQ_IOPRIO_CLASSES - 1) ;
-+			}
-+#endif
- 			i = BFQ_IOPRIO_CLASSES - 1;
- 			bfqd->bfq_class_idle_last_service = jiffies;
- 			sd->next_in_service = entity;
-@@ -1057,6 +1274,24 @@ static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd,
- 	for (; i < BFQ_IOPRIO_CLASSES; i++) {
- 		entity = __bfq_lookup_next_entity(st + i, false);
- 		if (entity) {
-+			if (bfqd != NULL) {
-+			struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
-+			if (bfqq)
-+				bfq_log_bfqq(bfqd, bfqq,
-+					     "chosen from st %p %d",
-+					     st + i, i) ;
-+#ifdef CONFIG_BFQ_GROUP_IOSCHED
-+			else {
-+				struct bfq_group *bfqg =
-+				container_of(entity, struct bfq_group, entity);
-+
-+				bfq_log_bfqg(bfqd, bfqg,
-+					     "chosen from st %p %d",
-+					     st + i, i) ;
-+			}
-+#endif
-+			}
-+
- 			if (extract) {
- 				bfq_check_next_in_service(sd, entity);
- 				bfq_active_extract(st + i, entity);
-@@ -1070,6 +1305,13 @@ static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd,
  	return entity;
  }
  
@@ -5301,10 +7742,19 @@ index a64fec1..7d73b9d 100644
  /*
   * Get next queue for service.
   */
-@@ -1086,7 +1328,36 @@ static struct bfq_queue *bfq_get_next_queue(struct bfq_data *bfqd)
+@@ -1083,58 +1679,208 @@ static struct bfq_queue *bfq_get_next_queue(struct bfq_data *bfqd)
+ 	if (bfqd->busy_queues == 0)
+ 		return NULL;
  
++	/*
++	 * Traverse the path from the root to the leaf entity to
++	 * serve. Set in service all the entities visited along the
++	 * way.
++	 */
  	sd = &bfqd->root_group->sched_data;
  	for (; sd ; sd = entity->my_sched_data) {
+-		entity = bfq_lookup_next_entity(sd, 1, bfqd);
+-		BUG_ON(!entity);
 +#ifdef CONFIG_BFQ_GROUP_IOSCHED
 +		if (entity) {
 +			struct bfq_group *bfqg =
@@ -5312,13 +7762,96 @@ index a64fec1..7d73b9d 100644
 +
 +			bfq_log_bfqg(bfqd, bfqg,
 +				     "get_next_queue: lookup in this group");
-+		} else
++			if (!sd->next_in_service)
++				pr_crit("get_next_queue: lookup in this group");
++		} else {
 +			bfq_log_bfqg(bfqd, bfqd->root_group,
 +				     "get_next_queue: lookup in root group");
++			if (!sd->next_in_service)
++				pr_crit("get_next_queue: lookup in root group");
++		}
 +#endif
 +
- 		entity = bfq_lookup_next_entity(sd, 1, bfqd);
++		BUG_ON(!sd->next_in_service);
 +
++		/*
++		 * WARNING. We are about to set the in-service entity
++		 * to sd->next_in_service, i.e., to the (cached) value
++		 * returned by bfq_lookup_next_entity(sd) the last
++		 * time it was invoked, i.e., the last time when the
++		 * service order in sd changed as a consequence of the
++		 * activation or deactivation of an entity. In this
++		 * respect, if we execute bfq_lookup_next_entity(sd)
++		 * in this very moment, it may, although with low
++		 * probability, yield a different entity than that
++		 * pointed to by sd->next_in_service. This rare event
++		 * happens in case there was no CLASS_IDLE entity to
++		 * serve for sd when bfq_lookup_next_entity(sd) was
++		 * invoked for the last time, while there is now one
++		 * such entity.
++		 *
++		 * If the above event happens, then the scheduling of
++		 * such entity in CLASS_IDLE is postponed until the
++		 * service of the sd->next_in_service entity
++		 * finishes. In fact, when the latter is expired,
++		 * bfq_lookup_next_entity(sd) gets called again,
++		 * exactly to update sd->next_in_service.
++		 */
++
++		/* Make next_in_service entity become in_service_entity */
++		entity = sd->next_in_service;
++		sd->in_service_entity = entity;
++
++		/*
++		 * Reset the accumulator of the amount of service that
++		 * the entity is about to receive.
++		 */
+ 		entity->service = 0;
++
++		/*
++		 * If entity is no longer a candidate for next
++		 * service, then we extract it from its active tree,
++		 * for the following reason. To further boost the
++		 * throughput in some special case, BFQ needs to know
++		 * which is the next candidate entity to serve, while
++		 * there is already an entity in service. In this
++		 * respect, to make it easy to compute/update the next
++		 * candidate entity to serve after the current
++		 * candidate has been set in service, there is a case
++		 * where it is necessary to extract the current
++		 * candidate from its service tree. Such a case is
++		 * when the entity just set in service cannot be also
++		 * a candidate for next service. Details about when
++		 * this conditions holds are reported in the comments
++		 * on the function bfq_no_longer_next_in_service()
++		 * invoked below.
++		 */
++		if (bfq_no_longer_next_in_service(entity))
++			bfq_active_extract(bfq_entity_service_tree(entity),
++					   entity);
++
++		/*
++		 * For the same reason why we may have just extracted
++		 * entity from its active tree, we may need to update
++		 * next_in_service for the sched_data of entity too,
++		 * regardless of whether entity has been extracted.
++		 * In fact, even if entity has not been extracted, a
++		 * descendant entity may get extracted. Such an event
++		 * would cause a change in next_in_service for the
++		 * level of the descendant entity, and thus possibly
++		 * back to upper levels.
++		 *
++		 * We cannot perform the resulting needed update
++		 * before the end of this loop, because, to know which
++		 * is the correct next-to-serve candidate entity for
++		 * each level, we need first to find the leaf entity
++		 * to set in service. In fact, only after we know
++		 * which is the next-to-serve leaf entity, we can
++		 * discover whether the parent entity of the leaf
++		 * entity becomes the next-to-serve, and so on.
++		 */
++
++		/* Log some information */
 +		bfqq = bfq_entity_to_bfqq(entity);
 +		if (bfqq)
 +			bfq_log_bfqq(bfqd, bfqq,
@@ -5335,27 +7868,84 @@ index a64fec1..7d73b9d 100644
 +		}
 +#endif
 +
- 		BUG_ON(!entity);
- 		entity->service = 0;
  	}
-@@ -1113,9 +1384,7 @@ static void bfq_deactivate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+ 
++	BUG_ON(!entity);
+ 	bfqq = bfq_entity_to_bfqq(entity);
+ 	BUG_ON(!bfqq);
+ 
++	/*
++	 * We can finally update all next-to-serve entities along the
++	 * path from the leaf entity just set in service to the root.
++	 */
++	for_each_entity(entity) {
++		struct bfq_sched_data *sd = entity->sched_data;
++
++		if(!bfq_update_next_in_service(sd, NULL))
++			break;
++	}
++
+ 	return bfqq;
+ }
+ 
+ static void __bfq_bfqd_reset_in_service(struct bfq_data *bfqd)
+ {
++	struct bfq_entity *entity = &bfqd->in_service_queue->entity;
++
+ 	if (bfqd->in_service_bic) {
+ 		put_io_context(bfqd->in_service_bic->icq.ioc);
+ 		bfqd->in_service_bic = NULL;
+ 	}
+ 
++	bfq_clear_bfqq_wait_request(bfqd->in_service_queue);
++	hrtimer_try_to_cancel(&bfqd->idle_slice_timer);
+ 	bfqd->in_service_queue = NULL;
+-	del_timer(&bfqd->idle_slice_timer);
++
++	/*
++	 * When this function is called, all in-service entities have
++	 * been properly deactivated or requeued, so we can safely
++	 * execute the final step: reset in_service_entity along the
++	 * path from entity to the root.
++	 */
++	for_each_entity(entity)
++		entity->sched_data->in_service_entity = NULL;
+ }
+ 
+ static void bfq_deactivate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+-				int requeue)
++				bool ins_into_idle_tree, bool expiration)
  {
  	struct bfq_entity *entity = &bfqq->entity;
  
 -	if (bfqq == bfqd->in_service_queue)
 -		__bfq_bfqd_reset_in_service(bfqd);
 -
-+	BUG_ON(bfqq == bfqd->in_service_queue);
- 	bfq_deactivate_entity(entity, requeue);
+-	bfq_deactivate_entity(entity, requeue);
++	bfq_deactivate_entity(entity, ins_into_idle_tree, expiration);
  }
  
-@@ -1123,12 +1392,11 @@ static void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+ static void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  {
  	struct bfq_entity *entity = &bfqq->entity;
++	struct bfq_service_tree *st = bfq_entity_service_tree(entity);
++
++	BUG_ON(bfqq == bfqd->in_service_queue);
++	BUG_ON(entity->tree != &st->active && entity->tree != &st->idle &&
++	       entity->on_st);
  
 -	bfq_activate_entity(entity);
-+	bfq_activate_entity(entity, bfq_bfqq_non_blocking_wait_rq(bfqq));
++	bfq_activate_requeue_entity(entity, bfq_bfqq_non_blocking_wait_rq(bfqq),
++				    false);
 +	bfq_clear_bfqq_non_blocking_wait_rq(bfqq);
++}
++
++static void bfq_requeue_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)
++{
++	struct bfq_entity *entity = &bfqq->entity;
++
++	bfq_activate_requeue_entity(entity, false,
++				    bfqq == bfqd->in_service_queue);
  }
  
 -#ifdef CONFIG_BFQ_GROUP_IOSCHED
@@ -5364,15 +7954,17 @@ index a64fec1..7d73b9d 100644
  
  /*
   * Called when the bfqq no longer has requests pending, remove it from
-@@ -1139,6 +1407,7 @@ static void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+- * the service tree.
++ * the service tree. As a special case, it can be invoked during an
++ * expiration.
+  */
+ static void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+-			      int requeue)
++			      bool expiration)
  {
  	BUG_ON(!bfq_bfqq_busy(bfqq));
  	BUG_ON(!RB_EMPTY_ROOT(&bfqq->sort_list));
-+	BUG_ON(bfqq == bfqd->in_service_queue);
- 
- 	bfq_log_bfqq(bfqd, bfqq, "del from busy");
- 
-@@ -1147,27 +1416,20 @@ static void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -1146,27 +1892,20 @@ static void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  	BUG_ON(bfqd->busy_queues == 0);
  	bfqd->busy_queues--;
  
@@ -5398,15 +7990,16 @@ index a64fec1..7d73b9d 100644
  	bfqg_stats_update_dequeue(bfqq_group(bfqq));
 -#endif
  
+-	bfq_deactivate_bfqq(bfqd, bfqq, requeue);
 +	BUG_ON(bfqq->entity.budget < 0);
-+	
- 	bfq_deactivate_bfqq(bfqd, bfqq, requeue);
++
++	bfq_deactivate_bfqq(bfqd, bfqq, true, expiration);
 +
 +	BUG_ON(bfqq->entity.budget < 0);
  }
  
  /*
-@@ -1185,16 +1447,11 @@ static void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+@@ -1184,16 +1923,11 @@ static void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  	bfq_mark_bfqq_busy(bfqq);
  	bfqd->busy_queues++;
  
@@ -5426,17 +8019,28 @@ index a64fec1..7d73b9d 100644
  		bfqd->wr_busy_queues++;
  }
 diff --git a/block/bfq.h b/block/bfq.h
-index f73c942..49d28b9 100644
+index fcce855..2a2bc30 100644
 --- a/block/bfq.h
 +++ b/block/bfq.h
 @@ -1,5 +1,5 @@
  /*
 - * BFQ-v7r11 for 4.5.0: data structures and common functions prototypes.
-+ * BFQ-v8r3 for 4.7.0: data structures and common functions prototypes.
++ * BFQ v8r8 for 4.10.0: data structures and common functions prototypes.
   *
   * Based on ideas and code from CFQ:
   * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
-@@ -28,20 +28,21 @@
+@@ -7,7 +7,9 @@
+  * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
+  *		      Paolo Valente <paolo.valente@unimore.it>
+  *
+- * Copyright (C) 2010 Paolo Valente <paolo.valente@unimore.it>
++ * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
++ *
++ * Copyright (C) 2017 Paolo Valente <paolo.valente@linaro.org>
+  */
+ 
+ #ifndef _BFQ_H
+@@ -28,20 +30,21 @@
  
  #define BFQ_DEFAULT_QUEUE_IOPRIO	4
  
@@ -5465,7 +8069,7 @@ index f73c942..49d28b9 100644
   *
   * Each service tree represents a B-WF2Q+ scheduler on its own.  Each
   * ioprio_class has its own independent scheduler, and so its own
-@@ -49,27 +50,28 @@ struct bfq_entity;
+@@ -49,27 +52,28 @@ struct bfq_entity;
   * of the containing bfqd.
   */
  struct bfq_service_tree {
@@ -5476,8 +8080,8 @@ index f73c942..49d28b9 100644
  
 -	struct bfq_entity *first_idle;
 -	struct bfq_entity *last_idle;
-+	struct bfq_entity *first_idle; 	/* idle entity with minimum F_i */
-+	struct bfq_entity *last_idle; 	/* idle entity with maximum F_i */
++	struct bfq_entity *first_idle;	/* idle entity with minimum F_i */
++	struct bfq_entity *last_idle;	/* idle entity with maximum F_i */
  
 -	u64 vtime;
 +	u64 vtime; /* scheduler virtual time */
@@ -5504,7 +8108,7 @@ index f73c942..49d28b9 100644
   *
   * The supported ioprio_classes are the same as in CFQ, in descending
   * priority order, IOPRIO_CLASS_RT, IOPRIO_CLASS_BE, IOPRIO_CLASS_IDLE.
-@@ -79,48 +81,29 @@ struct bfq_service_tree {
+@@ -79,48 +83,32 @@ struct bfq_service_tree {
   * All the fields are protected by the queue lock of the containing bfqd.
   */
  struct bfq_sched_data {
@@ -5514,6 +8118,9 @@ index f73c942..49d28b9 100644
  	struct bfq_entity *next_in_service;
 +	/* array of service trees, one per ioprio_class */
  	struct bfq_service_tree service_tree[BFQ_IOPRIO_CLASSES];
++	/* last time CLASS_IDLE was served */
++	unsigned long bfq_class_idle_last_service;
++
  };
  
  /**
@@ -5562,7 +8169,7 @@ index f73c942..49d28b9 100644
   *
   * A bfq_entity is used to represent either a bfq_queue (leaf node in the
   * cgroup hierarchy) or a bfq_group into the upper level scheduler.  Each
-@@ -147,27 +130,52 @@ struct bfq_weight_counter {
+@@ -147,27 +135,52 @@ struct bfq_weight_counter {
   * containing bfqd.
   */
  struct bfq_entity {
@@ -5571,11 +8178,12 @@ index f73c942..49d28b9 100644
 +	/* pointer to the weight counter associated with this entity */
  	struct bfq_weight_counter *weight_counter;
  
+-	int on_st;
 +	/*
-+	 * flag, true if the entity is on a tree (either the active or
-+	 * the idle one of its service_tree).
++	 * Flag, true if the entity is on a tree (either the active or
++	 * the idle one of its service_tree) or is in service.
 +	 */
- 	int on_st;
++	bool on_st;
  
 -	u64 finish;
 -	u64 start;
@@ -5600,7 +8208,7 @@ index f73c942..49d28b9 100644
 +	/* budget, used also to calculate F_i: F_i = S_i + @budget / @weight */
 +	int budget;
 +
-+	unsigned int weight; 	/* weight of the queue */
++	unsigned int weight;	 /* weight of the queue */
 +	unsigned int new_weight; /* next weight if a change is in progress */
 +
 +	/* original weight, used to implement weight boosting */
@@ -5621,7 +8229,7 @@ index f73c942..49d28b9 100644
  	int prio_changed;
  };
  
-@@ -175,56 +183,6 @@ struct bfq_group;
+@@ -175,56 +188,6 @@ struct bfq_group;
  
  /**
   * struct bfq_queue - leaf schedulable entity.
@@ -5678,7 +8286,7 @@ index f73c942..49d28b9 100644
   *
   * A bfq_queue is a leaf request queue; it can be associated with an
   * io_context or more, if it  is  async or shared  between  cooperating
-@@ -235,117 +193,163 @@ struct bfq_group;
+@@ -235,117 +198,175 @@ struct bfq_group;
   * All the fields are protected by the queue lock of the containing bfqd.
   */
  struct bfq_queue {
@@ -5794,6 +8402,10 @@ index f73c942..49d28b9 100644
 +	 * last transition from idle to backlogged.
 +	 */
  	unsigned long service_from_backlogged;
++	/*
++	 * Value of wr start time when switching to soft rt
++	 */
++	unsigned long wr_start_at_switch_to_srt;
 +
 +	unsigned long split_time; /* time of last split */
  };
@@ -5806,11 +8418,11 @@ index f73c942..49d28b9 100644
   */
  struct bfq_ttime {
 -	unsigned long last_end_request;
-+	unsigned long last_end_request; /* completion time of last request */
++	u64 last_end_request; /* completion time of last request */
 +
-+	unsigned long ttime_total; /* total process thinktime */
++	u64 ttime_total; /* total process thinktime */
 +	unsigned long ttime_samples; /* number of thinktime samples */
-+	unsigned long ttime_mean; /* average process thinktime */
++	u64 ttime_mean; /* average process thinktime */
  
 -	unsigned long ttime_total;
 -	unsigned long ttime_samples;
@@ -5883,13 +8495,20 @@ index f73c942..49d28b9 100644
 +	 * with another cooperating queue.
 +	 */
  	bool was_in_burst_list;
--
+ 
 -	unsigned int cooperations;
 -	unsigned int failed_cooperations;
++	/*
++	 * Similar to previous fields: save wr information.
++	 */
++	unsigned long saved_wr_coeff;
++	unsigned long saved_last_wr_start_finish;
++	unsigned long saved_wr_start_at_switch_to_srt;
++	unsigned int saved_wr_cur_max_time;
  };
  
  enum bfq_device_speed {
-@@ -354,224 +358,216 @@ enum bfq_device_speed {
+@@ -354,224 +375,232 @@ enum bfq_device_speed {
  };
  
  /**
@@ -6000,10 +8619,10 @@ index f73c942..49d28b9 100644
 - *                      @last_ins_in_burst.
 - * @burst_size: number of queues in the current burst of queue activations.
 - * @bfq_large_burst_thresh: maximum burst size above which the current
-- * 			    queue-activation burst is deemed as 'large'.
+- *			    queue-activation burst is deemed as 'large'.
 - * @large_burst: true if a large queue-activation burst is in progress.
 - * @burst_list: head of the burst list (as for the above fields, more details
-- * 		in the comments to the function bfq_handle_burst).
+- *		in the comments to the function bfq_handle_burst).
 - * @low_latency: if set to true, low-latency heuristics are enabled.
 - * @bfq_wr_coeff: maximum factor by which the weight of a weight-raised
 - *                queue is multiplied.
@@ -6083,11 +8702,12 @@ index f73c942..49d28b9 100644
 +	/* number of budgets assigned */
  	int budgets_assigned;
  
+-	struct timer_list idle_slice_timer;
 +	/*
 +	 * Timer set when idling (waiting) for the next request from
 +	 * the queue in service.
 +	 */
- 	struct timer_list idle_slice_timer;
++	struct hrtimer idle_slice_timer;
 +	/* delayed work to restart dispatching on the request queue */
  	struct work_struct unplug_work;
  
@@ -6099,14 +8719,33 @@ index f73c942..49d28b9 100644
 +	/* on-disk position of the last served request */
  	sector_t last_position;
  
++	/* time of last request completion (ns) */
++	u64 last_completion;
++
++	/* time of first rq dispatch in current observation interval (ns) */
++	u64 first_dispatch;
++	/* time of last rq dispatch in current observation interval (ns) */
++	u64 last_dispatch;
++
 +	/* beginning of the last budget */
  	ktime_t last_budget_start;
 +	/* beginning of the last idle slice */
  	ktime_t last_idling_start;
-+	/* number of samples used to calculate @peak_rate */
++
++	/* number of samples in current observation interval */
  	int peak_rate_samples;
-+	/* peak transfer rate observed for a budget */
- 	u64 peak_rate;
+-	u64 peak_rate;
++	/* num of samples of seq dispatches in current observation interval */
++	u32 sequential_samples;
++	/* total num of sectors transferred in current observation interval */
++	u64 tot_sectors_dispatched;
++	/* max rq size seen during current observation interval (sectors) */
++	u32 last_rq_max_size;
++	/* time elapsed from first dispatch in current observ. interval (us) */
++	u64 delta_from_first;
++	/* current estimate of device peak rate */
++	u32 peak_rate;
++
 +	/* maximum budget allotted to a bfq_queue before rescheduling */
  	int bfq_max_budget;
  
@@ -6115,19 +8754,20 @@ index f73c942..49d28b9 100644
 +	/* list of all the bfq_queues idle on the device */
  	struct list_head idle_list;
  
+-	unsigned int bfq_fifo_expire[2];
 +	/*
 +	 * Timeout for async/sync requests; when it fires, requests
 +	 * are served in fifo order.
 +	 */
- 	unsigned int bfq_fifo_expire[2];
++	u64 bfq_fifo_expire[2];
 +	/* weight of backward seeks wrt forward ones */
  	unsigned int bfq_back_penalty;
 +	/* maximum allowed backward seek */
  	unsigned int bfq_back_max;
+-	unsigned int bfq_slice_idle;
+-	u64 bfq_class_idle_last_service;
 +	/* maximum idling time */
- 	unsigned int bfq_slice_idle;
-+	/* last time CLASS_IDLE was served */
- 	u64 bfq_class_idle_last_service;
++	u32 bfq_slice_idle;
  
 +	/* user-configured max budget value (0 for auto-tuning) */
  	int bfq_user_max_budget;
@@ -6250,7 +8890,7 @@ index f73c942..49d28b9 100644
  	BFQ_BFQQ_FLAG_IO_bound,		/*
  					 * bfqq has timed-out at least once
  					 * having consumed at most 2/10 of
-@@ -581,17 +577,12 @@ enum bfqq_state_flags {
+@@ -581,17 +610,12 @@ enum bfqq_state_flags {
  					 * bfqq activated in a large burst,
  					 * see comments to bfq_handle_burst.
  					 */
@@ -6269,7 +8909,7 @@ index f73c942..49d28b9 100644
  };
  
  #define BFQ_BFQQ_FNS(name)						\
-@@ -608,25 +599,53 @@ static int bfq_bfqq_##name(const struct bfq_queue *bfqq)		\
+@@ -608,28 +632,94 @@ static int bfq_bfqq_##name(const struct bfq_queue *bfqq)		\
  	return ((bfqq)->flags & (1 << BFQ_BFQQ_FLAG_##name)) != 0;	\
  }
  
@@ -6294,6 +8934,43 @@ index f73c942..49d28b9 100644
  /* Logging facilities. */
 -#define bfq_log_bfqq(bfqd, bfqq, fmt, args...) \
 -	blk_add_trace_msg((bfqd)->queue, "bfq%d " fmt, (bfqq)->pid, ##args)
++#ifdef CONFIG_BFQ_REDIRECT_TO_CONSOLE
++#ifdef CONFIG_BFQ_GROUP_IOSCHED
++static struct bfq_group *bfqq_group(struct bfq_queue *bfqq);
++static struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg);
++
++#define bfq_log_bfqq(bfqd, bfqq, fmt, args...)	do {			\
++	char __pbuf[128];						\
++									\
++	assert_spin_locked((bfqd)->queue->queue_lock);			\
++	blkg_path(bfqg_to_blkg(bfqq_group(bfqq)), __pbuf, sizeof(__pbuf)); \
++	pr_crit("bfq%d%c %s " fmt "\n", 			\
++		(bfqq)->pid,						\
++		bfq_bfqq_sync((bfqq)) ? 'S' : 'A',			\
++		__pbuf, ##args);					\
++} while (0)
++
++#define bfq_log_bfqg(bfqd, bfqg, fmt, args...)	do {			\
++	char __pbuf[128];						\
++									\
++	blkg_path(bfqg_to_blkg(bfqg), __pbuf, sizeof(__pbuf));		\
++	pr_crit("%s " fmt "\n", __pbuf, ##args);	\
++} while (0)
++
++#else /* CONFIG_BFQ_GROUP_IOSCHED */
++
++#define bfq_log_bfqq(bfqd, bfqq, fmt, args...)		\
++	pr_crit("bfq%d%c " fmt "\n", (bfqq)->pid,		\
++		bfq_bfqq_sync((bfqq)) ? 'S' : 'A',	\
++		##args)
++#define bfq_log_bfqg(bfqd, bfqg, fmt, args...)		do {} while (0)
++
++#endif /* CONFIG_BFQ_GROUP_IOSCHED */
++
++#define bfq_log(bfqd, fmt, args...) \
++	pr_crit("bfq " fmt "\n", ##args)
++
++#else /* CONFIG_BFQ_REDIRECT_TO_CONSOLE */
 +#ifdef CONFIG_BFQ_GROUP_IOSCHED
 +static struct bfq_group *bfqq_group(struct bfq_queue *bfqq);
 +static struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg);
@@ -6328,7 +9005,11 @@ index f73c942..49d28b9 100644
  
  #define bfq_log(bfqd, fmt, args...) \
  	blk_add_trace_msg((bfqd)->queue, "bfq " fmt, ##args)
-@@ -640,15 +659,12 @@ enum bfqq_expiration {
++#endif /* CONFIG_BFQ_REDIRECT_TO_CONSOLE */
+ 
+ /* Expiration reasons. */
+ enum bfqq_expiration {
+@@ -640,15 +730,12 @@ enum bfqq_expiration {
  	BFQ_BFQQ_BUDGET_TIMEOUT,	/* budget took too long to be used */
  	BFQ_BFQQ_BUDGET_EXHAUSTED,	/* budget consumed */
  	BFQ_BFQQ_NO_MORE_REQUESTS,	/* the queue has no more requests */
@@ -6346,7 +9027,7 @@ index f73c942..49d28b9 100644
  	/* number of ios merged */
  	struct blkg_rwstat		merged;
  	/* total time spent on device in ns, may not be accurate w/ queueing */
-@@ -657,12 +673,8 @@ struct bfqg_stats {
+@@ -657,12 +744,8 @@ struct bfqg_stats {
  	struct blkg_rwstat		wait_time;
  	/* number of IOs queued up */
  	struct blkg_rwstat		queued;
@@ -6359,7 +9040,7 @@ index f73c942..49d28b9 100644
  	/* sum of number of ios queued across all samples */
  	struct blkg_stat		avg_queue_size_sum;
  	/* count of samples taken for average */
-@@ -680,8 +692,10 @@ struct bfqg_stats {
+@@ -680,8 +763,10 @@ struct bfqg_stats {
  	uint64_t			start_idle_time;
  	uint64_t			start_empty_time;
  	uint16_t			flags;
@@ -6370,7 +9051,7 @@ index f73c942..49d28b9 100644
  /*
   * struct bfq_group_data - per-blkcg storage for the blkio subsystem.
   *
-@@ -692,7 +706,7 @@ struct bfq_group_data {
+@@ -692,7 +777,7 @@ struct bfq_group_data {
  	/* must be the first member */
  	struct blkcg_policy_data pd;
  
@@ -6379,7 +9060,7 @@ index f73c942..49d28b9 100644
  };
  
  /**
-@@ -712,7 +726,7 @@ struct bfq_group_data {
+@@ -712,7 +797,7 @@ struct bfq_group_data {
   *                   unused for the root group. Used to know whether there
   *                   are groups with more than one active @bfq_entity
   *                   (see the comments to the function
@@ -6388,7 +9069,7 @@ index f73c942..49d28b9 100644
   * @rq_pos_tree: rbtree sorted by next_request position, used when
   *               determining if two or more queues have interleaving
   *               requests (see bfq_find_close_cooperator()).
-@@ -745,7 +759,6 @@ struct bfq_group {
+@@ -745,7 +830,6 @@ struct bfq_group {
  	struct rb_root rq_pos_tree;
  
  	struct bfqg_stats stats;
@@ -6396,12 +9077,26 @@ index f73c942..49d28b9 100644
  };
  
  #else
-@@ -767,11 +780,25 @@ bfq_entity_service_tree(struct bfq_entity *entity)
+@@ -761,17 +845,38 @@ struct bfq_group {
+ 
+ static struct bfq_queue *bfq_entity_to_bfqq(struct bfq_entity *entity);
+ 
++static unsigned int bfq_class_idx(struct bfq_entity *entity)
++{
++	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
++
++	return bfqq ? bfqq->ioprio_class - 1 :
++		BFQ_DEFAULT_GRP_CLASS - 1;
++}
++
+ static struct bfq_service_tree *
+ bfq_entity_service_tree(struct bfq_entity *entity)
+ {
  	struct bfq_sched_data *sched_data = entity->sched_data;
  	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
- 	unsigned int idx = bfqq ? bfqq->ioprio_class - 1 :
+-	unsigned int idx = bfqq ? bfqq->ioprio_class - 1 :
 -				  BFQ_DEFAULT_GRP_CLASS;
-+				  BFQ_DEFAULT_GRP_CLASS - 1;
++	unsigned int idx = bfq_class_idx(entity);
  
  	BUG_ON(idx >= BFQ_IOPRIO_CLASSES);
  	BUG_ON(sched_data == NULL);
@@ -6409,7 +9104,7 @@ index f73c942..49d28b9 100644
 +	if (bfqq)
 +		bfq_log_bfqq(bfqq->bfqd, bfqq,
 +			     "entity_service_tree %p %d",
-+			     sched_data->service_tree + idx, idx) ;
++			     sched_data->service_tree + idx, idx);
 +#ifdef CONFIG_BFQ_GROUP_IOSCHED
 +	else {
 +		struct bfq_group *bfqg =
@@ -6417,13 +9112,13 @@ index f73c942..49d28b9 100644
 +
 +		bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg,
 +			     "entity_service_tree %p %d",
-+			     sched_data->service_tree + idx, idx) ;
++			     sched_data->service_tree + idx, idx);
 +	}
 +#endif
  	return sched_data->service_tree + idx;
  }
  
-@@ -791,47 +818,6 @@ static struct bfq_data *bic_to_bfqd(struct bfq_io_cq *bic)
+@@ -791,47 +896,6 @@ static struct bfq_data *bic_to_bfqd(struct bfq_io_cq *bic)
  	return bic->icq.q->elevator->elevator_data;
  }
  
@@ -6471,7 +9166,7 @@ index f73c942..49d28b9 100644
  #ifdef CONFIG_BFQ_GROUP_IOSCHED
  
  static struct bfq_group *bfq_bfqq_to_bfqg(struct bfq_queue *bfqq)
-@@ -857,11 +843,13 @@ static void bfq_check_ioprio_change(struct bfq_io_cq *bic, struct bio *bio);
+@@ -857,11 +921,13 @@ static void bfq_check_ioprio_change(struct bfq_io_cq *bic, struct bio *bio);
  static void bfq_put_queue(struct bfq_queue *bfqq);
  static void bfq_dispatch_insert(struct request_queue *q, struct request *rq);
  static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd,
@@ -6488,5 +9183,5 @@ index f73c942..49d28b9 100644
  
  #endif /* _BFQ_H */
 -- 
-1.9.1
+2.10.0
 
diff --git a/patches/0009-Implement-min-and-msec-hrtimeout-un-interruptible-sc.patch b/patches/0009-Implement-min-and-msec-hrtimeout-un-interruptible-sc.patch
new file mode 100644
index 0000000..ba027c0
--- /dev/null
+++ b/patches/0009-Implement-min-and-msec-hrtimeout-un-interruptible-sc.patch
@@ -0,0 +1,111 @@
+From a7fb2842267fd275cae9cf44dd3037469f75eeef Mon Sep 17 00:00:00 2001
+From: Con Kolivas <kernel@kolivas.org>
+Date: Tue, 1 Nov 2016 12:54:20 +1100
+Subject: [PATCH 09/25] Implement min and msec hrtimeout un/interruptible
+ schedule timeout variants with a lower resolution of 1ms to work around low
+ Hz time resolutions.
+
+---
+ include/linux/sched.h |  6 +++++
+ kernel/time/hrtimer.c | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++
+ 2 files changed, 77 insertions(+)
+
+diff --git a/include/linux/sched.h b/include/linux/sched.h
+index d752ef6..46544f4 100644
+--- a/include/linux/sched.h
++++ b/include/linux/sched.h
+@@ -457,6 +457,12 @@ extern signed long schedule_timeout_interruptible(signed long timeout);
+ extern signed long schedule_timeout_killable(signed long timeout);
+ extern signed long schedule_timeout_uninterruptible(signed long timeout);
+ extern signed long schedule_timeout_idle(signed long timeout);
++
++extern signed long schedule_msec_hrtimeout(signed long timeout);
++extern signed long schedule_min_hrtimeout(void);
++extern signed long schedule_msec_hrtimeout_interruptible(signed long timeout);
++extern signed long schedule_msec_hrtimeout_uninterruptible(signed long timeout);
++
+ asmlinkage void schedule(void);
+ extern void schedule_preempt_disabled(void);
+ 
+diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
+index c6ecedd..a47f5b3 100644
+--- a/kernel/time/hrtimer.c
++++ b/kernel/time/hrtimer.c
+@@ -1796,3 +1796,74 @@ int __sched schedule_hrtimeout(ktime_t *expires,
+ 	return schedule_hrtimeout_range(expires, 0, mode);
+ }
+ EXPORT_SYMBOL_GPL(schedule_hrtimeout);
++
++/*
++ * As per schedule_hrtimeout but taskes a millisecond value and returns how
++ * many milliseconds are left.
++ */
++signed long __sched schedule_msec_hrtimeout(signed long timeout)
++{
++	struct hrtimer_sleeper t;
++	int delta, secs, jiffs;
++	ktime_t expires;
++
++	if (!timeout) {
++		__set_current_state(TASK_RUNNING);
++		return 0;
++	}
++
++	jiffs = msecs_to_jiffies(timeout);
++	/*
++	 * If regular timer resolution is adequate or hrtimer resolution is not
++	 * (yet) better than Hz, as would occur during startup, use regular
++	 * timers.
++	 */
++	if (jiffs > 4 || hrtimer_resolution >= NSEC_PER_SEC / HZ)
++		return schedule_timeout(jiffs);
++
++	secs = timeout / 1000;
++	delta = (timeout % 1000) * NSEC_PER_MSEC;
++	expires = ktime_set(secs, delta);
++
++	hrtimer_init_on_stack(&t.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
++	hrtimer_set_expires_range_ns(&t.timer, expires, delta);
++
++	hrtimer_init_sleeper(&t, current);
++
++	hrtimer_start_expires(&t.timer, HRTIMER_MODE_REL);
++
++	if (likely(t.task))
++		schedule();
++
++	hrtimer_cancel(&t.timer);
++	destroy_hrtimer_on_stack(&t.timer);
++
++	__set_current_state(TASK_RUNNING);
++
++	expires = hrtimer_expires_remaining(&t.timer);
++	timeout = ktime_to_ms(expires);
++	return timeout < 0 ? 0 : timeout;
++}
++
++EXPORT_SYMBOL(schedule_msec_hrtimeout);
++
++signed long __sched schedule_min_hrtimeout(void)
++{
++	return schedule_msec_hrtimeout(1);
++}
++
++EXPORT_SYMBOL(schedule_min_hrtimeout);
++
++signed long __sched schedule_msec_hrtimeout_interruptible(signed long timeout)
++{
++	__set_current_state(TASK_INTERRUPTIBLE);
++	return schedule_msec_hrtimeout(timeout);
++}
++EXPORT_SYMBOL(schedule_msec_hrtimeout_interruptible);
++
++signed long __sched schedule_msec_hrtimeout_uninterruptible(signed long timeout)
++{
++	__set_current_state(TASK_UNINTERRUPTIBLE);
++	return schedule_msec_hrtimeout(timeout);
++}
++EXPORT_SYMBOL(schedule_msec_hrtimeout_uninterruptible);
+-- 
+2.9.3
+
diff --git a/patches/0010-Special-case-calls-of-schedule_timeout-1-to-use-the-.patch b/patches/0010-Special-case-calls-of-schedule_timeout-1-to-use-the-.patch
new file mode 100644
index 0000000..ee79d88
--- /dev/null
+++ b/patches/0010-Special-case-calls-of-schedule_timeout-1-to-use-the-.patch
@@ -0,0 +1,48 @@
+From a4f3820228ebab3d5d480d720fecebd3f7e71771 Mon Sep 17 00:00:00 2001
+From: Con Kolivas <kernel@kolivas.org>
+Date: Sat, 5 Nov 2016 09:27:36 +1100
+Subject: [PATCH 10/25] Special case calls of schedule_timeout(1) to use the
+ min hrtimeout of 1ms, working around low Hz resolutions.
+
+---
+ kernel/time/timer.c | 15 +++++++++++++--
+ 1 file changed, 13 insertions(+), 2 deletions(-)
+
+diff --git a/kernel/time/timer.c b/kernel/time/timer.c
+index ef3128f..3f72c13 100644
+--- a/kernel/time/timer.c
++++ b/kernel/time/timer.c
+@@ -1750,6 +1750,17 @@ signed long __sched schedule_timeout(signed long timeout)
+ 
+ 	expire = timeout + jiffies;
+ 
++	if (timeout == 1 && hrtimer_resolution < NSEC_PER_SEC / HZ) {
++		/*
++		 * Special case 1 as being a request for the minimum timeout
++		 * and use highres timers to timeout after 1ms to workaround
++		 * the granularity of low Hz tick timers.
++		 */
++		if (!schedule_min_hrtimeout())
++			return 0;
++		goto out_timeout;
++	}
++
+ 	setup_timer_on_stack(&timer, process_timeout, (unsigned long)current);
+ 	__mod_timer(&timer, expire, false);
+ 	schedule();
+@@ -1757,10 +1768,10 @@ signed long __sched schedule_timeout(signed long timeout)
+ 
+ 	/* Remove the timer from the object tracker */
+ 	destroy_timer_on_stack(&timer);
+-
++out_timeout:
+ 	timeout = expire - jiffies;
+ 
+- out:
++out:
+ 	return timeout < 0 ? 0 : timeout;
+ }
+ EXPORT_SYMBOL(schedule_timeout);
+-- 
+2.9.3
+
diff --git a/patches/0011-Convert-msleep-to-use-hrtimers-when-active.patch b/patches/0011-Convert-msleep-to-use-hrtimers-when-active.patch
new file mode 100644
index 0000000..6eafa9a
--- /dev/null
+++ b/patches/0011-Convert-msleep-to-use-hrtimers-when-active.patch
@@ -0,0 +1,54 @@
+From 534bc9d3e559420eaf57771f48d2c2f549dcc4d2 Mon Sep 17 00:00:00 2001
+From: Con Kolivas <kernel@kolivas.org>
+Date: Fri, 4 Nov 2016 09:25:54 +1100
+Subject: [PATCH 11/25] Convert msleep to use hrtimers when active.
+
+---
+ kernel/time/timer.c | 24 ++++++++++++++++++++++--
+ 1 file changed, 22 insertions(+), 2 deletions(-)
+
+diff --git a/kernel/time/timer.c b/kernel/time/timer.c
+index 3f72c13..bc53598 100644
+--- a/kernel/time/timer.c
++++ b/kernel/time/timer.c
+@@ -1892,7 +1892,19 @@ void __init init_timers(void)
+  */
+ void msleep(unsigned int msecs)
+ {
+-	unsigned long timeout = msecs_to_jiffies(msecs) + 1;
++	int jiffs = msecs_to_jiffies(msecs);
++	unsigned long timeout;
++
++	/*
++	 * Use high resolution timers where the resolution of tick based
++	 * timers is inadequate.
++	 */
++	if (jiffs < 5 && hrtimer_resolution < NSEC_PER_SEC / HZ) {
++		while (msecs)
++			msecs = schedule_msec_hrtimeout_uninterruptible(msecs);
++		return;
++	}
++	timeout = msecs_to_jiffies(msecs) + 1;
+ 
+ 	while (timeout)
+ 		timeout = schedule_timeout_uninterruptible(timeout);
+@@ -1906,7 +1918,15 @@ EXPORT_SYMBOL(msleep);
+  */
+ unsigned long msleep_interruptible(unsigned int msecs)
+ {
+-	unsigned long timeout = msecs_to_jiffies(msecs) + 1;
++	int jiffs = msecs_to_jiffies(msecs);
++	unsigned long timeout;
++
++	if (jiffs < 5 && hrtimer_resolution < NSEC_PER_SEC / HZ) {
++		while (msecs && !signal_pending(current))
++			msecs = schedule_msec_hrtimeout_interruptible(msecs);
++		return msecs;
++	}
++	timeout = msecs_to_jiffies(msecs) + 1;
+ 
+ 	while (timeout && !signal_pending(current))
+ 		timeout = schedule_timeout_interruptible(timeout);
+-- 
+2.9.3
+
diff --git a/patches/0012-Replace-all-schedule-timeout-1-with-schedule_min_hrt.patch b/patches/0012-Replace-all-schedule-timeout-1-with-schedule_min_hrt.patch
new file mode 100644
index 0000000..14be4c1
--- /dev/null
+++ b/patches/0012-Replace-all-schedule-timeout-1-with-schedule_min_hrt.patch
@@ -0,0 +1,226 @@
+From 8fef7b75352d874af02881de3493f2ce2d47a341 Mon Sep 17 00:00:00 2001
+From: Con Kolivas <kernel@kolivas.org>
+Date: Mon, 20 Feb 2017 13:28:30 +1100
+Subject: [PATCH 12/25] Replace all schedule timeout(1) with
+ schedule_min_hrtimeout()
+
+---
+ drivers/block/swim.c                           | 6 +++---
+ drivers/char/ipmi/ipmi_msghandler.c            | 2 +-
+ drivers/char/ipmi/ipmi_ssif.c                  | 2 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c           | 2 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_irq.c            | 2 +-
+ drivers/mfd/ucb1x00-core.c                     | 2 +-
+ drivers/misc/sgi-xp/xpc_channel.c              | 2 +-
+ drivers/net/caif/caif_hsi.c                    | 2 +-
+ drivers/ntb/test/ntb_perf.c                    | 2 +-
+ drivers/staging/comedi/drivers/ni_mio_common.c | 2 +-
+ fs/afs/vlocation.c                             | 2 +-
+ fs/btrfs/extent-tree.c                         | 2 +-
+ fs/btrfs/inode-map.c                           | 2 +-
+ sound/usb/line6/pcm.c                          | 2 +-
+ 14 files changed, 16 insertions(+), 16 deletions(-)
+
+diff --git a/drivers/block/swim.c b/drivers/block/swim.c
+index b5afd49..7d09955 100644
+--- a/drivers/block/swim.c
++++ b/drivers/block/swim.c
+@@ -332,7 +332,7 @@ static inline void swim_motor(struct swim __iomem *base,
+ 			if (swim_readbit(base, MOTOR_ON))
+ 				break;
+ 			current->state = TASK_INTERRUPTIBLE;
+-			schedule_timeout(1);
++			schedule_min_hrtimeout();
+ 		}
+ 	} else if (action == OFF) {
+ 		swim_action(base, MOTOR_OFF);
+@@ -351,7 +351,7 @@ static inline void swim_eject(struct swim __iomem *base)
+ 		if (!swim_readbit(base, DISK_IN))
+ 			break;
+ 		current->state = TASK_INTERRUPTIBLE;
+-		schedule_timeout(1);
++		schedule_min_hrtimeout();
+ 	}
+ 	swim_select(base, RELAX);
+ }
+@@ -375,7 +375,7 @@ static inline int swim_step(struct swim __iomem *base)
+ 	for (wait = 0; wait < HZ; wait++) {
+ 
+ 		current->state = TASK_INTERRUPTIBLE;
+-		schedule_timeout(1);
++		schedule_min_hrtimeout();
+ 
+ 		swim_select(base, RELAX);
+ 		if (!swim_readbit(base, STEP))
+diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c
+index 92e53ac..a2418e7 100644
+--- a/drivers/char/ipmi/ipmi_msghandler.c
++++ b/drivers/char/ipmi/ipmi_msghandler.c
+@@ -2953,7 +2953,7 @@ static void cleanup_smi_msgs(ipmi_smi_t intf)
+ 	/* Current message first, to preserve order */
+ 	while (intf->curr_msg && !list_empty(&intf->waiting_rcv_msgs)) {
+ 		/* Wait for the message to clear out. */
+-		schedule_timeout(1);
++		schedule_min_hrtimeout();
+ 	}
+ 
+ 	/* No need for locks, the interface is down. */
+diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c
+index cca6e5b..fd3c7da 100644
+--- a/drivers/char/ipmi/ipmi_ssif.c
++++ b/drivers/char/ipmi/ipmi_ssif.c
+@@ -1185,7 +1185,7 @@ static int ssif_remove(struct i2c_client *client)
+ 
+ 	/* make sure the driver is not looking for flags any more. */
+ 	while (ssif_info->ssif_state != SSIF_NORMAL)
+-		schedule_timeout(1);
++		schedule_min_hrtimeout();
+ 
+ 	ssif_info->stopping = true;
+ 	del_timer_sync(&ssif_info->retry_timer);
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c
+index b6a0806..b5b02cf 100644
+--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c
++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c
+@@ -235,7 +235,7 @@ static int vmw_fifo_wait_noirq(struct vmw_private *dev_priv,
+ 			DRM_ERROR("SVGA device lockup.\n");
+ 			break;
+ 		}
+-		schedule_timeout(1);
++		schedule_min_hrtimeout();
+ 		if (interruptible && signal_pending(current)) {
+ 			ret = -ERESTARTSYS;
+ 			break;
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c b/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c
+index 0c7e172..4c1555c 100644
+--- a/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c
++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c
+@@ -156,7 +156,7 @@ int vmw_fallback_wait(struct vmw_private *dev_priv,
+ 			break;
+ 		}
+ 		if (lazy)
+-			schedule_timeout(1);
++			schedule_min_hrtimeout();
+ 		else if ((++count & 0x0F) == 0) {
+ 			/**
+ 			 * FIXME: Use schedule_hr_timeout here for
+diff --git a/drivers/mfd/ucb1x00-core.c b/drivers/mfd/ucb1x00-core.c
+index d6fb2e1..7ac951b 100644
+--- a/drivers/mfd/ucb1x00-core.c
++++ b/drivers/mfd/ucb1x00-core.c
+@@ -253,7 +253,7 @@ unsigned int ucb1x00_adc_read(struct ucb1x00 *ucb, int adc_channel, int sync)
+ 			break;
+ 		/* yield to other processes */
+ 		set_current_state(TASK_INTERRUPTIBLE);
+-		schedule_timeout(1);
++		schedule_min_hrtimeout();
+ 	}
+ 
+ 	return UCB_ADC_DAT(val);
+diff --git a/drivers/misc/sgi-xp/xpc_channel.c b/drivers/misc/sgi-xp/xpc_channel.c
+index 128d561..38e68e9 100644
+--- a/drivers/misc/sgi-xp/xpc_channel.c
++++ b/drivers/misc/sgi-xp/xpc_channel.c
+@@ -837,7 +837,7 @@ xpc_allocate_msg_wait(struct xpc_channel *ch)
+ 
+ 	atomic_inc(&ch->n_on_msg_allocate_wq);
+ 	prepare_to_wait(&ch->msg_allocate_wq, &wait, TASK_INTERRUPTIBLE);
+-	ret = schedule_timeout(1);
++	ret = schedule_min_hrtimeout();
+ 	finish_wait(&ch->msg_allocate_wq, &wait);
+ 	atomic_dec(&ch->n_on_msg_allocate_wq);
+ 
+diff --git a/drivers/net/caif/caif_hsi.c b/drivers/net/caif/caif_hsi.c
+index ddabce7..67fb5ce 100644
+--- a/drivers/net/caif/caif_hsi.c
++++ b/drivers/net/caif/caif_hsi.c
+@@ -944,7 +944,7 @@ static void cfhsi_wake_down(struct work_struct *work)
+ 			break;
+ 
+ 		set_current_state(TASK_INTERRUPTIBLE);
+-		schedule_timeout(1);
++		schedule_min_hrtimeout();
+ 		retry--;
+ 	}
+ 
+diff --git a/drivers/ntb/test/ntb_perf.c b/drivers/ntb/test/ntb_perf.c
+index 434e1d4..2f9543b 100644
+--- a/drivers/ntb/test/ntb_perf.c
++++ b/drivers/ntb/test/ntb_perf.c
+@@ -308,7 +308,7 @@ static int perf_move_data(struct pthr_ctx *pctx, char __iomem *dst, char *src,
+ 		if (unlikely((jiffies - last_sleep) > 5 * HZ)) {
+ 			last_sleep = jiffies;
+ 			set_current_state(TASK_INTERRUPTIBLE);
+-			schedule_timeout(1);
++			schedule_min_hrtimeout();
+ 		}
+ 
+ 		if (unlikely(kthread_should_stop()))
+diff --git a/drivers/staging/comedi/drivers/ni_mio_common.c b/drivers/staging/comedi/drivers/ni_mio_common.c
+index b2e3828..beae38b 100644
+--- a/drivers/staging/comedi/drivers/ni_mio_common.c
++++ b/drivers/staging/comedi/drivers/ni_mio_common.c
+@@ -4655,7 +4655,7 @@ static int cs5529_wait_for_idle(struct comedi_device *dev)
+ 		if ((status & NI67XX_CAL_STATUS_BUSY) == 0)
+ 			break;
+ 		set_current_state(TASK_INTERRUPTIBLE);
+-		if (schedule_timeout(1))
++		if (schedule_min_hrtimeout())
+ 			return -EIO;
+ 	}
+ 	if (i == timeout) {
+diff --git a/fs/afs/vlocation.c b/fs/afs/vlocation.c
+index 45a8639..855d08e 100644
+--- a/fs/afs/vlocation.c
++++ b/fs/afs/vlocation.c
+@@ -129,7 +129,7 @@ static int afs_vlocation_access_vl_by_id(struct afs_vlocation *vl,
+ 				if (vl->upd_busy_cnt > 1) {
+ 					/* second+ BUSY - sleep a little bit */
+ 					set_current_state(TASK_UNINTERRUPTIBLE);
+-					schedule_timeout(1);
++					schedule_min_hrtimeout();
+ 				}
+ 				continue;
+ 			}
+diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c
+index dcd2e79..16bf891 100644
+--- a/fs/btrfs/extent-tree.c
++++ b/fs/btrfs/extent-tree.c
+@@ -5952,7 +5952,7 @@ int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes)
+ 
+ 	if (flush != BTRFS_RESERVE_NO_FLUSH &&
+ 	    btrfs_transaction_in_commit(fs_info))
+-		schedule_timeout(1);
++		schedule_min_hrtimeout();
+ 
+ 	if (delalloc_lock)
+ 		mutex_lock(&BTRFS_I(inode)->delalloc_mutex);
+diff --git a/fs/btrfs/inode-map.c b/fs/btrfs/inode-map.c
+index 144b119..03d2e8e 100644
+--- a/fs/btrfs/inode-map.c
++++ b/fs/btrfs/inode-map.c
+@@ -89,7 +89,7 @@ static int caching_kthread(void *data)
+ 				btrfs_release_path(path);
+ 				root->ino_cache_progress = last;
+ 				up_read(&fs_info->commit_root_sem);
+-				schedule_timeout(1);
++				schedule_min_hrtimeout();
+ 				goto again;
+ 			} else
+ 				continue;
+diff --git a/sound/usb/line6/pcm.c b/sound/usb/line6/pcm.c
+index fab53f5..fda1ab5 100644
+--- a/sound/usb/line6/pcm.c
++++ b/sound/usb/line6/pcm.c
+@@ -131,7 +131,7 @@ static void line6_wait_clear_audio_urbs(struct snd_line6_pcm *line6pcm,
+ 		if (!alive)
+ 			break;
+ 		set_current_state(TASK_UNINTERRUPTIBLE);
+-		schedule_timeout(1);
++		schedule_min_hrtimeout();
+ 	} while (--timeout > 0);
+ 	if (alive)
+ 		dev_err(line6pcm->line6->ifcdev,
+-- 
+2.9.3
+
diff --git a/patches/0013-Change-all-schedule_timeout-with-msecs_to_jiffies-po.patch b/patches/0013-Change-all-schedule_timeout-with-msecs_to_jiffies-po.patch
new file mode 100644
index 0000000..78e0322
--- /dev/null
+++ b/patches/0013-Change-all-schedule_timeout-with-msecs_to_jiffies-po.patch
@@ -0,0 +1,397 @@
+From 56e8b01452fbb6c1aa85b0a52fbd352fddf7e959 Mon Sep 17 00:00:00 2001
+From: Con Kolivas <kernel@kolivas.org>
+Date: Mon, 20 Feb 2017 13:29:16 +1100
+Subject: [PATCH 13/25] Change all schedule_timeout with msecs_to_jiffies
+ potentially under 50ms to use schedule_msec_hrtimeout.
+
+---
+ drivers/bluetooth/hci_qca.c                     | 2 +-
+ drivers/char/snsc.c                             | 4 ++--
+ drivers/media/pci/ivtv/ivtv-ioctl.c             | 2 +-
+ drivers/media/pci/ivtv/ivtv-streams.c           | 2 +-
+ drivers/net/can/usb/peak_usb/pcan_usb.c         | 2 +-
+ drivers/net/usb/lan78xx.c                       | 2 +-
+ drivers/net/usb/usbnet.c                        | 2 +-
+ drivers/scsi/fnic/fnic_scsi.c                   | 4 ++--
+ drivers/scsi/snic/snic_scsi.c                   | 2 +-
+ drivers/staging/lustre/lnet/lnet/lib-eq.c       | 2 +-
+ drivers/staging/rts5208/rtsx.c                  | 2 +-
+ drivers/staging/speakup/speakup_acntpc.c        | 4 ++--
+ drivers/staging/speakup/speakup_apollo.c        | 2 +-
+ drivers/staging/speakup/speakup_decext.c        | 2 +-
+ drivers/staging/speakup/speakup_decpc.c         | 2 +-
+ drivers/staging/speakup/speakup_dectlk.c        | 2 +-
+ drivers/staging/speakup/speakup_dtlk.c          | 4 ++--
+ drivers/staging/speakup/speakup_keypc.c         | 4 ++--
+ drivers/staging/speakup/synth.c                 | 2 +-
+ drivers/staging/unisys/visornic/visornic_main.c | 6 +++---
+ drivers/target/target_core_user.c               | 2 +-
+ drivers/video/fbdev/omap/hwa742.c               | 2 +-
+ drivers/video/fbdev/pxafb.c                     | 2 +-
+ 23 files changed, 30 insertions(+), 30 deletions(-)
+
+diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c
+index 05c2307..6954d29 100644
+--- a/drivers/bluetooth/hci_qca.c
++++ b/drivers/bluetooth/hci_qca.c
+@@ -880,7 +880,7 @@ static int qca_set_baudrate(struct hci_dev *hdev, uint8_t baudrate)
+ 	 * then host can communicate with new baudrate to controller
+ 	 */
+ 	set_current_state(TASK_UNINTERRUPTIBLE);
+-	schedule_timeout(msecs_to_jiffies(BAUDRATE_SETTLE_TIMEOUT_MS));
++	schedule_msec_hrtimeout((BAUDRATE_SETTLE_TIMEOUT_MS));
+ 	set_current_state(TASK_INTERRUPTIBLE);
+ 
+ 	return 0;
+diff --git a/drivers/char/snsc.c b/drivers/char/snsc.c
+index ec07f0e..3410b46 100644
+--- a/drivers/char/snsc.c
++++ b/drivers/char/snsc.c
+@@ -198,7 +198,7 @@ scdrv_read(struct file *file, char __user *buf, size_t count, loff_t *f_pos)
+ 		add_wait_queue(&sd->sd_rq, &wait);
+ 		spin_unlock_irqrestore(&sd->sd_rlock, flags);
+ 
+-		schedule_timeout(msecs_to_jiffies(SCDRV_TIMEOUT));
++		schedule_msec_hrtimeout((SCDRV_TIMEOUT));
+ 
+ 		remove_wait_queue(&sd->sd_rq, &wait);
+ 		if (signal_pending(current)) {
+@@ -294,7 +294,7 @@ scdrv_write(struct file *file, const char __user *buf,
+ 		add_wait_queue(&sd->sd_wq, &wait);
+ 		spin_unlock_irqrestore(&sd->sd_wlock, flags);
+ 
+-		schedule_timeout(msecs_to_jiffies(SCDRV_TIMEOUT));
++		schedule_msec_hrtimeout((SCDRV_TIMEOUT));
+ 
+ 		remove_wait_queue(&sd->sd_wq, &wait);
+ 		if (signal_pending(current)) {
+diff --git a/drivers/media/pci/ivtv/ivtv-ioctl.c b/drivers/media/pci/ivtv/ivtv-ioctl.c
+index 2dc4b20..8e061cf 100644
+--- a/drivers/media/pci/ivtv/ivtv-ioctl.c
++++ b/drivers/media/pci/ivtv/ivtv-ioctl.c
+@@ -1151,7 +1151,7 @@ void ivtv_s_std_dec(struct ivtv *itv, v4l2_std_id std)
+ 				TASK_UNINTERRUPTIBLE);
+ 		if ((read_reg(IVTV_REG_DEC_LINE_FIELD) >> 16) < 100)
+ 			break;
+-		schedule_timeout(msecs_to_jiffies(25));
++		schedule_msec_hrtimeout((25));
+ 	}
+ 	finish_wait(&itv->vsync_waitq, &wait);
+ 	mutex_lock(&itv->serialize_lock);
+diff --git a/drivers/media/pci/ivtv/ivtv-streams.c b/drivers/media/pci/ivtv/ivtv-streams.c
+index d27c6df..e9ffc4e 100644
+--- a/drivers/media/pci/ivtv/ivtv-streams.c
++++ b/drivers/media/pci/ivtv/ivtv-streams.c
+@@ -834,7 +834,7 @@ int ivtv_stop_v4l2_encode_stream(struct ivtv_stream *s, int gop_end)
+ 			while (!test_bit(IVTV_F_I_EOS, &itv->i_flags) &&
+ 				time_before(jiffies,
+ 					    then + msecs_to_jiffies(2000))) {
+-				schedule_timeout(msecs_to_jiffies(10));
++				schedule_msec_hrtimeout((10));
+ 			}
+ 
+ 			/* To convert jiffies to ms, we must multiply by 1000
+diff --git a/drivers/net/can/usb/peak_usb/pcan_usb.c b/drivers/net/can/usb/peak_usb/pcan_usb.c
+index 838545c..34f8972 100644
+--- a/drivers/net/can/usb/peak_usb/pcan_usb.c
++++ b/drivers/net/can/usb/peak_usb/pcan_usb.c
+@@ -250,7 +250,7 @@ static int pcan_usb_write_mode(struct peak_usb_device *dev, u8 onoff)
+ 	} else {
+ 		/* the PCAN-USB needs time to init */
+ 		set_current_state(TASK_INTERRUPTIBLE);
+-		schedule_timeout(msecs_to_jiffies(PCAN_USB_STARTUP_TIMEOUT));
++		schedule_msec_hrtimeout((PCAN_USB_STARTUP_TIMEOUT));
+ 	}
+ 
+ 	return err;
+diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c
+index 08f8703..3b3bc86 100644
+--- a/drivers/net/usb/lan78xx.c
++++ b/drivers/net/usb/lan78xx.c
+@@ -2544,7 +2544,7 @@ static void lan78xx_terminate_urbs(struct lan78xx_net *dev)
+ 	while (!skb_queue_empty(&dev->rxq) &&
+ 	       !skb_queue_empty(&dev->txq) &&
+ 	       !skb_queue_empty(&dev->done)) {
+-		schedule_timeout(msecs_to_jiffies(UNLINK_TIMEOUT_MS));
++		schedule_msec_hrtimeout((UNLINK_TIMEOUT_MS));
+ 		set_current_state(TASK_UNINTERRUPTIBLE);
+ 		netif_dbg(dev, ifdown, dev->net,
+ 			  "waited for %d urb completions\n", temp);
+diff --git a/drivers/net/usb/usbnet.c b/drivers/net/usb/usbnet.c
+index 3de65ea..f8a4b18 100644
+--- a/drivers/net/usb/usbnet.c
++++ b/drivers/net/usb/usbnet.c
+@@ -767,7 +767,7 @@ static void wait_skb_queue_empty(struct sk_buff_head *q)
+ 	spin_lock_irqsave(&q->lock, flags);
+ 	while (!skb_queue_empty(q)) {
+ 		spin_unlock_irqrestore(&q->lock, flags);
+-		schedule_timeout(msecs_to_jiffies(UNLINK_TIMEOUT_MS));
++		schedule_msec_hrtimeout((UNLINK_TIMEOUT_MS));
+ 		set_current_state(TASK_UNINTERRUPTIBLE);
+ 		spin_lock_irqsave(&q->lock, flags);
+ 	}
+diff --git a/drivers/scsi/fnic/fnic_scsi.c b/drivers/scsi/fnic/fnic_scsi.c
+index adb3d58..de73e78 100644
+--- a/drivers/scsi/fnic/fnic_scsi.c
++++ b/drivers/scsi/fnic/fnic_scsi.c
+@@ -217,7 +217,7 @@ int fnic_fw_reset_handler(struct fnic *fnic)
+ 
+ 	/* wait for io cmpl */
+ 	while (atomic_read(&fnic->in_flight))
+-		schedule_timeout(msecs_to_jiffies(1));
++		schedule_msec_hrtimeout((1));
+ 
+ 	spin_lock_irqsave(&fnic->wq_copy_lock[0], flags);
+ 
+@@ -2201,7 +2201,7 @@ static int fnic_clean_pending_aborts(struct fnic *fnic,
+ 		}
+ 	}
+ 
+-	schedule_timeout(msecs_to_jiffies(2 * fnic->config.ed_tov));
++	schedule_msec_hrtimeout((2 * fnic->config.ed_tov));
+ 
+ 	/* walk again to check, if IOs are still pending in fw */
+ 	if (fnic_is_abts_pending(fnic, lr_sc))
+diff --git a/drivers/scsi/snic/snic_scsi.c b/drivers/scsi/snic/snic_scsi.c
+index abada16..0bf30dc 100644
+--- a/drivers/scsi/snic/snic_scsi.c
++++ b/drivers/scsi/snic/snic_scsi.c
+@@ -2356,7 +2356,7 @@ snic_reset(struct Scsi_Host *shost, struct scsi_cmnd *sc)
+ 
+ 	/* Wait for all the IOs that are entered in Qcmd */
+ 	while (atomic_read(&snic->ios_inflight))
+-		schedule_timeout(msecs_to_jiffies(1));
++		schedule_msec_hrtimeout((1));
+ 
+ 	ret = snic_issue_hba_reset(snic, sc);
+ 	if (ret) {
+diff --git a/drivers/staging/lustre/lnet/lnet/lib-eq.c b/drivers/staging/lustre/lnet/lnet/lib-eq.c
+index d05c6cc..3f62b6f 100644
+--- a/drivers/staging/lustre/lnet/lnet/lib-eq.c
++++ b/drivers/staging/lustre/lnet/lnet/lib-eq.c
+@@ -328,7 +328,7 @@ __must_hold(&the_lnet.ln_eq_wait_lock)
+ 		schedule();
+ 	} else {
+ 		now = jiffies;
+-		schedule_timeout(msecs_to_jiffies(tms));
++		schedule_msec_hrtimeout((tms));
+ 		tms -= jiffies_to_msecs(jiffies - now);
+ 		if (tms < 0) /* no more wait but may have new event */
+ 			tms = 0;
+diff --git a/drivers/staging/rts5208/rtsx.c b/drivers/staging/rts5208/rtsx.c
+index 68d75d0..aef88c4 100644
+--- a/drivers/staging/rts5208/rtsx.c
++++ b/drivers/staging/rts5208/rtsx.c
+@@ -537,7 +537,7 @@ static int rtsx_polling_thread(void *__dev)
+ 
+ 	for (;;) {
+ 		set_current_state(TASK_INTERRUPTIBLE);
+-		schedule_timeout(msecs_to_jiffies(POLLING_INTERVAL));
++		schedule_msec_hrtimeout((POLLING_INTERVAL));
+ 
+ 		/* lock the device pointers */
+ 		mutex_lock(&dev->dev_mutex);
+diff --git a/drivers/staging/speakup/speakup_acntpc.c b/drivers/staging/speakup/speakup_acntpc.c
+index efb791b..fd02fb2 100644
+--- a/drivers/staging/speakup/speakup_acntpc.c
++++ b/drivers/staging/speakup/speakup_acntpc.c
+@@ -204,7 +204,7 @@ static void do_catch_up(struct spk_synth *synth)
+ 		full_time_val = full_time->u.n.value;
+ 		spin_unlock_irqrestore(&speakup_info.spinlock, flags);
+ 		if (synth_full()) {
+-			schedule_timeout(msecs_to_jiffies(full_time_val));
++			schedule_msec_hrtimeout((full_time_val));
+ 			continue;
+ 		}
+ 		set_current_state(TASK_RUNNING);
+@@ -232,7 +232,7 @@ static void do_catch_up(struct spk_synth *synth)
+ 			jiffy_delta_val = jiffy_delta->u.n.value;
+ 			delay_time_val = delay_time->u.n.value;
+ 			spin_unlock_irqrestore(&speakup_info.spinlock, flags);
+-			schedule_timeout(msecs_to_jiffies(delay_time_val));
++			schedule_msec_hrtimeout((delay_time_val));
+ 			jiff_max = jiffies+jiffy_delta_val;
+ 		}
+ 	}
+diff --git a/drivers/staging/speakup/speakup_apollo.c b/drivers/staging/speakup/speakup_apollo.c
+index 3cbc8a7..3c17854 100644
+--- a/drivers/staging/speakup/speakup_apollo.c
++++ b/drivers/staging/speakup/speakup_apollo.c
+@@ -172,7 +172,7 @@ static void do_catch_up(struct spk_synth *synth)
+ 			outb(UART_MCR_DTR, speakup_info.port_tts + UART_MCR);
+ 			outb(UART_MCR_DTR | UART_MCR_RTS,
+ 					speakup_info.port_tts + UART_MCR);
+-			schedule_timeout(msecs_to_jiffies(full_time_val));
++			schedule_msec_hrtimeout((full_time_val));
+ 			continue;
+ 		}
+ 		if (time_after_eq(jiffies, jiff_max) && (ch == SPACE)) {
+diff --git a/drivers/staging/speakup/speakup_decext.c b/drivers/staging/speakup/speakup_decext.c
+index 1a5cf3d..fa2b4e1 100644
+--- a/drivers/staging/speakup/speakup_decext.c
++++ b/drivers/staging/speakup/speakup_decext.c
+@@ -186,7 +186,7 @@ static void do_catch_up(struct spk_synth *synth)
+ 		if (ch == '\n')
+ 			ch = 0x0D;
+ 		if (synth_full() || !spk_serial_out(ch)) {
+-			schedule_timeout(msecs_to_jiffies(delay_time_val));
++			schedule_msec_hrtimeout((delay_time_val));
+ 			continue;
+ 		}
+ 		set_current_state(TASK_RUNNING);
+diff --git a/drivers/staging/speakup/speakup_decpc.c b/drivers/staging/speakup/speakup_decpc.c
+index d6479bd..f7554bf 100644
+--- a/drivers/staging/speakup/speakup_decpc.c
++++ b/drivers/staging/speakup/speakup_decpc.c
+@@ -403,7 +403,7 @@ static void do_catch_up(struct spk_synth *synth)
+ 		if (ch == '\n')
+ 			ch = 0x0D;
+ 		if (dt_sendchar(ch)) {
+-			schedule_timeout(msecs_to_jiffies(delay_time_val));
++			schedule_msec_hrtimeout((delay_time_val));
+ 			continue;
+ 		}
+ 		set_current_state(TASK_RUNNING);
+diff --git a/drivers/staging/speakup/speakup_dectlk.c b/drivers/staging/speakup/speakup_dectlk.c
+index 7646567..639192e 100644
+--- a/drivers/staging/speakup/speakup_dectlk.c
++++ b/drivers/staging/speakup/speakup_dectlk.c
+@@ -251,7 +251,7 @@ static void do_catch_up(struct spk_synth *synth)
+ 		if (ch == '\n')
+ 			ch = 0x0D;
+ 		if (synth_full_val || !spk_serial_out(ch)) {
+-			schedule_timeout(msecs_to_jiffies(delay_time_val));
++			schedule_msec_hrtimeout((delay_time_val));
+ 			continue;
+ 		}
+ 		set_current_state(TASK_RUNNING);
+diff --git a/drivers/staging/speakup/speakup_dtlk.c b/drivers/staging/speakup/speakup_dtlk.c
+index 38aa401..1640519 100644
+--- a/drivers/staging/speakup/speakup_dtlk.c
++++ b/drivers/staging/speakup/speakup_dtlk.c
+@@ -217,7 +217,7 @@ static void do_catch_up(struct spk_synth *synth)
+ 		delay_time_val = delay_time->u.n.value;
+ 		spin_unlock_irqrestore(&speakup_info.spinlock, flags);
+ 		if (synth_full()) {
+-			schedule_timeout(msecs_to_jiffies(delay_time_val));
++			schedule_msec_hrtimeout((delay_time_val));
+ 			continue;
+ 		}
+ 		set_current_state(TASK_RUNNING);
+@@ -233,7 +233,7 @@ static void do_catch_up(struct spk_synth *synth)
+ 			delay_time_val = delay_time->u.n.value;
+ 			jiffy_delta_val = jiffy_delta->u.n.value;
+ 			spin_unlock_irqrestore(&speakup_info.spinlock, flags);
+-			schedule_timeout(msecs_to_jiffies(delay_time_val));
++			schedule_msec_hrtimeout((delay_time_val));
+ 			jiff_max = jiffies + jiffy_delta_val;
+ 		}
+ 	}
+diff --git a/drivers/staging/speakup/speakup_keypc.c b/drivers/staging/speakup/speakup_keypc.c
+index 5e2170b..30b5df7 100644
+--- a/drivers/staging/speakup/speakup_keypc.c
++++ b/drivers/staging/speakup/speakup_keypc.c
+@@ -206,7 +206,7 @@ spin_lock_irqsave(&speakup_info.spinlock, flags);
+ 		full_time_val = full_time->u.n.value;
+ 		spin_unlock_irqrestore(&speakup_info.spinlock, flags);
+ 		if (synth_full()) {
+-			schedule_timeout(msecs_to_jiffies(full_time_val));
++			schedule_msec_hrtimeout((full_time_val));
+ 			continue;
+ 		}
+ 		set_current_state(TASK_RUNNING);
+@@ -239,7 +239,7 @@ spin_lock_irqsave(&speakup_info.spinlock, flags);
+ 			jiffy_delta_val = jiffy_delta->u.n.value;
+ 			delay_time_val = delay_time->u.n.value;
+ 			spin_unlock_irqrestore(&speakup_info.spinlock, flags);
+-			schedule_timeout(msecs_to_jiffies(delay_time_val));
++			schedule_msec_hrtimeout((delay_time_val));
+ 			jiff_max = jiffies+jiffy_delta_val;
+ 		}
+ 	}
+diff --git a/drivers/staging/speakup/synth.c b/drivers/staging/speakup/synth.c
+index a61c02b..14299e5 100644
+--- a/drivers/staging/speakup/synth.c
++++ b/drivers/staging/speakup/synth.c
+@@ -120,7 +120,7 @@ void spk_do_catch_up(struct spk_synth *synth)
+ 		if (ch == '\n')
+ 			ch = synth->procspeech;
+ 		if (!spk_serial_out(ch)) {
+-			schedule_timeout(msecs_to_jiffies(full_time_val));
++			schedule_msec_hrtimeout((full_time_val));
+ 			continue;
+ 		}
+ 		if (time_after_eq(jiffies, jiff_max) && (ch == SPACE)) {
+diff --git a/drivers/staging/unisys/visornic/visornic_main.c b/drivers/staging/unisys/visornic/visornic_main.c
+index c1f674f..4f30a7a 100644
+--- a/drivers/staging/unisys/visornic/visornic_main.c
++++ b/drivers/staging/unisys/visornic/visornic_main.c
+@@ -468,7 +468,7 @@ visornic_disable_with_timeout(struct net_device *netdev, const int timeout)
+ 		}
+ 		set_current_state(TASK_INTERRUPTIBLE);
+ 		spin_unlock_irqrestore(&devdata->priv_lock, flags);
+-		wait += schedule_timeout(msecs_to_jiffies(10));
++		wait += schedule_msec_hrtimeout((10));
+ 		spin_lock_irqsave(&devdata->priv_lock, flags);
+ 	}
+ 
+@@ -479,7 +479,7 @@ visornic_disable_with_timeout(struct net_device *netdev, const int timeout)
+ 		while (1) {
+ 			set_current_state(TASK_INTERRUPTIBLE);
+ 			spin_unlock_irqrestore(&devdata->priv_lock, flags);
+-			schedule_timeout(msecs_to_jiffies(10));
++			schedule_msec_hrtimeout((10));
+ 			spin_lock_irqsave(&devdata->priv_lock, flags);
+ 			if (atomic_read(&devdata->usage))
+ 				break;
+@@ -611,7 +611,7 @@ visornic_enable_with_timeout(struct net_device *netdev, const int timeout)
+ 		}
+ 		set_current_state(TASK_INTERRUPTIBLE);
+ 		spin_unlock_irqrestore(&devdata->priv_lock, flags);
+-		wait += schedule_timeout(msecs_to_jiffies(10));
++		wait += schedule_msec_hrtimeout((10));
+ 		spin_lock_irqsave(&devdata->priv_lock, flags);
+ 	}
+ 
+diff --git a/drivers/target/target_core_user.c b/drivers/target/target_core_user.c
+index 8041710..f907a81 100644
+--- a/drivers/target/target_core_user.c
++++ b/drivers/target/target_core_user.c
+@@ -451,7 +451,7 @@ tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd)
+ 
+ 		pr_debug("sleeping for ring space\n");
+ 		spin_unlock_irq(&udev->cmdr_lock);
+-		ret = schedule_timeout(msecs_to_jiffies(TCMU_TIME_OUT));
++		ret = schedule_msec_hrtimeout((TCMU_TIME_OUT));
+ 		finish_wait(&udev->wait_cmdr, &__wait);
+ 		if (!ret) {
+ 			pr_warn("tcmu: command timed out\n");
+diff --git a/drivers/video/fbdev/omap/hwa742.c b/drivers/video/fbdev/omap/hwa742.c
+index a4ee65b..cf38bcb 100644
+--- a/drivers/video/fbdev/omap/hwa742.c
++++ b/drivers/video/fbdev/omap/hwa742.c
+@@ -926,7 +926,7 @@ static void hwa742_resume(void)
+ 		if (hwa742_read_reg(HWA742_PLL_DIV_REG) & (1 << 7))
+ 			break;
+ 		set_current_state(TASK_UNINTERRUPTIBLE);
+-		schedule_timeout(msecs_to_jiffies(5));
++		schedule_msec_hrtimeout((5));
+ 	}
+ 	hwa742_set_update_mode(hwa742.update_mode_before_suspend);
+ }
+diff --git a/drivers/video/fbdev/pxafb.c b/drivers/video/fbdev/pxafb.c
+index ef73f14..7b5483b 100644
+--- a/drivers/video/fbdev/pxafb.c
++++ b/drivers/video/fbdev/pxafb.c
+@@ -1287,7 +1287,7 @@ static int pxafb_smart_thread(void *arg)
+ 		mutex_unlock(&fbi->ctrlr_lock);
+ 
+ 		set_current_state(TASK_INTERRUPTIBLE);
+-		schedule_timeout(msecs_to_jiffies(30));
++		schedule_msec_hrtimeout((30));
+ 	}
+ 
+ 	pr_debug("%s(): task ending\n", __func__);
+-- 
+2.9.3
+
diff --git a/patches/0014-Replace-all-calls-to-schedule_timeout_interruptible-.patch b/patches/0014-Replace-all-calls-to-schedule_timeout_interruptible-.patch
new file mode 100644
index 0000000..2159cb1
--- /dev/null
+++ b/patches/0014-Replace-all-calls-to-schedule_timeout_interruptible-.patch
@@ -0,0 +1,325 @@
+From af94b59651831b7e176ce8cb98441bdccb87eac0 Mon Sep 17 00:00:00 2001
+From: Con Kolivas <kernel@kolivas.org>
+Date: Mon, 20 Feb 2017 13:30:07 +1100
+Subject: [PATCH 14/25] Replace all calls to schedule_timeout_interruptible of
+ potentially under 50ms to use schedule_msec_hrtimeout_interruptible.
+
+---
+ drivers/hwmon/fam15h_power.c        |  2 +-
+ drivers/iio/light/tsl2563.c         |  6 +-----
+ drivers/media/i2c/msp3400-driver.c  |  4 ++--
+ drivers/media/pci/ivtv/ivtv-gpio.c  |  6 +++---
+ drivers/media/radio/radio-mr800.c   |  2 +-
+ drivers/media/radio/radio-tea5777.c |  2 +-
+ drivers/media/radio/tea575x.c       |  2 +-
+ drivers/misc/panel.c                |  2 +-
+ drivers/parport/ieee1284.c          |  2 +-
+ drivers/parport/ieee1284_ops.c      |  2 +-
+ drivers/platform/x86/intel_ips.c    |  8 ++++----
+ net/core/pktgen.c                   |  2 +-
+ sound/soc/codecs/wm8350.c           | 12 ++++++------
+ sound/soc/codecs/wm8900.c           |  2 +-
+ sound/soc/codecs/wm9713.c           |  4 ++--
+ 15 files changed, 27 insertions(+), 31 deletions(-)
+
+diff --git a/drivers/hwmon/fam15h_power.c b/drivers/hwmon/fam15h_power.c
+index 15aa49d..991e8a7 100644
+--- a/drivers/hwmon/fam15h_power.c
++++ b/drivers/hwmon/fam15h_power.c
+@@ -238,7 +238,7 @@ static ssize_t acc_show_power(struct device *dev,
+ 		prev_ptsc[cu] = data->cpu_sw_pwr_ptsc[cu];
+ 	}
+ 
+-	leftover = schedule_timeout_interruptible(msecs_to_jiffies(data->power_period));
++	leftover = schedule_msec_hrtimeout_interruptible((data->power_period));
+ 	if (leftover)
+ 		return 0;
+ 
+diff --git a/drivers/iio/light/tsl2563.c b/drivers/iio/light/tsl2563.c
+index 04598ae..a8c095d 100644
+--- a/drivers/iio/light/tsl2563.c
++++ b/drivers/iio/light/tsl2563.c
+@@ -282,11 +282,7 @@ static void tsl2563_wait_adc(struct tsl2563_chip *chip)
+ 	default:
+ 		delay = 402;
+ 	}
+-	/*
+-	 * TODO: Make sure that we wait at least required delay but why we
+-	 * have to extend it one tick more?
+-	 */
+-	schedule_timeout_interruptible(msecs_to_jiffies(delay) + 2);
++	schedule_msec_hrtimeout_interruptible(delay + 1);
+ }
+ 
+ static int tsl2563_adjust_gainlevel(struct tsl2563_chip *chip, u16 adc)
+diff --git a/drivers/media/i2c/msp3400-driver.c b/drivers/media/i2c/msp3400-driver.c
+index 201a9800..5cebabc 100644
+--- a/drivers/media/i2c/msp3400-driver.c
++++ b/drivers/media/i2c/msp3400-driver.c
+@@ -184,7 +184,7 @@ static int msp_read(struct i2c_client *client, int dev, int addr)
+ 			break;
+ 		dev_warn(&client->dev, "I/O error #%d (read 0x%02x/0x%02x)\n", err,
+ 		       dev, addr);
+-		schedule_timeout_interruptible(msecs_to_jiffies(10));
++		schedule_msec_hrtimeout_interruptible((10));
+ 	}
+ 	if (err == 3) {
+ 		dev_warn(&client->dev, "resetting chip, sound will go off.\n");
+@@ -225,7 +225,7 @@ static int msp_write(struct i2c_client *client, int dev, int addr, int val)
+ 			break;
+ 		dev_warn(&client->dev, "I/O error #%d (write 0x%02x/0x%02x)\n", err,
+ 		       dev, addr);
+-		schedule_timeout_interruptible(msecs_to_jiffies(10));
++		schedule_msec_hrtimeout_interruptible((10));
+ 	}
+ 	if (err == 3) {
+ 		dev_warn(&client->dev, "resetting chip, sound will go off.\n");
+diff --git a/drivers/media/pci/ivtv/ivtv-gpio.c b/drivers/media/pci/ivtv/ivtv-gpio.c
+index f752f39..23372af6 100644
+--- a/drivers/media/pci/ivtv/ivtv-gpio.c
++++ b/drivers/media/pci/ivtv/ivtv-gpio.c
+@@ -117,7 +117,7 @@ void ivtv_reset_ir_gpio(struct ivtv *itv)
+ 	curout = (curout & ~0xF) | 1;
+ 	write_reg(curout, IVTV_REG_GPIO_OUT);
+ 	/* We could use something else for smaller time */
+-	schedule_timeout_interruptible(msecs_to_jiffies(1));
++	schedule_msec_hrtimeout_interruptible((1));
+ 	curout |= 2;
+ 	write_reg(curout, IVTV_REG_GPIO_OUT);
+ 	curdir &= ~0x80;
+@@ -137,11 +137,11 @@ int ivtv_reset_tuner_gpio(void *dev, int component, int cmd, int value)
+ 	curout = read_reg(IVTV_REG_GPIO_OUT);
+ 	curout &= ~(1 << itv->card->xceive_pin);
+ 	write_reg(curout, IVTV_REG_GPIO_OUT);
+-	schedule_timeout_interruptible(msecs_to_jiffies(1));
++	schedule_msec_hrtimeout_interruptible((1));
+ 
+ 	curout |= 1 << itv->card->xceive_pin;
+ 	write_reg(curout, IVTV_REG_GPIO_OUT);
+-	schedule_timeout_interruptible(msecs_to_jiffies(1));
++	schedule_msec_hrtimeout_interruptible((1));
+ 	return 0;
+ }
+ 
+diff --git a/drivers/media/radio/radio-mr800.c b/drivers/media/radio/radio-mr800.c
+index c2927fd..bdee269 100644
+--- a/drivers/media/radio/radio-mr800.c
++++ b/drivers/media/radio/radio-mr800.c
+@@ -382,7 +382,7 @@ static int vidioc_s_hw_freq_seek(struct file *file, void *priv,
+ 			retval = -ENODATA;
+ 			break;
+ 		}
+-		if (schedule_timeout_interruptible(msecs_to_jiffies(10))) {
++		if (schedule_msec_hrtimeout_interruptible((10))) {
+ 			retval = -ERESTARTSYS;
+ 			break;
+ 		}
+diff --git a/drivers/media/radio/radio-tea5777.c b/drivers/media/radio/radio-tea5777.c
+index 83fe7ab..aaae5fa 100644
+--- a/drivers/media/radio/radio-tea5777.c
++++ b/drivers/media/radio/radio-tea5777.c
+@@ -249,7 +249,7 @@ static int radio_tea5777_update_read_reg(struct radio_tea5777 *tea, int wait)
+ 	}
+ 
+ 	if (wait) {
+-		if (schedule_timeout_interruptible(msecs_to_jiffies(wait)))
++		if (schedule_msec_hrtimeout_interruptible((wait)))
+ 			return -ERESTARTSYS;
+ 	}
+ 
+diff --git a/drivers/media/radio/tea575x.c b/drivers/media/radio/tea575x.c
+index 4dc2067..29f4416 100644
+--- a/drivers/media/radio/tea575x.c
++++ b/drivers/media/radio/tea575x.c
+@@ -416,7 +416,7 @@ int snd_tea575x_s_hw_freq_seek(struct file *file, struct snd_tea575x *tea,
+ 	for (;;) {
+ 		if (time_after(jiffies, timeout))
+ 			break;
+-		if (schedule_timeout_interruptible(msecs_to_jiffies(10))) {
++		if (schedule_msec_hrtimeout_interruptible((10))) {
+ 			/* some signal arrived, stop search */
+ 			tea->val &= ~TEA575X_BIT_SEARCH;
+ 			snd_tea575x_set_freq(tea);
+diff --git a/drivers/misc/panel.c b/drivers/misc/panel.c
+index 6030ac5..f0c1a101 100644
+--- a/drivers/misc/panel.c
++++ b/drivers/misc/panel.c
+@@ -760,7 +760,7 @@ static void long_sleep(int ms)
+ 	if (in_interrupt())
+ 		mdelay(ms);
+ 	else
+-		schedule_timeout_interruptible(msecs_to_jiffies(ms));
++		schedule_msec_hrtimeout_interruptible((ms));
+ }
+ 
+ /*
+diff --git a/drivers/parport/ieee1284.c b/drivers/parport/ieee1284.c
+index f9fd4b3..00ad2f3 100644
+--- a/drivers/parport/ieee1284.c
++++ b/drivers/parport/ieee1284.c
+@@ -215,7 +215,7 @@ int parport_wait_peripheral(struct parport *port,
+ 			/* parport_wait_event didn't time out, but the
+ 			 * peripheral wasn't actually ready either.
+ 			 * Wait for another 10ms. */
+-			schedule_timeout_interruptible(msecs_to_jiffies(10));
++			schedule_msec_hrtimeout_interruptible((10));
+ 		}
+ 	}
+ 
+diff --git a/drivers/parport/ieee1284_ops.c b/drivers/parport/ieee1284_ops.c
+index c0e7d21..e1b4fd4 100644
+--- a/drivers/parport/ieee1284_ops.c
++++ b/drivers/parport/ieee1284_ops.c
+@@ -536,7 +536,7 @@ size_t parport_ieee1284_ecp_read_data (struct parport *port,
+ 			/* Yield the port for a while. */
+ 			if (count && dev->port->irq != PARPORT_IRQ_NONE) {
+ 				parport_release (dev);
+-				schedule_timeout_interruptible(msecs_to_jiffies(40));
++				schedule_msec_hrtimeout_interruptible((40));
+ 				parport_claim_or_block (dev);
+ 			}
+ 			else
+diff --git a/drivers/platform/x86/intel_ips.c b/drivers/platform/x86/intel_ips.c
+index 55663b3..0363fed 100644
+--- a/drivers/platform/x86/intel_ips.c
++++ b/drivers/platform/x86/intel_ips.c
+@@ -812,7 +812,7 @@ static int ips_adjust(void *data)
+ 			ips_gpu_lower(ips);
+ 
+ sleep:
+-		schedule_timeout_interruptible(msecs_to_jiffies(IPS_ADJUST_PERIOD));
++		schedule_msec_hrtimeout_interruptible((IPS_ADJUST_PERIOD));
+ 	} while (!kthread_should_stop());
+ 
+ 	dev_dbg(&ips->dev->dev, "ips-adjust thread stopped\n");
+@@ -991,7 +991,7 @@ static int ips_monitor(void *data)
+ 	seqno_timestamp = get_jiffies_64();
+ 
+ 	old_cpu_power = thm_readl(THM_CEC);
+-	schedule_timeout_interruptible(msecs_to_jiffies(IPS_SAMPLE_PERIOD));
++	schedule_msec_hrtimeout_interruptible((IPS_SAMPLE_PERIOD));
+ 
+ 	/* Collect an initial average */
+ 	for (i = 0; i < IPS_SAMPLE_COUNT; i++) {
+@@ -1018,7 +1018,7 @@ static int ips_monitor(void *data)
+ 			mchp_samples[i] = mchp;
+ 		}
+ 
+-		schedule_timeout_interruptible(msecs_to_jiffies(IPS_SAMPLE_PERIOD));
++		schedule_msec_hrtimeout_interruptible((IPS_SAMPLE_PERIOD));
+ 		if (kthread_should_stop())
+ 			break;
+ 	}
+@@ -1045,7 +1045,7 @@ static int ips_monitor(void *data)
+ 	 * us to reduce the sample frequency if the CPU and GPU are idle.
+ 	 */
+ 	old_cpu_power = thm_readl(THM_CEC);
+-	schedule_timeout_interruptible(msecs_to_jiffies(IPS_SAMPLE_PERIOD));
++	schedule_msec_hrtimeout_interruptible((IPS_SAMPLE_PERIOD));
+ 	last_sample_period = IPS_SAMPLE_PERIOD;
+ 
+ 	setup_deferrable_timer_on_stack(&timer, monitor_timeout,
+diff --git a/net/core/pktgen.c b/net/core/pktgen.c
+index 8e69ce4..0227415 100644
+--- a/net/core/pktgen.c
++++ b/net/core/pktgen.c
+@@ -1992,7 +1992,7 @@ static void pktgen_mark_device(const struct pktgen_net *pn, const char *ifname)
+ 		mutex_unlock(&pktgen_thread_lock);
+ 		pr_debug("%s: waiting for %s to disappear....\n",
+ 			 __func__, ifname);
+-		schedule_timeout_interruptible(msecs_to_jiffies(msec_per_try));
++		schedule_msec_hrtimeout_interruptible((msec_per_try));
+ 		mutex_lock(&pktgen_thread_lock);
+ 
+ 		if (++i >= max_tries) {
+diff --git a/sound/soc/codecs/wm8350.c b/sound/soc/codecs/wm8350.c
+index 2efc5b4..3e3248c 100644
+--- a/sound/soc/codecs/wm8350.c
++++ b/sound/soc/codecs/wm8350.c
+@@ -236,10 +236,10 @@ static void wm8350_pga_work(struct work_struct *work)
+ 		    out2->ramp == WM8350_RAMP_UP) {
+ 			/* delay is longer over 0dB as increases are larger */
+ 			if (i >= WM8350_OUTn_0dB)
+-				schedule_timeout_interruptible(msecs_to_jiffies
++				schedule_msec_hrtimeout_interruptible(
+ 							       (2));
+ 			else
+-				schedule_timeout_interruptible(msecs_to_jiffies
++				schedule_msec_hrtimeout_interruptible(
+ 							       (1));
+ 		} else
+ 			udelay(50);	/* doesn't matter if we delay longer */
+@@ -1123,7 +1123,7 @@ static int wm8350_set_bias_level(struct snd_soc_codec *codec,
+ 					 (platform->dis_out4 << 6));
+ 
+ 			/* wait for discharge */
+-			schedule_timeout_interruptible(msecs_to_jiffies
++			schedule_msec_hrtimeout_interruptible(
+ 						       (platform->
+ 							cap_discharge_msecs));
+ 
+@@ -1139,7 +1139,7 @@ static int wm8350_set_bias_level(struct snd_soc_codec *codec,
+ 					 WM8350_VBUFEN);
+ 
+ 			/* wait for vmid */
+-			schedule_timeout_interruptible(msecs_to_jiffies
++			schedule_msec_hrtimeout_interruptible(
+ 						       (platform->
+ 							vmid_charge_msecs));
+ 
+@@ -1190,7 +1190,7 @@ static int wm8350_set_bias_level(struct snd_soc_codec *codec,
+ 		wm8350_reg_write(wm8350, WM8350_POWER_MGMT_1, pm1);
+ 
+ 		/* wait */
+-		schedule_timeout_interruptible(msecs_to_jiffies
++		schedule_msec_hrtimeout_interruptible(
+ 					       (platform->
+ 						vmid_discharge_msecs));
+ 
+@@ -1208,7 +1208,7 @@ static int wm8350_set_bias_level(struct snd_soc_codec *codec,
+ 				 pm1 | WM8350_OUTPUT_DRAIN_EN);
+ 
+ 		/* wait */
+-		schedule_timeout_interruptible(msecs_to_jiffies
++		schedule_msec_hrtimeout_interruptible(
+ 					       (platform->drain_msecs));
+ 
+ 		pm1 &= ~WM8350_BIASEN;
+diff --git a/sound/soc/codecs/wm8900.c b/sound/soc/codecs/wm8900.c
+index c77b49a..fc50456 100644
+--- a/sound/soc/codecs/wm8900.c
++++ b/sound/soc/codecs/wm8900.c
+@@ -1112,7 +1112,7 @@ static int wm8900_set_bias_level(struct snd_soc_codec *codec,
+ 		/* Need to let things settle before stopping the clock
+ 		 * to ensure that restart works, see "Stopping the
+ 		 * master clock" in the datasheet. */
+-		schedule_timeout_interruptible(msecs_to_jiffies(1));
++		schedule_msec_hrtimeout_interruptible((1));
+ 		snd_soc_write(codec, WM8900_REG_POWER2,
+ 			     WM8900_REG_POWER2_SYSCLK_ENA);
+ 		break;
+diff --git a/sound/soc/codecs/wm9713.c b/sound/soc/codecs/wm9713.c
+index 7e48221..0c85a20 100644
+--- a/sound/soc/codecs/wm9713.c
++++ b/sound/soc/codecs/wm9713.c
+@@ -199,7 +199,7 @@ static int wm9713_voice_shutdown(struct snd_soc_dapm_widget *w,
+ 
+ 	/* Gracefully shut down the voice interface. */
+ 	snd_soc_update_bits(codec, AC97_HANDSET_RATE, 0x0f00, 0x0200);
+-	schedule_timeout_interruptible(msecs_to_jiffies(1));
++	schedule_msec_hrtimeout_interruptible((1));
+ 	snd_soc_update_bits(codec, AC97_HANDSET_RATE, 0x0f00, 0x0f00);
+ 	snd_soc_update_bits(codec, AC97_EXTENDED_MID, 0x1000, 0x1000);
+ 
+@@ -868,7 +868,7 @@ static int wm9713_set_pll(struct snd_soc_codec *codec,
+ 	wm9713->pll_in = freq_in;
+ 
+ 	/* wait 10ms AC97 link frames for the link to stabilise */
+-	schedule_timeout_interruptible(msecs_to_jiffies(10));
++	schedule_msec_hrtimeout_interruptible((10));
+ 	return 0;
+ }
+ 
+-- 
+2.9.3
+
diff --git a/patches/0015-Replace-all-calls-to-schedule_timeout_uninterruptibl.patch b/patches/0015-Replace-all-calls-to-schedule_timeout_uninterruptibl.patch
new file mode 100644
index 0000000..be929c6
--- /dev/null
+++ b/patches/0015-Replace-all-calls-to-schedule_timeout_uninterruptibl.patch
@@ -0,0 +1,160 @@
+From 1137ff2bfa5eb63b53747fe303fdb3937c5e1077 Mon Sep 17 00:00:00 2001
+From: Con Kolivas <kernel@kolivas.org>
+Date: Mon, 20 Feb 2017 13:30:32 +1100
+Subject: [PATCH 15/25] Replace all calls to schedule_timeout_uninterruptible
+ of potentially under 50ms to use schedule_msec_hrtimeout_uninterruptible
+
+---
+ drivers/media/pci/cx18/cx18-gpio.c           | 4 ++--
+ drivers/net/wireless/intel/ipw2x00/ipw2100.c | 4 ++--
+ drivers/rtc/rtc-wm8350.c                     | 6 +++---
+ drivers/scsi/lpfc/lpfc_scsi.c                | 2 +-
+ sound/pci/maestro3.c                         | 4 ++--
+ sound/soc/codecs/rt5631.c                    | 4 ++--
+ sound/soc/soc-dapm.c                         | 2 +-
+ 7 files changed, 13 insertions(+), 13 deletions(-)
+
+diff --git a/drivers/media/pci/cx18/cx18-gpio.c b/drivers/media/pci/cx18/cx18-gpio.c
+index 38dc6b8..3cd3098 100644
+--- a/drivers/media/pci/cx18/cx18-gpio.c
++++ b/drivers/media/pci/cx18/cx18-gpio.c
+@@ -95,11 +95,11 @@ static void gpio_reset_seq(struct cx18 *cx, u32 active_lo, u32 active_hi,
+ 
+ 	/* Assert */
+ 	gpio_update(cx, mask, ~active_lo);
+-	schedule_timeout_uninterruptible(msecs_to_jiffies(assert_msecs));
++	schedule_msec_hrtimeout_uninterruptible((assert_msecs));
+ 
+ 	/* Deassert */
+ 	gpio_update(cx, mask, ~active_hi);
+-	schedule_timeout_uninterruptible(msecs_to_jiffies(recovery_msecs));
++	schedule_msec_hrtimeout_uninterruptible((recovery_msecs));
+ }
+ 
+ /*
+diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2100.c b/drivers/net/wireless/intel/ipw2x00/ipw2100.c
+index 356aba9..d2cc761 100644
+--- a/drivers/net/wireless/intel/ipw2x00/ipw2100.c
++++ b/drivers/net/wireless/intel/ipw2x00/ipw2100.c
+@@ -830,7 +830,7 @@ static int ipw2100_hw_send_command(struct ipw2100_priv *priv,
+ 	 * doesn't seem to have as many firmware restart cycles...
+ 	 *
+ 	 * As a test, we're sticking in a 1/100s delay here */
+-	schedule_timeout_uninterruptible(msecs_to_jiffies(10));
++	schedule_msec_hrtimeout_uninterruptible((10));
+ 
+ 	return 0;
+ 
+@@ -1281,7 +1281,7 @@ static int ipw2100_start_adapter(struct ipw2100_priv *priv)
+ 	IPW_DEBUG_FW("Waiting for f/w initialization to complete...\n");
+ 	i = 5000;
+ 	do {
+-		schedule_timeout_uninterruptible(msecs_to_jiffies(40));
++		schedule_msec_hrtimeout_uninterruptible((40));
+ 		/* Todo... wait for sync command ... */
+ 
+ 		read_register(priv->net_dev, IPW_REG_INTA, &inta);
+diff --git a/drivers/rtc/rtc-wm8350.c b/drivers/rtc/rtc-wm8350.c
+index fa247de..f1a28d8 100644
+--- a/drivers/rtc/rtc-wm8350.c
++++ b/drivers/rtc/rtc-wm8350.c
+@@ -121,7 +121,7 @@ static int wm8350_rtc_settime(struct device *dev, struct rtc_time *tm)
+ 	/* Wait until confirmation of stopping */
+ 	do {
+ 		rtc_ctrl = wm8350_reg_read(wm8350, WM8350_RTC_TIME_CONTROL);
+-		schedule_timeout_uninterruptible(msecs_to_jiffies(1));
++		schedule_msec_hrtimeout_uninterruptible((1));
+ 	} while (--retries && !(rtc_ctrl & WM8350_RTC_STS));
+ 
+ 	if (!retries) {
+@@ -204,7 +204,7 @@ static int wm8350_rtc_stop_alarm(struct wm8350 *wm8350)
+ 	/* Wait until confirmation of stopping */
+ 	do {
+ 		rtc_ctrl = wm8350_reg_read(wm8350, WM8350_RTC_TIME_CONTROL);
+-		schedule_timeout_uninterruptible(msecs_to_jiffies(1));
++		schedule_msec_hrtimeout_uninterruptible((1));
+ 	} while (retries-- && !(rtc_ctrl & WM8350_RTC_ALMSTS));
+ 
+ 	if (!(rtc_ctrl & WM8350_RTC_ALMSTS))
+@@ -227,7 +227,7 @@ static int wm8350_rtc_start_alarm(struct wm8350 *wm8350)
+ 	/* Wait until confirmation */
+ 	do {
+ 		rtc_ctrl = wm8350_reg_read(wm8350, WM8350_RTC_TIME_CONTROL);
+-		schedule_timeout_uninterruptible(msecs_to_jiffies(1));
++		schedule_msec_hrtimeout_uninterruptible((1));
+ 	} while (retries-- && rtc_ctrl & WM8350_RTC_ALMSTS);
+ 
+ 	if (rtc_ctrl & WM8350_RTC_ALMSTS)
+diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c
+index ad350d9..69a58a8 100644
+--- a/drivers/scsi/lpfc/lpfc_scsi.c
++++ b/drivers/scsi/lpfc/lpfc_scsi.c
+@@ -5109,7 +5109,7 @@ lpfc_reset_flush_io_context(struct lpfc_vport *vport, uint16_t tgt_id,
+ 					tgt_id, lun_id, context);
+ 	later = msecs_to_jiffies(2 * vport->cfg_devloss_tmo * 1000) + jiffies;
+ 	while (time_after(later, jiffies) && cnt) {
+-		schedule_timeout_uninterruptible(msecs_to_jiffies(20));
++		schedule_msec_hrtimeout_uninterruptible((20));
+ 		cnt = lpfc_sli_sum_iocb(vport, tgt_id, lun_id, context);
+ 	}
+ 	if (cnt) {
+diff --git a/sound/pci/maestro3.c b/sound/pci/maestro3.c
+index cafea6d..d374514 100644
+--- a/sound/pci/maestro3.c
++++ b/sound/pci/maestro3.c
+@@ -2016,7 +2016,7 @@ static void snd_m3_ac97_reset(struct snd_m3 *chip)
+ 		outw(0, io + GPIO_DATA);
+ 		outw(dir | GPO_PRIMARY_AC97, io + GPIO_DIRECTION);
+ 
+-		schedule_timeout_uninterruptible(msecs_to_jiffies(delay1));
++		schedule_msec_hrtimeout_uninterruptible((delay1));
+ 
+ 		outw(GPO_PRIMARY_AC97, io + GPIO_DATA);
+ 		udelay(5);
+@@ -2024,7 +2024,7 @@ static void snd_m3_ac97_reset(struct snd_m3 *chip)
+ 		outw(IO_SRAM_ENABLE | SERIAL_AC_LINK_ENABLE, io + RING_BUS_CTRL_A);
+ 		outw(~0, io + GPIO_MASK);
+ 
+-		schedule_timeout_uninterruptible(msecs_to_jiffies(delay2));
++		schedule_msec_hrtimeout_uninterruptible((delay2));
+ 
+ 		if (! snd_m3_try_read_vendor(chip))
+ 			break;
+diff --git a/sound/soc/codecs/rt5631.c b/sound/soc/codecs/rt5631.c
+index 0e41808..611cb9f 100644
+--- a/sound/soc/codecs/rt5631.c
++++ b/sound/soc/codecs/rt5631.c
+@@ -419,7 +419,7 @@ static void onebit_depop_mute_stage(struct snd_soc_codec *codec, int enable)
+ 	hp_zc = snd_soc_read(codec, RT5631_INT_ST_IRQ_CTRL_2);
+ 	snd_soc_write(codec, RT5631_INT_ST_IRQ_CTRL_2, hp_zc & 0xf7ff);
+ 	if (enable) {
+-		schedule_timeout_uninterruptible(msecs_to_jiffies(10));
++		schedule_msec_hrtimeout_uninterruptible((10));
+ 		/* config one-bit depop parameter */
+ 		rt5631_write_index(codec, RT5631_SPK_INTL_CTRL, 0x307f);
+ 		snd_soc_update_bits(codec, RT5631_HP_OUT_VOL,
+@@ -529,7 +529,7 @@ static void depop_seq_mute_stage(struct snd_soc_codec *codec, int enable)
+ 	hp_zc = snd_soc_read(codec, RT5631_INT_ST_IRQ_CTRL_2);
+ 	snd_soc_write(codec, RT5631_INT_ST_IRQ_CTRL_2, hp_zc & 0xf7ff);
+ 	if (enable) {
+-		schedule_timeout_uninterruptible(msecs_to_jiffies(10));
++		schedule_msec_hrtimeout_uninterruptible((10));
+ 
+ 		/* config depop sequence parameter */
+ 		rt5631_write_index(codec, RT5631_SPK_INTL_CTRL, 0x302f);
+diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c
+index 27dd02e..7ba49f4 100644
+--- a/sound/soc/soc-dapm.c
++++ b/sound/soc/soc-dapm.c
+@@ -134,7 +134,7 @@ static void dapm_assert_locked(struct snd_soc_dapm_context *dapm)
+ static void pop_wait(u32 pop_time)
+ {
+ 	if (pop_time)
+-		schedule_timeout_uninterruptible(msecs_to_jiffies(pop_time));
++		schedule_msec_hrtimeout_uninterruptible((pop_time));
+ }
+ 
+ static void pop_dbg(struct device *dev, u32 pop_time, const char *fmt, ...)
+-- 
+2.9.3
+
diff --git a/patches/0016-Fix-build-for-disabled-highres-timers-with-hrtimeout.patch b/patches/0016-Fix-build-for-disabled-highres-timers-with-hrtimeout.patch
new file mode 100644
index 0000000..0e1e0b7
--- /dev/null
+++ b/patches/0016-Fix-build-for-disabled-highres-timers-with-hrtimeout.patch
@@ -0,0 +1,84 @@
+From 37496baeea800e745a77620e90660496135f7fa5 Mon Sep 17 00:00:00 2001
+From: Con Kolivas <kernel@kolivas.org>
+Date: Mon, 20 Feb 2017 13:31:42 +1100
+Subject: [PATCH 16/25] Fix build for disabled highres timers with hrtimeout
+ code.
+
+---
+ include/linux/freezer.h |  1 +
+ include/linux/sched.h   | 22 ++++++++++++++++++++++
+ kernel/time/timer.c     |  2 ++
+ 3 files changed, 25 insertions(+)
+
+diff --git a/include/linux/freezer.h b/include/linux/freezer.h
+index dd03e83..2fda682 100644
+--- a/include/linux/freezer.h
++++ b/include/linux/freezer.h
+@@ -296,6 +296,7 @@ static inline void set_freezable(void) {}
+ #define wait_event_freezekillable_unsafe(wq, condition)			\
+ 		wait_event_killable(wq, condition)
+ 
++#define pm_freezing (false)
+ #endif /* !CONFIG_FREEZER */
+ 
+ #endif	/* FREEZER_H_INCLUDED */
+diff --git a/include/linux/sched.h b/include/linux/sched.h
+index 46544f4..680494d 100644
+--- a/include/linux/sched.h
++++ b/include/linux/sched.h
+@@ -458,10 +458,32 @@ extern signed long schedule_timeout_killable(signed long timeout);
+ extern signed long schedule_timeout_uninterruptible(signed long timeout);
+ extern signed long schedule_timeout_idle(signed long timeout);
+ 
++#ifdef CONFIG_HIGH_RES_TIMERS
+ extern signed long schedule_msec_hrtimeout(signed long timeout);
+ extern signed long schedule_min_hrtimeout(void);
+ extern signed long schedule_msec_hrtimeout_interruptible(signed long timeout);
+ extern signed long schedule_msec_hrtimeout_uninterruptible(signed long timeout);
++#else
++static inline signed long schedule_msec_hrtimeout(signed long timeout)
++{
++	return schedule_timeout(msecs_to_jiffies(timeout));
++}
++
++static inline signed long schedule_min_hrtimeout(void)
++{
++	return schedule_timeout(1);
++}
++
++static inline signed long schedule_msec_hrtimeout_interruptible(signed long timeout)
++{
++	return schedule_timeout_interruptible(msecs_to_jiffies(timeout));
++}
++
++static inline signed long schedule_msec_hrtimeout_uninterruptible(signed long timeout)
++{
++	return schedule_timeout_uninterruptible(msecs_to_jiffies(timeout));
++}
++#endif
+ 
+ asmlinkage void schedule(void);
+ extern void schedule_preempt_disabled(void);
+diff --git a/kernel/time/timer.c b/kernel/time/timer.c
+index bc53598..0e22641 100644
+--- a/kernel/time/timer.c
++++ b/kernel/time/timer.c
+@@ -1750,6 +1750,7 @@ signed long __sched schedule_timeout(signed long timeout)
+ 
+ 	expire = timeout + jiffies;
+ 
++#ifdef CONFIG_HIGH_RES_TIMERS
+ 	if (timeout == 1 && hrtimer_resolution < NSEC_PER_SEC / HZ) {
+ 		/*
+ 		 * Special case 1 as being a request for the minimum timeout
+@@ -1760,6 +1761,7 @@ signed long __sched schedule_timeout(signed long timeout)
+ 			return 0;
+ 		goto out_timeout;
+ 	}
++#endif
+ 
+ 	setup_timer_on_stack(&timer, process_timeout, (unsigned long)current);
+ 	__mod_timer(&timer, expire, false);
+-- 
+2.9.3
+
diff --git a/patches/0018-Make-threaded-IRQs-optionally-the-default-which-can-.patch b/patches/0018-Make-threaded-IRQs-optionally-the-default-which-can-.patch
new file mode 100644
index 0000000..fe49e0e
--- /dev/null
+++ b/patches/0018-Make-threaded-IRQs-optionally-the-default-which-can-.patch
@@ -0,0 +1,61 @@
+From a0d29f014dbcb29649dff1a9b8df58bad7be3926 Mon Sep 17 00:00:00 2001
+From: Con Kolivas <kernel@kolivas.org>
+Date: Wed, 7 Dec 2016 21:13:16 +1100
+Subject: [PATCH 18/25] Make threaded IRQs optionally the default which can be
+ disabled.
+
+---
+ kernel/irq/Kconfig  | 14 ++++++++++++++
+ kernel/irq/manage.c | 10 ++++++++++
+ 2 files changed, 24 insertions(+)
+
+diff --git a/kernel/irq/Kconfig b/kernel/irq/Kconfig
+index 3bbfd6a..351bf16 100644
+--- a/kernel/irq/Kconfig
++++ b/kernel/irq/Kconfig
+@@ -95,6 +95,20 @@ config IRQ_DOMAIN_DEBUG
+ config IRQ_FORCED_THREADING
+        bool
+ 
++config FORCE_IRQ_THREADING
++	bool "Make IRQ threading compulsory"
++	depends on IRQ_FORCED_THREADING
++	default y
++	---help---
++
++	  Make IRQ threading mandatory for any IRQ handlers that support it
++	  instead of being optional and requiring the threadirqs kernel
++	  parameter. Instead they can be optionally disabled with the
++	  nothreadirqs kernel parameter.
++
++	  Enable if you are building for a desktop or low latency system,
++	  otherwise say N.
++
+ config SPARSE_IRQ
+ 	bool "Support sparse irq numbering" if MAY_HAVE_SPARSE_IRQ
+ 	---help---
+diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
+index 6b66959..6b3fb17 100644
+--- a/kernel/irq/manage.c
++++ b/kernel/irq/manage.c
+@@ -22,7 +22,17 @@
+ #include "internals.h"
+ 
+ #ifdef CONFIG_IRQ_FORCED_THREADING
++#ifdef CONFIG_FORCE_IRQ_THREADING
++__read_mostly bool force_irqthreads = true;
++#else
+ __read_mostly bool force_irqthreads;
++#endif
++static int __init setup_noforced_irqthreads(char *arg)
++{
++	force_irqthreads = false;
++	return 0;
++}
++early_param("nothreadirqs", setup_noforced_irqthreads);
+ 
+ static int __init setup_forced_irqthreads(char *arg)
+ {
+-- 
+2.9.3
+
diff --git a/patches/0020-Don-t-use-hrtimer-overlay-when-pm_freezing-since-som.patch b/patches/0020-Don-t-use-hrtimer-overlay-when-pm_freezing-since-som.patch
new file mode 100644
index 0000000..1bf923d
--- /dev/null
+++ b/patches/0020-Don-t-use-hrtimer-overlay-when-pm_freezing-since-som.patch
@@ -0,0 +1,69 @@
+From a278cad439033005610ddda23882f2c681c669d1 Mon Sep 17 00:00:00 2001
+From: Con Kolivas <kernel@kolivas.org>
+Date: Mon, 20 Feb 2017 13:32:58 +1100
+Subject: [PATCH 20/25] Don't use hrtimer overlay when pm_freezing since some
+ drivers still don't correctly use freezable timeouts.
+
+---
+ kernel/time/hrtimer.c | 2 +-
+ kernel/time/timer.c   | 9 +++++----
+ 2 files changed, 6 insertions(+), 5 deletions(-)
+
+diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
+index 26ac1f8..25e1555 100644
+--- a/kernel/time/hrtimer.c
++++ b/kernel/time/hrtimer.c
+@@ -1818,7 +1818,7 @@ signed long __sched schedule_msec_hrtimeout(signed long timeout)
+ 	 * (yet) better than Hz, as would occur during startup, use regular
+ 	 * timers.
+ 	 */
+-	if (jiffs > 4 || hrtimer_resolution >= NSEC_PER_SEC / HZ)
++	if (jiffs > 4 || hrtimer_resolution >= NSEC_PER_SEC / HZ || pm_freezing)
+ 		return schedule_timeout(jiffs);
+ 
+ 	delta = (timeout % 1000) * NSEC_PER_MSEC;
+diff --git a/kernel/time/timer.c b/kernel/time/timer.c
+index 0e22641..45a6e1f 100644
+--- a/kernel/time/timer.c
++++ b/kernel/time/timer.c
+@@ -42,6 +42,7 @@
+ #include <linux/sched/sysctl.h>
+ #include <linux/slab.h>
+ #include <linux/compat.h>
++#include <linux/freezer.h>
+ 
+ #include <linux/uaccess.h>
+ #include <asm/unistd.h>
+@@ -1901,12 +1902,12 @@ void msleep(unsigned int msecs)
+ 	 * Use high resolution timers where the resolution of tick based
+ 	 * timers is inadequate.
+ 	 */
+-	if (jiffs < 5 && hrtimer_resolution < NSEC_PER_SEC / HZ) {
++	if (jiffs < 5 && hrtimer_resolution < NSEC_PER_SEC / HZ && !pm_freezing) {
+ 		while (msecs)
+ 			msecs = schedule_msec_hrtimeout_uninterruptible(msecs);
+ 		return;
+ 	}
+-	timeout = msecs_to_jiffies(msecs) + 1;
++	timeout = jiffs + 1;
+ 
+ 	while (timeout)
+ 		timeout = schedule_timeout_uninterruptible(timeout);
+@@ -1923,12 +1924,12 @@ unsigned long msleep_interruptible(unsigned int msecs)
+ 	int jiffs = msecs_to_jiffies(msecs);
+ 	unsigned long timeout;
+ 
+-	if (jiffs < 5 && hrtimer_resolution < NSEC_PER_SEC / HZ) {
++	if (jiffs < 5 && hrtimer_resolution < NSEC_PER_SEC / HZ && !pm_freezing) {
+ 		while (msecs && !signal_pending(current))
+ 			msecs = schedule_msec_hrtimeout_interruptible(msecs);
+ 		return msecs;
+ 	}
+-	timeout = msecs_to_jiffies(msecs) + 1;
++	timeout = jiffs + 1;
+ 
+ 	while (timeout && !signal_pending(current))
+ 		timeout = schedule_timeout_interruptible(timeout);
+-- 
+2.9.3
+
diff --git a/patches/0021-Make-writeback-throttling-default-enabled.patch b/patches/0021-Make-writeback-throttling-default-enabled.patch
new file mode 100644
index 0000000..0d587bb
--- /dev/null
+++ b/patches/0021-Make-writeback-throttling-default-enabled.patch
@@ -0,0 +1,34 @@
+From da915e0f3abeb61f6a132bb77b7d0a9bf0573233 Mon Sep 17 00:00:00 2001
+From: Con Kolivas <kernel@kolivas.org>
+Date: Mon, 20 Feb 2017 13:38:23 +1100
+Subject: [PATCH 21/25] Make writeback throttling default enabled.
+
+---
+ block/Kconfig | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/block/Kconfig b/block/Kconfig
+index 8bf114a..83e6f9d 100644
+--- a/block/Kconfig
++++ b/block/Kconfig
+@@ -123,7 +123,7 @@ config BLK_CMDLINE_PARSER
+ 
+ config BLK_WBT
+ 	bool "Enable support for block device writeback throttling"
+-	default n
++	default y
+ 	---help---
+ 	Enabling this option enables the block layer to throttle buffered
+ 	background writeback from the VM, making it more smooth and having
+@@ -133,7 +133,7 @@ config BLK_WBT
+ 
+ config BLK_WBT_SQ
+ 	bool "Single queue writeback throttling"
+-	default n
++	default y
+ 	depends on BLK_WBT
+ 	---help---
+ 	Enable writeback throttling by default on legacy single queue devices
+-- 
+2.9.3
+
diff --git a/patches/0022-Swap-sucks.patch b/patches/0022-Swap-sucks.patch
new file mode 100644
index 0000000..ddab4f8
--- /dev/null
+++ b/patches/0022-Swap-sucks.patch
@@ -0,0 +1,43 @@
+From 2f96168f72bbd431c0e6d28b44393e98b49ca787 Mon Sep 17 00:00:00 2001
+From: Con Kolivas <kernel@kolivas.org>
+Date: Mon, 20 Feb 2017 13:48:54 +1100
+Subject: [PATCH 22/25] Swap sucks.
+
+---
+ include/linux/swap.h | 6 +-----
+ mm/vmscan.c          | 2 +-
+ 2 files changed, 2 insertions(+), 6 deletions(-)
+
+diff --git a/include/linux/swap.h b/include/linux/swap.h
+index 7f47b70..1c2ed28 100644
+--- a/include/linux/swap.h
++++ b/include/linux/swap.h
+@@ -361,11 +361,7 @@ extern struct page *swapin_readahead(swp_entry_t, gfp_t,
+ extern atomic_long_t nr_swap_pages;
+ extern long total_swap_pages;
+ 
+-/* Swap 50% full? Release swapcache more aggressively.. */
+-static inline bool vm_swap_full(void)
+-{
+-	return atomic_long_read(&nr_swap_pages) * 2 < total_swap_pages;
+-}
++#define vm_swap_full()	1
+ 
+ static inline long get_nr_swap_pages(void)
+ {
+diff --git a/mm/vmscan.c b/mm/vmscan.c
+index 532a2a7..15e4260 100644
+--- a/mm/vmscan.c
++++ b/mm/vmscan.c
+@@ -141,7 +141,7 @@ struct scan_control {
+ /*
+  * From 0 .. 100.  Higher means more swappy.
+  */
+-int vm_swappiness = 60;
++int vm_swappiness = 33;
+ /*
+  * The total number of pages which are beyond the high watermark within all
+  * zones.
+-- 
+2.9.3
+
diff --git a/patches/enable_additional_cpu_optimizations_for_gcc_v4.9+_kernel_v3.15+.patch b/patches/enable_additional_cpu_optimizations_for_gcc_v4.9+_kernel_v3.15+.patch
index d9729b2..76cbd9d 100644
--- a/patches/enable_additional_cpu_optimizations_for_gcc_v4.9+_kernel_v3.15+.patch
+++ b/patches/enable_additional_cpu_optimizations_for_gcc_v4.9+_kernel_v3.15+.patch
@@ -1,33 +1,51 @@
-WARNING - this version of the patch works with version 4.9+ of gcc and with
-kernel version 3.15.x+ and should NOT be applied when compiling on older
-versions due to name changes of the flags with the 4.9 release of gcc.
+WARNING
+This patch works with gcc versions 4.9+ and with kernel version 3.15+ and should
+NOT be applied when compiling on older versions of gcc due to key name changes
+of the march flags introduced with the version 4.9 release of gcc.[1]
+
 Use the older version of this patch hosted on the same github for older
-versions of gcc. For example:
+versions of gcc.
 
-corei7 --> nehalem
-corei7-avx --> sandybridge
-core-avx-i --> ivybridge
-core-avx2 --> haswell
+FEATURES
+This patch adds additional CPU options to the Linux kernel accessible under:
+ Processor type and features  --->
+  Processor family --->
 
-For more, see: https://gcc.gnu.org/gcc-4.9/changes.html
+The expanded microarchitectures include:
+* AMD Improved K8-family
+* AMD K10-family
+* AMD Family 10h (Barcelona)
+* AMD Family 14h (Bobcat)
+* AMD Family 16h (Jaguar)
+* AMD Family 15h (Bulldozer)
+* AMD Family 15h (Piledriver)
+* AMD Family 15h (Steamroller)
+* AMD Family 15h (Excavator)
+* AMD Family 17h (Zen)
+* Intel Silvermont low-power processors
+* Intel 1st Gen Core i3/i5/i7 (Nehalem)
+* Intel 1.5 Gen Core i3/i5/i7 (Westmere)
+* Intel 2nd Gen Core i3/i5/i7 (Sandybridge)
+* Intel 3rd Gen Core i3/i5/i7 (Ivybridge)
+* Intel 4th Gen Core i3/i5/i7 (Haswell)
+* Intel 5th Gen Core i3/i5/i7 (Broadwell)
+* Intel 6th Gen Core i3/i5.i7 (Skylake)
 
-It also changes 'atom' to 'bonnell' in accordance with the gcc v4.9 changes.
-Note that upstream is using the deprecated 'match=atom' flags when I believe it
-should use the newer 'march=bonnell' flag for atom processors.
+It also offers to compile passing the 'native' option which, "selects the CPU
+to generate code for at compilation time by determining the processor type of
+the compiling machine. Using -march=native enables all instruction subsets
+supported by the local machine and will produce code optimized for the local
+machine under the constraints of the selected instruction set."[3]
 
-I have made that change to this patch set as well.  See the following kernel
-bug report to see if I'm right: https://bugzilla.kernel.org/show_bug.cgi?id=77461
+MINOR NOTES
+This patch also changes 'atom' to 'bonnell' in accordance with the gcc v4.9
+changes. Note that upstream is using the deprecated 'match=atom' flags when I
+believe it should use the newer 'march=bonnell' flag for atom processors.[2]
 
-This patch will expand the number of microarchitectures to include newer
-processors including: AMD K10-family, AMD Family 10h (Barcelona), AMD Family
-14h (Bobcat), AMD Family 15h (Bulldozer), AMD Family 15h (Piledriver), AMD
-Family 15h (Steamroller), Family 16h (Jaguar), Intel 1st Gen Core i3/i5/i7
-(Nehalem), Intel 1.5 Gen Core i3/i5/i7 (Westmere), Intel 2nd Gen Core i3/i5/i7
-(Sandybridge), Intel 3rd Gen Core i3/i5/i7 (Ivybridge), Intel 4th Gen Core
-i3/i5/i7 (Haswell), Intel 5th Gen Core i3/i5/i7 (Broadwell), and the low power
-Silvermont series of Atom processors (Silvermont). It also offers the compiler
-the 'native' flag.
+It is not recommended to compile on Atom-CPUs with the 'native' option.[4] The
+recommendation is use to the 'atom' option instead.
 
+BENEFITS
 Small but real speed increases are measurable using a make endpoint comparing
 a generic kernel to one built with one of the respective microarchs.
 
@@ -38,8 +56,18 @@ REQUIREMENTS
 linux version >=3.15
 gcc version >=4.9
 
---- a/arch/x86/include/asm/module.h	2015-08-30 14:34:09.000000000 -0400
-+++ b/arch/x86/include/asm/module.h	2015-11-06 14:18:24.234941036 -0500
+ACKNOWLEDGMENTS
+This patch builds on the seminal work by Jeroen.[5]
+
+REFERENCES
+1. https://gcc.gnu.org/gcc-4.9/changes.html
+2. https://bugzilla.kernel.org/show_bug.cgi?id=77461
+3. https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html
+4. https://github.com/graysky2/kernel_gcc_patch/issues/15
+5. http://www.linuxforge.net/docs/linux/linux-gcc.php
+
+--- a/arch/x86/include/asm/module.h	2016-12-11 14:17:54.000000000 -0500
++++ b/arch/x86/include/asm/module.h	2017-01-06 20:44:36.602227264 -0500
 @@ -15,6 +15,24 @@
  #define MODULE_PROC_FAMILY "586MMX "
  #elif defined CONFIG_MCORE2
@@ -65,7 +93,7 @@ gcc version >=4.9
  #elif defined CONFIG_MATOM
  #define MODULE_PROC_FAMILY "ATOM "
  #elif defined CONFIG_M686
-@@ -33,6 +51,22 @@
+@@ -33,6 +51,26 @@
  #define MODULE_PROC_FAMILY "K7 "
  #elif defined CONFIG_MK8
  #define MODULE_PROC_FAMILY "K8 "
@@ -80,17 +108,29 @@ gcc version >=4.9
 +#elif defined CONFIG_MBULLDOZER
 +#define MODULE_PROC_FAMILY "BULLDOZER "
 +#elif defined CONFIG_MPILEDRIVER
-+#define MODULE_PROC_FAMILY "STEAMROLLER "
-+#elif defined CONFIG_MSTEAMROLLER
 +#define MODULE_PROC_FAMILY "PILEDRIVER "
++#elif defined CONFIG_MSTEAMROLLER
++#define MODULE_PROC_FAMILY "STEAMROLLER "
 +#elif defined CONFIG_MJAGUAR
 +#define MODULE_PROC_FAMILY "JAGUAR "
++#elif defined CONFIG_MEXCAVATOR
++#define MODULE_PROC_FAMILY "EXCAVATOR "
++#elif defined CONFIG_MZEN
++#define MODULE_PROC_FAMILY "ZEN "
  #elif defined CONFIG_MELAN
  #define MODULE_PROC_FAMILY "ELAN "
  #elif defined CONFIG_MCRUSOE
---- a/arch/x86/Kconfig.cpu	2015-08-30 14:34:09.000000000 -0400
-+++ b/arch/x86/Kconfig.cpu	2015-11-06 14:20:14.948369244 -0500
-@@ -137,9 +137,8 @@ config MPENTIUM4
+--- a/arch/x86/Kconfig.cpu	2016-12-11 14:17:54.000000000 -0500
++++ b/arch/x86/Kconfig.cpu	2017-01-06 20:46:14.004109597 -0500
+@@ -115,6 +115,7 @@ config MPENTIUMM
+ config MPENTIUM4
+ 	bool "Pentium-4/Celeron(P4-based)/Pentium-4 M/older Xeon"
+ 	depends on X86_32
++	select X86_P6_NOP
+ 	---help---
+ 	  Select this for Intel Pentium 4 chips.  This includes the
+ 	  Pentium 4, Pentium D, P4-based Celeron and Xeon, and
+@@ -147,9 +148,8 @@ config MPENTIUM4
  		-Paxville
  		-Dempsey
  
@@ -101,7 +141,7 @@ gcc version >=4.9
  	depends on X86_32
  	---help---
  	  Select this for an AMD K6-family processor.  Enables use of
-@@ -147,7 +146,7 @@ config MK6
+@@ -157,7 +157,7 @@ config MK6
  	  flags to GCC.
  
  config MK7
@@ -110,7 +150,7 @@ gcc version >=4.9
  	depends on X86_32
  	---help---
  	  Select this for an AMD Athlon K7-family processor.  Enables use of
-@@ -155,12 +154,69 @@ config MK7
+@@ -165,12 +165,83 @@ config MK7
  	  flags to GCC.
  
  config MK8
@@ -139,54 +179,77 @@ gcc version >=4.9
 +config MBARCELONA
 +	bool "AMD Barcelona"
 +	---help---
-+	  Select this for AMD Barcelona and newer processors.
++	  Select this for AMD Family 10h Barcelona processors.
 +
 +	  Enables -march=barcelona
 +
 +config MBOBCAT
 +	bool "AMD Bobcat"
 +	---help---
-+	  Select this for AMD Bobcat processors.
++	  Select this for AMD Family 14h Bobcat processors.
 +
 +	  Enables -march=btver1
 +
++config MJAGUAR
++	bool "AMD Jaguar"
++	---help---
++	  Select this for AMD Family 16h Jaguar processors.
++
++	  Enables -march=btver2
++
 +config MBULLDOZER
 +	bool "AMD Bulldozer"
 +	---help---
-+	  Select this for AMD Bulldozer processors.
++	  Select this for AMD Family 15h Bulldozer processors.
 +
 +	  Enables -march=bdver1
 +
 +config MPILEDRIVER
 +	bool "AMD Piledriver"
 +	---help---
-+	  Select this for AMD Piledriver processors.
++	  Select this for AMD Family 15h Piledriver processors.
 +
 +	  Enables -march=bdver2
 +
 +config MSTEAMROLLER
 +	bool "AMD Steamroller"
 +	---help---
-+	  Select this for AMD Steamroller processors.
++	  Select this for AMD Family 15h Steamroller processors.
 +
 +	  Enables -march=bdver3
 +
-+config MJAGUAR
-+	bool "AMD Jaguar"
++config MEXCAVATOR
++	bool "AMD Excavator"
 +	---help---
-+	  Select this for AMD Jaguar processors.
++	  Select this for AMD Family 15h Excavator processors.
 +
-+	  Enables -march=btver2
++	  Enables -march=bdver4
++
++config MZEN
++	bool "AMD Zen"
++	---help---
++	  Select this for AMD Family 17h Zen processors.
++
++	  Enables -march=znver1
 +
  config MCRUSOE
  	bool "Crusoe"
  	depends on X86_32
-@@ -251,8 +307,17 @@ config MPSC
+@@ -252,6 +323,7 @@ config MVIAC7
+ 
+ config MPSC
+ 	bool "Intel P4 / older Netburst based Xeon"
++	select X86_P6_NOP
+ 	depends on X86_64
+ 	---help---
+ 	  Optimize for Intel Pentium 4, Pentium D and older Nocona/Dempsey
+@@ -261,8 +333,19 @@ config MPSC
  	  using the cpu family field
  	  in /proc/cpuinfo. Family 15 is an older Xeon, Family 6 a newer one.
  
 +config MATOM
 +	bool "Intel Atom"
++	select X86_P6_NOP
 +	---help---
 +
 +	  Select this for the Intel Atom platform. Intel Atom CPUs have an
@@ -197,10 +260,11 @@ gcc version >=4.9
  config MCORE2
 -	bool "Core 2/newer Xeon"
 +	bool "Intel Core 2"
++	select X86_P6_NOP
  	---help---
  
  	  Select this for Intel Core 2 and newer Core 2 Xeons (Xeon 51xx and
-@@ -260,14 +325,71 @@ config MCORE2
+@@ -270,14 +353,79 @@ config MCORE2
  	  family in /proc/cpuinfo. Newer ones have 6 and older ones 15
  	  (not a typo)
  
@@ -210,6 +274,7 @@ gcc version >=4.9
 +
 +config MNEHALEM
 +	bool "Intel Nehalem"
++	select X86_P6_NOP
  	---help---
  
 -	  Select this for the Intel Atom platform. Intel Atom CPUs have an
@@ -222,6 +287,7 @@ gcc version >=4.9
 +
 +config MWESTMERE
 +	bool "Intel Westmere"
++	select X86_P6_NOP
 +	---help---
 +
 +	  Select this for the Intel Westmere formerly Nehalem-C family.
@@ -230,6 +296,7 @@ gcc version >=4.9
 +
 +config MSILVERMONT
 +	bool "Intel Silvermont"
++	select X86_P6_NOP
 +	---help---
 +
 +	  Select this for the Intel Silvermont platform.
@@ -238,6 +305,7 @@ gcc version >=4.9
 +
 +config MSANDYBRIDGE
 +	bool "Intel Sandy Bridge"
++	select X86_P6_NOP
 +	---help---
 +
 +	  Select this for 2nd Gen Core processors in the Sandy Bridge family.
@@ -246,6 +314,7 @@ gcc version >=4.9
 +
 +config MIVYBRIDGE
 +	bool "Intel Ivy Bridge"
++	select X86_P6_NOP
 +	---help---
 +
 +	  Select this for 3rd Gen Core processors in the Ivy Bridge family.
@@ -254,6 +323,7 @@ gcc version >=4.9
 +
 +config MHASWELL
 +	bool "Intel Haswell"
++	select X86_P6_NOP
 +	---help---
 +
 +	  Select this for 4th Gen Core processors in the Haswell family.
@@ -262,6 +332,7 @@ gcc version >=4.9
 +
 +config MBROADWELL
 +	bool "Intel Broadwell"
++	select X86_P6_NOP
 +	---help---
 +
 +	  Select this for 5th Gen Core processors in the Broadwell family.
@@ -270,6 +341,7 @@ gcc version >=4.9
 +
 +config MSKYLAKE
 +	bool "Intel Skylake"
++	select X86_P6_NOP
 +	---help---
 +
 +	  Select this for 6th Gen Core processors in the Skylake family.
@@ -278,7 +350,7 @@ gcc version >=4.9
  
  config GENERIC_CPU
  	bool "Generic-x86-64"
-@@ -276,6 +398,19 @@ config GENERIC_CPU
+@@ -286,6 +434,19 @@ config GENERIC_CPU
  	  Generic x86-64 CPU.
  	  Run equally well on all x86-64 CPUs.
  
@@ -298,16 +370,16 @@ gcc version >=4.9
  endchoice
  
  config X86_GENERIC
-@@ -300,7 +435,7 @@ config X86_INTERNODE_CACHE_SHIFT
+@@ -310,7 +471,7 @@ config X86_INTERNODE_CACHE_SHIFT
  config X86_L1_CACHE_SHIFT
  	int
  	default "7" if MPENTIUM4 || MPSC
 -	default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || X86_GENERIC || GENERIC_CPU
-+	default "6" if MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MJAGUAR || MPENTIUMM || MCORE2 || MNEHALEM || MWESTMERE || MSILVERMONT || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MNATIVE || MATOM || MVIAC7 || X86_GENERIC || GENERIC_CPU
++	default "6" if MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MJAGUAR || MPENTIUMM || MCORE2 || MNEHALEM || MWESTMERE || MSILVERMONT || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MNATIVE || MATOM || MVIAC7 || X86_GENERIC || GENERIC_CPU
  	default "4" if MELAN || M486 || MGEODEGX1
  	default "5" if MWINCHIP3D || MWINCHIPC6 || MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODE_LX
  
-@@ -331,11 +466,11 @@ config X86_ALIGNMENT_16
+@@ -341,45 +502,46 @@ config X86_ALIGNMENT_16
  
  config X86_INTEL_USERCOPY
  	def_bool y
@@ -321,7 +393,38 @@ gcc version >=4.9
  
  config X86_USE_3DNOW
  	def_bool y
-@@ -359,17 +494,17 @@ config X86_P6_NOP
+ 	depends on (MCYRIXIII || MK7 || MGEODE_LX) && !UML
+ 
+-#
+-# P6_NOPs are a relatively minor optimization that require a family >=
+-# 6 processor, except that it is broken on certain VIA chips.
+-# Furthermore, AMD chips prefer a totally different sequence of NOPs
+-# (which work on all CPUs).  In addition, it looks like Virtual PC
+-# does not understand them.
+-#
+-# As a result, disallow these if we're not compiling for X86_64 (these
+-# NOPs do work on all x86-64 capable chips); the list of processors in
+-# the right-hand clause are the cores that benefit from this optimization.
+-#
+ config X86_P6_NOP
+-	def_bool y
+-	depends on X86_64
+-	depends on (MCORE2 || MPENTIUM4 || MPSC)
++	default n
++	bool "Support for P6_NOPs on Intel chips"
++	depends on (MCORE2 || MPENTIUM4 || MPSC || MATOM || MNEHALEM || MWESTMERE || MSILVERMONT  || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MNATIVE)
++	---help---
++	P6_NOPs are a relatively minor optimization that require a family >=
++	6 processor, except that it is broken on certain VIA chips.
++	Furthermore, AMD chips prefer a totally different sequence of NOPs
++	(which work on all CPUs).  In addition, it looks like Virtual PC
++	does not understand them.
++
++	As a result, disallow these if we're not compiling for X86_64 (these
++	NOPs do work on all x86-64 capable chips); the list of processors in
++	the right-hand clause are the cores that benefit from this optimization.
++
++	Say Y if you have Intel CPU newer than Pentium Pro, N otherwise.
  
  config X86_TSC
  	def_bool y
@@ -338,13 +441,13 @@ gcc version >=4.9
  config X86_CMOV
  	def_bool y
 -	depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX)
-+	depends on (MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MJAGUAR || MK7 || MCORE2 || MNEHALEM || MWESTMERE || MSILVERMONT || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MNATIVE || MATOM || MGEODE_LX)
++	depends on (MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MJAGUAR || MK7 || MCORE2 || MNEHALEM || MWESTMERE || MSILVERMONT || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MNATIVE || MATOM || MGEODE_LX)
  
  config X86_MINIMUM_CPU_FAMILY
  	int
---- a/arch/x86/Makefile	2015-08-30 14:34:09.000000000 -0400
-+++ b/arch/x86/Makefile	2015-11-06 14:21:05.708983344 -0500
-@@ -94,13 +94,38 @@ else
+--- a/arch/x86/Makefile	2016-12-11 14:17:54.000000000 -0500
++++ b/arch/x86/Makefile	2017-01-06 20:44:36.603227283 -0500
+@@ -104,13 +104,40 @@ else
  	KBUILD_CFLAGS += $(call cc-option,-mskip-rax-setup)
  
          # FIXME - should be integrated in Makefile.cpu (Makefile_32.cpu)
@@ -354,10 +457,12 @@ gcc version >=4.9
 +        cflags-$(CONFIG_MK10) += $(call cc-option,-march=amdfam10)
 +        cflags-$(CONFIG_MBARCELONA) += $(call cc-option,-march=barcelona)
 +        cflags-$(CONFIG_MBOBCAT) += $(call cc-option,-march=btver1)
++        cflags-$(CONFIG_MJAGUAR) += $(call cc-option,-march=btver2)
 +        cflags-$(CONFIG_MBULLDOZER) += $(call cc-option,-march=bdver1)
 +        cflags-$(CONFIG_MPILEDRIVER) += $(call cc-option,-march=bdver2)
 +        cflags-$(CONFIG_MSTEAMROLLER) += $(call cc-option,-march=bdver3)
-+        cflags-$(CONFIG_MJAGUAR) += $(call cc-option,-march=btver2)
++        cflags-$(CONFIG_MEXCAVATOR) += $(call cc-option,-march=bdver4)
++        cflags-$(CONFIG_MZEN) += $(call cc-option,-march=znver1)
          cflags-$(CONFIG_MPSC) += $(call cc-option,-march=nocona)
  
          cflags-$(CONFIG_MCORE2) += \
@@ -386,9 +491,9 @@ gcc version >=4.9
          cflags-$(CONFIG_GENERIC_CPU) += $(call cc-option,-mtune=generic)
          KBUILD_CFLAGS += $(cflags-y)
  
---- a/arch/x86/Makefile_32.cpu	2015-08-30 14:34:09.000000000 -0400
-+++ b/arch/x86/Makefile_32.cpu	2015-11-06 14:21:43.604429077 -0500
-@@ -23,7 +23,16 @@ cflags-$(CONFIG_MK6)		+= -march=k6
+--- a/arch/x86/Makefile_32.cpu	2016-12-11 14:17:54.000000000 -0500
++++ b/arch/x86/Makefile_32.cpu	2017-01-06 20:44:36.603227283 -0500
+@@ -23,7 +23,18 @@ cflags-$(CONFIG_MK6)		+= -march=k6
  # Please note, that patches that add -march=athlon-xp and friends are pointless.
  # They make zero difference whatsosever to performance at this time.
  cflags-$(CONFIG_MK7)		+= -march=athlon
@@ -398,14 +503,16 @@ gcc version >=4.9
 +cflags-$(CONFIG_MK10)	+= $(call cc-option,-march=amdfam10,-march=athlon)
 +cflags-$(CONFIG_MBARCELONA)	+= $(call cc-option,-march=barcelona,-march=athlon)
 +cflags-$(CONFIG_MBOBCAT)	+= $(call cc-option,-march=btver1,-march=athlon)
++cflags-$(CONFIG_MJAGUAR)	+= $(call cc-option,-march=btver2,-march=athlon)
 +cflags-$(CONFIG_MBULLDOZER)	+= $(call cc-option,-march=bdver1,-march=athlon)
 +cflags-$(CONFIG_MPILEDRIVER)	+= $(call cc-option,-march=bdver2,-march=athlon)
 +cflags-$(CONFIG_MSTEAMROLLER)	+= $(call cc-option,-march=bdver3,-march=athlon)
-+cflags-$(CONFIG_MJAGUAR)	+= $(call cc-option,-march=btver2,-march=athlon)
++cflags-$(CONFIG_MEXCAVATOR)	+= $(call cc-option,-march=bdver4,-march=athlon)
++cflags-$(CONFIG_MZEN)	+= $(call cc-option,-march=znver1,-march=athlon)
  cflags-$(CONFIG_MCRUSOE)	+= -march=i686 $(align)-functions=0 $(align)-jumps=0 $(align)-loops=0
  cflags-$(CONFIG_MEFFICEON)	+= -march=i686 $(call tune,pentium3) $(align)-functions=0 $(align)-jumps=0 $(align)-loops=0
  cflags-$(CONFIG_MWINCHIPC6)	+= $(call cc-option,-march=winchip-c6,-march=i586)
-@@ -32,8 +41,16 @@ cflags-$(CONFIG_MCYRIXIII)	+= $(call cc-
+@@ -32,8 +43,16 @@ cflags-$(CONFIG_MCYRIXIII)	+= $(call cc-
  cflags-$(CONFIG_MVIAC3_2)	+= $(call cc-option,-march=c3-2,-march=i686)
  cflags-$(CONFIG_MVIAC7)		+= -march=i686
  cflags-$(CONFIG_MCORE2)		+= -march=i686 $(call tune,core2)
diff --git a/patches/uksm-0.1.2.6-for-v4.10.patch b/patches/uksm-0.1.2.6-for-v4.10.patch
new file mode 100644
index 0000000..3c32e5e
--- /dev/null
+++ b/patches/uksm-0.1.2.6-for-v4.10.patch
@@ -0,0 +1,6933 @@
+diff --git a/Documentation/vm/00-INDEX b/Documentation/vm/00-INDEX
+index 6a5e2a1..09eaa9a1 100644
+--- a/Documentation/vm/00-INDEX
++++ b/Documentation/vm/00-INDEX
+@@ -18,6 +18,8 @@ idle_page_tracking.txt
+ 	- description of the idle page tracking feature.
+ ksm.txt
+ 	- how to use the Kernel Samepage Merging feature.
++uksm.txt
++	- Introduction to Ultra KSM
+ numa
+ 	- information about NUMA specific code in the Linux vm.
+ numa_memory_policy.txt
+diff --git a/Documentation/vm/uksm.txt b/Documentation/vm/uksm.txt
+new file mode 100644
+index 0000000..b7a110f
+--- /dev/null
++++ b/Documentation/vm/uksm.txt
+@@ -0,0 +1,61 @@
++The Ultra Kernel Samepage Merging feature
++----------------------------------------------
++/*
++ * Ultra KSM. Copyright (C) 2011-2012 Nai Xia
++ *
++ * This is an improvement upon KSM. Some basic data structures and routines
++ * are borrowed from ksm.c .
++ *
++ * Its new features:
++ * 1. Full system scan:
++ *      It automatically scans all user processes' anonymous VMAs. Kernel-user
++ *      interaction to submit a memory area to KSM is no longer needed.
++ *
++ * 2. Rich area detection:
++ *      It automatically detects rich areas containing abundant duplicated
++ *      pages based. Rich areas are given a full scan speed. Poor areas are
++ *      sampled at a reasonable speed with very low CPU consumption.
++ *
++ * 3. Ultra Per-page scan speed improvement:
++ *      A new hash algorithm is proposed. As a result, on a machine with
++ *      Core(TM)2 Quad Q9300 CPU in 32-bit mode and 800MHZ DDR2 main memory, it
++ *      can scan memory areas that does not contain duplicated pages at speed of
++ *      627MB/sec ~ 2445MB/sec and can merge duplicated areas at speed of
++ *      477MB/sec ~ 923MB/sec.
++ *
++ * 4. Thrashing area avoidance:
++ *      Thrashing area(an VMA that has frequent Ksm page break-out) can be
++ *      filtered out. My benchmark shows it's more efficient than KSM's per-page
++ *      hash value based volatile page detection.
++ *
++ *
++ * 5. Misc changes upon KSM:
++ *      * It has a fully x86-opitmized memcmp dedicated for 4-byte-aligned page
++ *        comparison. It's much faster than default C version on x86.
++ *      * rmap_item now has an struct *page member to loosely cache a
++ *        address-->page mapping, which reduces too much time-costly
++ *        follow_page().
++ *      * The VMA creation/exit procedures are hooked to let the Ultra KSM know.
++ *      * try_to_merge_two_pages() now can revert a pte if it fails. No break_
++ *        ksm is needed for this case.
++ *
++ * 6. Full Zero Page consideration(contributed by Figo Zhang)
++ *    Now uksmd consider full zero pages as special pages and merge them to an
++ *    special unswappable uksm zero page.
++ */
++
++ChangeLog:
++
++2012-05-05 The creation of this Doc
++2012-05-08 UKSM 0.1.1.1 libc crash bug fix, api clean up, doc clean up.
++2012-05-28 UKSM 0.1.1.2 bug fix release
++2012-06-26 UKSM 0.1.2-beta1 first beta release for 0.1.2
++2012-07-2  UKSM 0.1.2-beta2
++2012-07-10 UKSM 0.1.2-beta3
++2012-07-26 UKSM 0.1.2 Fine grained speed control, more scan optimization.
++2012-10-13 UKSM 0.1.2.1 Bug fixes.
++2012-12-31 UKSM 0.1.2.2 Minor bug fixes.
++2014-07-02 UKSM 0.1.2.3 Fix a " __this_cpu_read() in preemptible bug".
++2015-04-22 UKSM 0.1.2.4 Fix a race condition that can sometimes trigger anonying warnings.
++2016-09-10 UKSM 0.1.2.5 Fix a bug in dedup ratio calculation.
++2017-02-26 UKSM 0.1.2.6 Fix a bug in hugetlbpage handling and a race bug with page migration. 
+diff --git a/fs/exec.c b/fs/exec.c
+index e579466..3da3dd68 100644
+--- a/fs/exec.c
++++ b/fs/exec.c
+@@ -57,6 +57,7 @@
+ #include <linux/oom.h>
+ #include <linux/compat.h>
+ #include <linux/vmalloc.h>
++#include <linux/ksm.h>
+ 
+ #include <linux/uaccess.h>
+ #include <asm/mmu_context.h>
+@@ -1336,6 +1337,7 @@ void setup_new_exec(struct linux_binprm * bprm)
+ 	/* An exec changes our domain. We are no longer part of the thread
+ 	   group */
+ 	current->self_exec_id++;
++
+ 	flush_signal_handlers(current, 0);
+ }
+ EXPORT_SYMBOL(setup_new_exec);
+diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c
+index 8a42849..47480dd 100644
+--- a/fs/proc/meminfo.c
++++ b/fs/proc/meminfo.c
+@@ -117,6 +117,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
+ 		   global_page_state(NR_KERNEL_STACK_KB));
+ 	show_val_kb(m, "PageTables:     ",
+ 		    global_page_state(NR_PAGETABLE));
++#ifdef CONFIG_UKSM
++	show_val_kb(m, "KsmZeroPages:     ",
++		    global_page_state(NR_UKSM_ZERO_PAGES));
++#endif
+ #ifdef CONFIG_QUICKLIST
+ 	show_val_kb(m, "Quicklists:     ", quicklist_total_size());
+ #endif
+diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h
+index 18af2bc..c536344 100644
+--- a/include/asm-generic/pgtable.h
++++ b/include/asm-generic/pgtable.h
+@@ -600,12 +600,25 @@ extern void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
+ extern void untrack_pfn_moved(struct vm_area_struct *vma);
+ #endif
+ 
++#ifdef CONFIG_UKSM
++static inline int is_uksm_zero_pfn(unsigned long pfn)
++{
++	extern unsigned long uksm_zero_pfn;
++        return pfn == uksm_zero_pfn;
++}
++#else
++static inline int is_uksm_zero_pfn(unsigned long pfn)
++{
++        return 0;
++}
++#endif
++
+ #ifdef __HAVE_COLOR_ZERO_PAGE
+ static inline int is_zero_pfn(unsigned long pfn)
+ {
+ 	extern unsigned long zero_pfn;
+ 	unsigned long offset_from_zero_pfn = pfn - zero_pfn;
+-	return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT);
++	return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT) || is_uksm_zero_pfn(pfn);
+ }
+ 
+ #define my_zero_pfn(addr)	page_to_pfn(ZERO_PAGE(addr))
+@@ -614,7 +627,7 @@ static inline int is_zero_pfn(unsigned long pfn)
+ static inline int is_zero_pfn(unsigned long pfn)
+ {
+ 	extern unsigned long zero_pfn;
+-	return pfn == zero_pfn;
++	return (pfn == zero_pfn) || (is_uksm_zero_pfn(pfn));
+ }
+ 
+ static inline unsigned long my_zero_pfn(unsigned long addr)
+diff --git a/include/linux/ksm.h b/include/linux/ksm.h
+index 481c8c4..5329b23 100644
+--- a/include/linux/ksm.h
++++ b/include/linux/ksm.h
+@@ -19,21 +19,6 @@ struct mem_cgroup;
+ #ifdef CONFIG_KSM
+ int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
+ 		unsigned long end, int advice, unsigned long *vm_flags);
+-int __ksm_enter(struct mm_struct *mm);
+-void __ksm_exit(struct mm_struct *mm);
+-
+-static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm)
+-{
+-	if (test_bit(MMF_VM_MERGEABLE, &oldmm->flags))
+-		return __ksm_enter(mm);
+-	return 0;
+-}
+-
+-static inline void ksm_exit(struct mm_struct *mm)
+-{
+-	if (test_bit(MMF_VM_MERGEABLE, &mm->flags))
+-		__ksm_exit(mm);
+-}
+ 
+ static inline struct stable_node *page_stable_node(struct page *page)
+ {
+@@ -63,6 +48,33 @@ struct page *ksm_might_need_to_copy(struct page *page,
+ int rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc);
+ void ksm_migrate_page(struct page *newpage, struct page *oldpage);
+ 
++#ifdef CONFIG_KSM_LEGACY
++int __ksm_enter(struct mm_struct *mm);
++void __ksm_exit(struct mm_struct *mm);
++static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm)
++{
++	if (test_bit(MMF_VM_MERGEABLE, &oldmm->flags))
++		return __ksm_enter(mm);
++	return 0;
++}
++
++static inline void ksm_exit(struct mm_struct *mm)
++{
++	if (test_bit(MMF_VM_MERGEABLE, &mm->flags))
++		__ksm_exit(mm);
++}
++
++#elif defined(CONFIG_UKSM)
++static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm)
++{
++	return 0;
++}
++
++static inline void ksm_exit(struct mm_struct *mm)
++{
++}
++#endif /* !CONFIG_UKSM */
++
+ #else  /* !CONFIG_KSM */
+ 
+ static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm)
+@@ -105,4 +117,6 @@ static inline void ksm_migrate_page(struct page *newpage, struct page *oldpage)
+ #endif /* CONFIG_MMU */
+ #endif /* !CONFIG_KSM */
+ 
++#include <linux/uksm.h>
++
+ #endif /* __LINUX_KSM_H */
+diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
+index 808751d..f2c9f6c 100644
+--- a/include/linux/mm_types.h
++++ b/include/linux/mm_types.h
+@@ -358,6 +358,9 @@ struct vm_area_struct {
+ 	struct mempolicy *vm_policy;	/* NUMA policy for the VMA */
+ #endif
+ 	struct vm_userfaultfd_ctx vm_userfaultfd_ctx;
++#ifdef CONFIG_UKSM
++	struct vma_slot *uksm_vma_slot;
++#endif
+ };
+ 
+ struct core_thread {
+diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
+index f4aac87..bff211e 100644
+--- a/include/linux/mmzone.h
++++ b/include/linux/mmzone.h
+@@ -138,6 +138,9 @@ enum zone_stat_item {
+ 	NUMA_OTHER,		/* allocation from other node */
+ #endif
+ 	NR_FREE_CMA_PAGES,
++#ifdef CONFIG_UKSM
++	NR_UKSM_ZERO_PAGES,
++#endif
+ 	NR_VM_ZONE_STAT_ITEMS };
+ 
+ enum node_stat_item {
+@@ -843,7 +846,7 @@ static inline int is_highmem_idx(enum zone_type idx)
+ }
+ 
+ /**
+- * is_highmem - helper function to quickly check if a struct zone is a 
++ * is_highmem - helper function to quickly check if a struct zone is a
+  *              highmem zone or not.  This is an attempt to keep references
+  *              to ZONE_{DMA/NORMAL/HIGHMEM/etc} in general code to a minimum.
+  * @zone - pointer to struct zone variable
+diff --git a/include/linux/sradix-tree.h b/include/linux/sradix-tree.h
+new file mode 100644
+index 0000000..6780fdb
+--- /dev/null
++++ b/include/linux/sradix-tree.h
+@@ -0,0 +1,77 @@
++#ifndef _LINUX_SRADIX_TREE_H
++#define _LINUX_SRADIX_TREE_H
++
++
++#define INIT_SRADIX_TREE(root, mask)					\
++do {									\
++	(root)->height = 0;						\
++	(root)->gfp_mask = (mask);					\
++	(root)->rnode = NULL;						\
++} while (0)
++
++#define ULONG_BITS	(sizeof(unsigned long) * 8)
++#define SRADIX_TREE_INDEX_BITS  (8 /* CHAR_BIT */ * sizeof(unsigned long))
++//#define SRADIX_TREE_MAP_SHIFT	6
++//#define SRADIX_TREE_MAP_SIZE	(1UL << SRADIX_TREE_MAP_SHIFT)
++//#define SRADIX_TREE_MAP_MASK	(SRADIX_TREE_MAP_SIZE-1)
++
++struct sradix_tree_node {
++	unsigned int	height;		/* Height from the bottom */
++	unsigned int	count;		
++	unsigned int	fulls;		/* Number of full sublevel trees */ 
++	struct sradix_tree_node *parent;
++	void *stores[0];
++};
++
++/* A simple radix tree implementation */
++struct sradix_tree_root {
++        unsigned int            height;
++        struct sradix_tree_node *rnode;
++
++	/* Where found to have available empty stores in its sublevels */
++        struct sradix_tree_node *enter_node;
++	unsigned int shift;
++	unsigned int stores_size;
++	unsigned int mask;
++	unsigned long min;	/* The first hole index */
++	unsigned long num;
++	//unsigned long *height_to_maxindex;
++
++	/* How the node is allocated and freed. */
++	struct sradix_tree_node *(*alloc)(void); 
++	void (*free)(struct sradix_tree_node *node);
++
++	/* When a new node is added and removed */
++	void (*extend)(struct sradix_tree_node *parent, struct sradix_tree_node *child);
++	void (*assign)(struct sradix_tree_node *node, unsigned index, void *item);
++	void (*rm)(struct sradix_tree_node *node, unsigned offset);
++};
++
++struct sradix_tree_path {
++	struct sradix_tree_node *node;
++	int offset;
++};
++
++static inline 
++void init_sradix_tree_root(struct sradix_tree_root *root, unsigned long shift)
++{
++	root->height = 0;
++	root->rnode = NULL;
++	root->shift = shift;
++	root->stores_size = 1UL << shift;
++	root->mask = root->stores_size - 1;
++}
++
++
++extern void *sradix_tree_next(struct sradix_tree_root *root,
++		       struct sradix_tree_node *node, unsigned long index,
++		       int (*iter)(void *, unsigned long));
++
++extern int sradix_tree_enter(struct sradix_tree_root *root, void **item, int num);
++
++extern void sradix_tree_delete_from_leaf(struct sradix_tree_root *root, 
++			struct sradix_tree_node *node, unsigned long index);
++
++extern void *sradix_tree_lookup(struct sradix_tree_root *root, unsigned long index);
++
++#endif /* _LINUX_SRADIX_TREE_H */
+diff --git a/include/linux/uksm.h b/include/linux/uksm.h
+new file mode 100644
+index 0000000..825f05e
+--- /dev/null
++++ b/include/linux/uksm.h
+@@ -0,0 +1,149 @@
++#ifndef __LINUX_UKSM_H
++#define __LINUX_UKSM_H
++/*
++ * Memory merging support.
++ *
++ * This code enables dynamic sharing of identical pages found in different
++ * memory areas, even if they are not shared by fork().
++ */
++
++/* if !CONFIG_UKSM this file should not be compiled at all. */
++#ifdef CONFIG_UKSM
++
++#include <linux/bitops.h>
++#include <linux/mm.h>
++#include <linux/pagemap.h>
++#include <linux/rmap.h>
++#include <linux/sched.h>
++
++extern unsigned long zero_pfn __read_mostly;
++extern unsigned long uksm_zero_pfn __read_mostly;
++extern struct page *empty_uksm_zero_page;
++
++/* must be done before linked to mm */
++extern void uksm_vma_add_new(struct vm_area_struct *vma);
++extern void uksm_remove_vma(struct vm_area_struct *vma);
++
++#define UKSM_SLOT_NEED_SORT	(1 << 0)
++#define UKSM_SLOT_NEED_RERAND 	(1 << 1)
++#define UKSM_SLOT_SCANNED     	(1 << 2) /* It's scanned in this round */
++#define UKSM_SLOT_FUL_SCANNED 	(1 << 3)
++#define UKSM_SLOT_IN_UKSM 	(1 << 4)
++
++struct vma_slot {
++	struct sradix_tree_node *snode;
++	unsigned long sindex;
++
++	struct list_head slot_list;
++	unsigned long fully_scanned_round;
++	unsigned long dedup_num;
++	unsigned long pages_scanned;
++	unsigned long this_sampled;
++	unsigned long last_scanned;
++	unsigned long pages_to_scan;
++	struct scan_rung *rung;
++	struct page **rmap_list_pool;
++	unsigned int *pool_counts;
++	unsigned long pool_size;
++	struct vm_area_struct *vma;
++	struct mm_struct *mm;
++	unsigned long ctime_j;
++	unsigned long pages;
++	unsigned long flags;
++	unsigned long pages_cowed; /* pages cowed this round */
++	unsigned long pages_merged; /* pages merged this round */
++	unsigned long pages_bemerged;
++
++	/* when it has page merged in this eval round */
++	struct list_head dedup_list;
++};
++
++static inline void uksm_unmap_zero_page(pte_t pte)
++{
++	if (pte_pfn(pte) == uksm_zero_pfn)
++		__dec_zone_page_state(empty_uksm_zero_page, NR_UKSM_ZERO_PAGES);
++}
++
++static inline void uksm_map_zero_page(pte_t pte)
++{
++	if (pte_pfn(pte) == uksm_zero_pfn)
++		__inc_zone_page_state(empty_uksm_zero_page, NR_UKSM_ZERO_PAGES);
++}
++
++static inline void uksm_cow_page(struct vm_area_struct *vma, struct page *page)
++{
++	if (vma->uksm_vma_slot && PageKsm(page))
++		vma->uksm_vma_slot->pages_cowed++;
++}
++
++static inline void uksm_cow_pte(struct vm_area_struct *vma, pte_t pte)
++{
++	if (vma->uksm_vma_slot && pte_pfn(pte) == uksm_zero_pfn)
++		vma->uksm_vma_slot->pages_cowed++;
++}
++
++static inline int uksm_flags_can_scan(unsigned long vm_flags)
++{
++#ifdef VM_SAO
++		if (vm_flags & VM_SAO)
++			return 0;
++#endif
++
++	return !(vm_flags & (VM_PFNMAP | VM_IO  | VM_DONTEXPAND |
++			     VM_HUGETLB | VM_MIXEDMAP | VM_SHARED
++			     | VM_MAYSHARE | VM_GROWSUP | VM_GROWSDOWN));
++}
++
++static inline void uksm_vm_flags_mod(unsigned long *vm_flags_p)
++{
++	if (uksm_flags_can_scan(*vm_flags_p))
++		*vm_flags_p |= VM_MERGEABLE;
++}
++
++/*
++ * Just a wrapper for BUG_ON for where ksm_zeropage must not be. TODO: it will
++ * be removed when uksm zero page patch is stable enough.
++ */
++static inline void uksm_bugon_zeropage(pte_t pte)
++{
++	BUG_ON(pte_pfn(pte) == uksm_zero_pfn);
++}
++#else
++static inline void uksm_vma_add_new(struct vm_area_struct *vma)
++{
++}
++
++static inline void uksm_remove_vma(struct vm_area_struct *vma)
++{
++}
++
++static inline void uksm_unmap_zero_page(pte_t pte)
++{
++}
++
++static inline void uksm_map_zero_page(pte_t pte)
++{
++}
++
++static inline void uksm_cow_page(struct vm_area_struct *vma, struct page *page)
++{
++}
++
++static inline void uksm_cow_pte(struct vm_area_struct *vma, pte_t pte)
++{
++}
++
++static inline int uksm_flags_can_scan(unsigned long vm_flags)
++{
++	return 0;
++}
++
++static inline void uksm_vm_flags_mod(unsigned long *vm_flags_p)
++{
++}
++
++static inline void uksm_bugon_zeropage(pte_t pte)
++{
++}
++#endif /* !CONFIG_UKSM */
++#endif /* __LINUX_UKSM_H */
+diff --git a/kernel/fork.c b/kernel/fork.c
+index 11c5c8a..78cd12d 100644
+--- a/kernel/fork.c
++++ b/kernel/fork.c
+@@ -606,7 +606,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
+ 				goto fail_nomem;
+ 			charge = len;
+ 		}
+-		tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
++		tmp = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
+ 		if (!tmp)
+ 			goto fail_nomem;
+ 		*tmp = *mpnt;
+@@ -659,7 +659,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
+ 		__vma_link_rb(mm, tmp, rb_link, rb_parent);
+ 		rb_link = &tmp->vm_rb.rb_right;
+ 		rb_parent = &tmp->vm_rb;
+-
++		uksm_vma_add_new(tmp);
+ 		mm->map_count++;
+ 		retval = copy_page_range(mm, oldmm, mpnt);
+ 
+diff --git a/lib/Makefile b/lib/Makefile
+index bc4073a..dbdb5ee 100644
+--- a/lib/Makefile
++++ b/lib/Makefile
+@@ -17,7 +17,7 @@ KCOV_INSTRUMENT_debugobjects.o := n
+ KCOV_INSTRUMENT_dynamic_debug.o := n
+ 
+ lib-y := ctype.o string.o vsprintf.o cmdline.o \
+-	 rbtree.o radix-tree.o dump_stack.o timerqueue.o\
++	 rbtree.o radix-tree.o sradix-tree.o dump_stack.o timerqueue.o\
+ 	 idr.o int_sqrt.o extable.o \
+ 	 sha1.o chacha20.o md5.o irq_regs.o argv_split.o \
+ 	 flex_proportions.o ratelimit.o show_mem.o \
+diff --git a/lib/sradix-tree.c b/lib/sradix-tree.c
+new file mode 100644
+index 0000000..8d06329
+--- /dev/null
++++ b/lib/sradix-tree.c
+@@ -0,0 +1,476 @@
++#include <linux/errno.h>
++#include <linux/mm.h>
++#include <linux/mman.h>
++#include <linux/spinlock.h>
++#include <linux/slab.h>
++#include <linux/gcd.h>
++#include <linux/sradix-tree.h>
++
++static inline int sradix_node_full(struct sradix_tree_root *root, struct sradix_tree_node *node)
++{
++	return node->fulls == root->stores_size || 
++		(node->height == 1 && node->count == root->stores_size);
++}
++
++/*
++ *	Extend a sradix tree so it can store key @index.
++ */
++static int sradix_tree_extend(struct sradix_tree_root *root, unsigned long index)
++{
++	struct sradix_tree_node *node;
++	unsigned int height;
++
++	if (unlikely(root->rnode == NULL)) {
++		if (!(node = root->alloc()))
++			return -ENOMEM;
++
++		node->height = 1;
++		root->rnode = node;
++		root->height = 1;
++	}
++
++	/* Figure out what the height should be.  */
++	height = root->height;
++	index >>= root->shift * height;
++
++	while (index) {
++		index >>= root->shift;
++		height++;
++	}
++
++	while (height > root->height) {
++		unsigned int newheight;
++		if (!(node = root->alloc()))
++			return -ENOMEM;
++
++		/* Increase the height.  */
++		node->stores[0] = root->rnode;
++		root->rnode->parent = node;
++		if (root->extend)
++			root->extend(node, root->rnode);
++
++		newheight = root->height + 1;
++		node->height = newheight;
++		node->count = 1;
++		if (sradix_node_full(root, root->rnode))
++			node->fulls = 1;
++
++		root->rnode = node;
++		root->height = newheight;
++	}
++
++	return 0;
++}
++
++/*
++ * Search the next item from the current node, that is not NULL
++ * and can satify root->iter().
++ */
++void *sradix_tree_next(struct sradix_tree_root *root,
++		       struct sradix_tree_node *node, unsigned long index,
++		       int (*iter)(void *item, unsigned long height))
++{
++	unsigned long offset;
++	void *item;
++
++	if (unlikely(node == NULL)) {
++		node = root->rnode;
++		for (offset = 0; offset < root->stores_size; offset++) {
++			item = node->stores[offset];
++			if (item && (!iter || iter(item, node->height)))
++				break;
++		}
++
++		if (unlikely(offset >= root->stores_size))
++			return NULL;
++
++		if (node->height == 1)
++			return item;
++		else
++			goto go_down;
++	}
++
++	while (node) {
++		offset = (index & root->mask) + 1;					
++		for (;offset < root->stores_size; offset++) {
++			item = node->stores[offset];
++			if (item && (!iter || iter(item, node->height)))
++				break;
++		}
++
++		if (offset < root->stores_size)
++			break;
++
++		node = node->parent;
++		index >>= root->shift;
++	}
++
++	if (!node)
++		return NULL;
++
++	while (node->height > 1) {
++go_down:
++		node = item;
++		for (offset = 0; offset < root->stores_size; offset++) {
++			item = node->stores[offset];
++			if (item && (!iter || iter(item, node->height)))
++				break;
++		}
++
++		if (unlikely(offset >= root->stores_size))
++			return NULL;
++	}
++
++	BUG_ON(offset > root->stores_size);
++
++	return item;
++}
++
++/*
++ * Blindly insert the item to the tree. Typically, we reuse the
++ * first empty store item.
++ */
++int sradix_tree_enter(struct sradix_tree_root *root, void **item, int num)
++{
++	unsigned long index;
++	unsigned int height;
++	struct sradix_tree_node *node, *tmp = NULL;
++	int offset, offset_saved;
++	void **store = NULL;
++	int error, i, j, shift;
++
++go_on:
++	index = root->min;
++
++	if (root->enter_node && !sradix_node_full(root, root->enter_node)) {
++		node = root->enter_node;
++		BUG_ON((index >> (root->shift * root->height)));
++	} else {
++		node = root->rnode;
++		if (node == NULL || (index >> (root->shift * root->height))
++		    || sradix_node_full(root, node)) {
++			error = sradix_tree_extend(root, index);
++			if (error)
++				return error;
++
++			node = root->rnode;
++		}
++	}
++
++
++	height = node->height;
++	shift = (height - 1) * root->shift;
++	offset = (index >> shift) & root->mask;
++	while (shift > 0) {
++		offset_saved = offset;
++		for (; offset < root->stores_size; offset++) {
++			store = &node->stores[offset];
++			tmp = *store;
++
++			if (!tmp || !sradix_node_full(root, tmp))
++				break;
++		}
++		BUG_ON(offset >= root->stores_size);
++
++		if (offset != offset_saved) {
++			index += (offset - offset_saved) << shift;
++			index &= ~((1UL << shift) - 1);
++		}
++
++		if (!tmp) {
++			if (!(tmp = root->alloc()))
++				return -ENOMEM;
++
++			tmp->height = shift / root->shift;
++			*store = tmp;
++			tmp->parent = node;
++			node->count++;
++//			if (root->extend)
++//				root->extend(node, tmp);
++		}
++
++		node = tmp;
++		shift -= root->shift;
++		offset = (index >> shift) & root->mask;
++	}
++
++	BUG_ON(node->height != 1);
++
++
++	store = &node->stores[offset];
++	for (i = 0, j = 0;
++	      j < root->stores_size - node->count && 
++	      i < root->stores_size - offset && j < num; i++) {
++		if (!store[i]) {
++			store[i] = item[j];
++			if (root->assign)
++				root->assign(node, index + i, item[j]);
++			j++;
++		}
++	}
++
++	node->count += j;
++	root->num += j;
++	num -= j;
++
++	while (sradix_node_full(root, node)) {
++		node = node->parent;
++		if (!node)
++			break;
++
++		node->fulls++;
++	}
++
++	if (unlikely(!node)) {
++		/* All nodes are full */
++		root->min = 1 << (root->height * root->shift);
++		root->enter_node = NULL;
++	} else {
++		root->min = index + i - 1;
++		root->min |= (1UL << (node->height - 1)) - 1;
++		root->min++;
++		root->enter_node = node;
++	}
++
++	if (num) {
++		item += j;
++		goto go_on;
++	}
++
++	return 0;
++}
++
++
++/**
++ *	sradix_tree_shrink    -    shrink height of a sradix tree to minimal
++ *      @root		sradix tree root
++ *  
++ */
++static inline void sradix_tree_shrink(struct sradix_tree_root *root)
++{
++	/* try to shrink tree height */
++	while (root->height > 1) {
++		struct sradix_tree_node *to_free = root->rnode;
++
++		/*
++		 * The candidate node has more than one child, or its child
++		 * is not at the leftmost store, we cannot shrink.
++		 */
++		if (to_free->count != 1 || !to_free->stores[0])
++			break;
++
++		root->rnode = to_free->stores[0];
++		root->rnode->parent = NULL;
++		root->height--;
++		if (unlikely(root->enter_node == to_free)) {
++			root->enter_node = NULL;
++		}
++		root->free(to_free);
++	}
++}
++
++/*
++ * Del the item on the known leaf node and index
++ */
++void sradix_tree_delete_from_leaf(struct sradix_tree_root *root, 
++				  struct sradix_tree_node *node, unsigned long index)
++{
++	unsigned int offset;
++	struct sradix_tree_node *start, *end;
++
++	BUG_ON(node->height != 1);
++
++	start = node;
++	while (node && !(--node->count))
++		node = node->parent;
++
++	end = node;
++	if (!node) {
++		root->rnode = NULL;
++		root->height = 0;
++		root->min = 0;
++		root->num = 0;
++		root->enter_node = NULL;
++	} else {
++		offset = (index >> (root->shift * (node->height - 1))) & root->mask;
++		if (root->rm)
++			root->rm(node, offset);
++		node->stores[offset] = NULL;
++		root->num--;
++		if (root->min > index) {
++			root->min = index;
++			root->enter_node = node;
++		}
++	}
++
++	if (start != end) {
++		do {
++			node = start;
++			start = start->parent;
++			if (unlikely(root->enter_node == node))
++				root->enter_node = end;
++			root->free(node);
++		} while (start != end);
++
++		/*
++		 * Note that shrink may free "end", so enter_node still need to
++		 * be checked inside.
++		 */
++		sradix_tree_shrink(root);
++	} else if (node->count == root->stores_size - 1) {
++		/* It WAS a full leaf node. Update the ancestors */
++		node = node->parent;
++		while (node) {
++			node->fulls--;
++			if (node->fulls != root->stores_size - 1)
++				break;
++
++			node = node->parent;
++		}
++	}
++}
++
++void *sradix_tree_lookup(struct sradix_tree_root *root, unsigned long index)
++{
++	unsigned int height, offset;
++	struct sradix_tree_node *node;
++	int shift;
++
++	node = root->rnode;
++	if (node == NULL || (index >> (root->shift * root->height)))
++		return NULL;
++
++	height = root->height;
++	shift = (height - 1) * root->shift;
++
++	do {
++		offset = (index >> shift) & root->mask;
++		node = node->stores[offset];
++		if (!node)
++			return NULL;
++
++		shift -= root->shift;
++	} while (shift >= 0);
++
++	return node;
++}
++
++/*
++ * Return the item if it exists, otherwise create it in place
++ * and return the created item.
++ */
++void *sradix_tree_lookup_create(struct sradix_tree_root *root, 
++			unsigned long index, void *(*item_alloc)(void))
++{
++	unsigned int height, offset;
++	struct sradix_tree_node *node, *tmp;
++	void *item;
++	int shift, error;
++
++	if (root->rnode == NULL || (index >> (root->shift * root->height))) {
++		if (item_alloc) {
++			error = sradix_tree_extend(root, index);
++			if (error)
++				return NULL;
++		} else {
++			return NULL;
++		}
++	}
++
++	node = root->rnode;
++	height = root->height;
++	shift = (height - 1) * root->shift;
++
++	do {
++		offset = (index >> shift) & root->mask;
++		if (!node->stores[offset]) {
++			if (!(tmp = root->alloc()))
++				return NULL;
++
++			tmp->height = shift / root->shift;
++			node->stores[offset] = tmp;
++			tmp->parent = node;
++			node->count++;
++			node = tmp;
++		} else {
++			node = node->stores[offset];
++		}
++
++		shift -= root->shift;
++	} while (shift > 0);
++
++	BUG_ON(node->height != 1);
++	offset = index & root->mask;
++	if (node->stores[offset]) {
++		return node->stores[offset];
++	} else if (item_alloc) {
++		if (!(item = item_alloc()))
++			return NULL;
++
++		node->stores[offset] = item;
++
++		/*
++		 * NOTE: we do NOT call root->assign here, since this item is
++		 * newly created by us having no meaning. Caller can call this
++		 * if it's necessary to do so.
++		 */
++
++		node->count++;
++		root->num++;
++
++		while (sradix_node_full(root, node)) {
++			node = node->parent;
++			if (!node)
++				break;
++
++			node->fulls++;
++		}
++
++		if (unlikely(!node)) {
++			/* All nodes are full */
++			root->min = 1 << (root->height * root->shift);
++		} else {
++			if (root->min == index) {
++				root->min |= (1UL << (node->height - 1)) - 1;
++				root->min++;
++				root->enter_node = node;
++			}
++		}
++
++		return item;
++	} else {
++		return NULL;
++	}
++
++}
++
++int sradix_tree_delete(struct sradix_tree_root *root, unsigned long index)
++{
++	unsigned int height, offset;
++	struct sradix_tree_node *node;
++	int shift;
++
++	node = root->rnode;
++	if (node == NULL || (index >> (root->shift * root->height)))
++		return -ENOENT;
++
++	height = root->height;
++	shift = (height - 1) * root->shift;
++
++	do {
++		offset = (index >> shift) & root->mask;
++		node = node->stores[offset];
++		if (!node)
++			return -ENOENT;
++
++		shift -= root->shift;
++	} while (shift > 0);
++
++	offset = index & root->mask;
++	if (!node->stores[offset])
++		return -ENOENT;
++
++	sradix_tree_delete_from_leaf(root, node, index);
++
++	return 0;
++}
+diff --git a/mm/Kconfig b/mm/Kconfig
+index 9b8fccb..d8a2f50 100644
+--- a/mm/Kconfig
++++ b/mm/Kconfig
+@@ -340,6 +340,32 @@ config KSM
+ 	  See Documentation/vm/ksm.txt for more information: KSM is inactive
+ 	  until a program has madvised that an area is MADV_MERGEABLE, and
+ 	  root has set /sys/kernel/mm/ksm/run to 1 (if CONFIG_SYSFS is set).
++choice
++	prompt "Choose UKSM/KSM strategy"
++	default UKSM
++	depends on KSM
++	help
++	  This option allows to select a UKSM/KSM stragety.
++
++config UKSM
++	bool "Ultra-KSM for page merging"
++	depends on KSM
++	help
++	UKSM is inspired by the Linux kernel project \u2014 KSM(Kernel Same
++	page Merging), but with a fundamentally rewritten core algorithm. With
++	an advanced algorithm, UKSM now can transparently scans all anonymously
++	mapped user space applications with an significantly improved scan speed
++	and CPU efficiency. Since KVM is friendly to KSM, KVM can also benefit from
++	UKSM. Now UKSM has its first stable release and first real world enterprise user.
++	For more information, please goto its project page.
++	(www.kerneldedup.org)
++
++config KSM_LEGACY
++	bool "Legacy KSM implementation"
++	depends on KSM
++	help
++	The legacy KSM implementation from Redhat.
++endchoice
+ 
+ config DEFAULT_MMAP_MIN_ADDR
+         int "Low address space to protect from user allocation"
+diff --git a/mm/Makefile b/mm/Makefile
+index 295bd7a..a47ed9b 100644
+--- a/mm/Makefile
++++ b/mm/Makefile
+@@ -63,7 +63,8 @@ obj-$(CONFIG_SPARSEMEM)	+= sparse.o
+ obj-$(CONFIG_SPARSEMEM_VMEMMAP) += sparse-vmemmap.o
+ obj-$(CONFIG_SLOB) += slob.o
+ obj-$(CONFIG_MMU_NOTIFIER) += mmu_notifier.o
+-obj-$(CONFIG_KSM) += ksm.o
++obj-$(CONFIG_KSM_LEGACY) += ksm.o
++obj-$(CONFIG_UKSM) += uksm.o
+ obj-$(CONFIG_PAGE_POISONING) += page_poison.o
+ obj-$(CONFIG_SLAB) += slab.o
+ obj-$(CONFIG_SLUB) += slub.o
+diff --git a/mm/memory.c b/mm/memory.c
+index 6bf2b47..ee62eea 100644
+--- a/mm/memory.c
++++ b/mm/memory.c
+@@ -124,6 +124,25 @@ unsigned long highest_memmap_pfn __read_mostly;
+ 
+ EXPORT_SYMBOL(zero_pfn);
+ 
++#ifdef CONFIG_UKSM
++unsigned long uksm_zero_pfn __read_mostly;
++EXPORT_SYMBOL_GPL(uksm_zero_pfn);
++struct page *empty_uksm_zero_page;
++
++static int __init setup_uksm_zero_page(void)
++{
++	empty_uksm_zero_page = alloc_pages(__GFP_ZERO & ~__GFP_MOVABLE, 0);
++	if (!empty_uksm_zero_page)
++		panic("Oh boy, that early out of memory?");
++
++	SetPageReserved(empty_uksm_zero_page);
++	uksm_zero_pfn = page_to_pfn(empty_uksm_zero_page);
++
++	return 0;
++}
++core_initcall(setup_uksm_zero_page);
++#endif
++
+ /*
+  * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
+  */
+@@ -135,6 +154,7 @@ static int __init init_zero_pfn(void)
+ core_initcall(init_zero_pfn);
+ 
+ 
++
+ #if defined(SPLIT_RSS_COUNTING)
+ 
+ void sync_mm_rss(struct mm_struct *mm)
+@@ -916,6 +936,11 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
+ 		get_page(page);
+ 		page_dup_rmap(page, false);
+ 		rss[mm_counter(page)]++;
++		
++		/* Should return NULL in vm_normal_page() */
++		uksm_bugon_zeropage(pte);
++	} else {
++		uksm_map_zero_page(pte);
+ 	}
+ 
+ out_set_pte:
+@@ -1150,8 +1175,10 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
+ 			ptent = ptep_get_and_clear_full(mm, addr, pte,
+ 							tlb->fullmm);
+ 			tlb_remove_tlb_entry(tlb, pte, addr);
+-			if (unlikely(!page))
++			if (unlikely(!page)) {
++				uksm_unmap_zero_page(ptent);
+ 				continue;
++			}
+ 
+ 			if (!PageAnon(page)) {
+ 				if (pte_dirty(ptent)) {
+@@ -2010,8 +2037,10 @@ static inline void cow_user_page(struct page *dst, struct page *src, unsigned lo
+ 			clear_page(kaddr);
+ 		kunmap_atomic(kaddr);
+ 		flush_dcache_page(dst);
+-	} else
++	} else {
+ 		copy_user_highpage(dst, src, va, vma);
++		uksm_cow_page(vma, src);
++	}
+ }
+ 
+ static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
+@@ -2160,6 +2189,7 @@ static int wp_page_copy(struct vm_fault *vmf)
+ 							      vmf->address);
+ 		if (!new_page)
+ 			goto oom;
++		uksm_cow_pte(vma, vmf->orig_pte);
+ 	} else {
+ 		new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
+ 				vmf->address);
+@@ -2186,7 +2216,9 @@ static int wp_page_copy(struct vm_fault *vmf)
+ 						mm_counter_file(old_page));
+ 				inc_mm_counter_fast(mm, MM_ANONPAGES);
+ 			}
++			uksm_bugon_zeropage(vmf->orig_pte);
+ 		} else {
++			uksm_unmap_zero_page(vmf->orig_pte);
+ 			inc_mm_counter_fast(mm, MM_ANONPAGES);
+ 		}
+ 		flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
+diff --git a/mm/mmap.c b/mm/mmap.c
+index dc4291d..a96fac9 100644
+--- a/mm/mmap.c
++++ b/mm/mmap.c
+@@ -44,6 +44,7 @@
+ #include <linux/userfaultfd_k.h>
+ #include <linux/moduleparam.h>
+ #include <linux/pkeys.h>
++#include <linux/ksm.h>
+ 
+ #include <linux/uaccess.h>
+ #include <asm/cacheflush.h>
+@@ -172,6 +173,7 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
+ 	if (vma->vm_file)
+ 		fput(vma->vm_file);
+ 	mpol_put(vma_policy(vma));
++	uksm_remove_vma(vma);
+ 	kmem_cache_free(vm_area_cachep, vma);
+ 	return next;
+ }
+@@ -675,9 +677,16 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
+ 	long adjust_next = 0;
+ 	int remove_next = 0;
+ 
++/*
++ * to avoid deadlock, ksm_remove_vma must be done before any spin_lock is
++ * acquired
++ */
++	uksm_remove_vma(vma);
++
+ 	if (next && !insert) {
+ 		struct vm_area_struct *exporter = NULL, *importer = NULL;
+ 
++		uksm_remove_vma(next);
+ 		if (end >= next->vm_end) {
+ 			/*
+ 			 * vma expands, overlapping all the next, and
+@@ -810,6 +819,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
+ 		end_changed = true;
+ 	}
+ 	vma->vm_pgoff = pgoff;
++
+ 	if (adjust_next) {
+ 		next->vm_start += adjust_next << PAGE_SHIFT;
+ 		next->vm_pgoff += adjust_next;
+@@ -915,6 +925,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
+ 		if (remove_next == 2) {
+ 			remove_next = 1;
+ 			end = next->vm_end;
++			uksm_remove_vma(next);
+ 			goto again;
+ 		}
+ 		else if (next)
+@@ -941,10 +952,14 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
+ 			 */
+ 			VM_WARN_ON(mm->highest_vm_end != end);
+ 		}
++	} else {
++		if (next && !insert)
++			uksm_vma_add_new(next);
+ 	}
+ 	if (insert && file)
+ 		uprobe_mmap(insert);
+ 
++ 	uksm_vma_add_new(vma);
+ 	validate_mm(mm);
+ 
+ 	return 0;
+@@ -1360,6 +1375,9 @@ unsigned long do_mmap(struct file *file, unsigned long addr,
+ 	vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) |
+ 			mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
+ 
++	/* If uksm is enabled, we add VM_MERGABLE to new VMAs. */
++	uksm_vm_flags_mod(&vm_flags);
++
+ 	if (flags & MAP_LOCKED)
+ 		if (!can_do_mlock())
+ 			return -EPERM;
+@@ -1698,6 +1716,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
+ 			allow_write_access(file);
+ 	}
+ 	file = vma->vm_file;
++	uksm_vma_add_new(vma);
+ out:
+ 	perf_event_mmap(vma);
+ 
+@@ -1739,6 +1758,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
+ 	if (vm_flags & VM_DENYWRITE)
+ 		allow_write_access(file);
+ free_vma:
++	uksm_remove_vma(vma);
+ 	kmem_cache_free(vm_area_cachep, vma);
+ unacct_error:
+ 	if (charged)
+@@ -2544,6 +2564,8 @@ static int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
+ 	else
+ 		err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
+ 
++	uksm_vma_add_new(new);
++
+ 	/* Success. */
+ 	if (!err)
+ 		return 0;
+@@ -2822,6 +2844,7 @@ static int do_brk(unsigned long addr, unsigned long request)
+ 		return 0;
+ 
+ 	flags = VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
++	uksm_vm_flags_mod(&flags);
+ 
+ 	error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
+ 	if (offset_in_page(error))
+@@ -2879,6 +2902,7 @@ static int do_brk(unsigned long addr, unsigned long request)
+ 	vma->vm_flags = flags;
+ 	vma->vm_page_prot = vm_get_page_prot(flags);
+ 	vma_link(mm, vma, prev, rb_link, rb_parent);
++	uksm_vma_add_new(vma);
+ out:
+ 	perf_event_mmap(vma);
+ 	mm->total_vm += len >> PAGE_SHIFT;
+@@ -2917,6 +2941,12 @@ void exit_mmap(struct mm_struct *mm)
+ 	/* mm's last user has gone, and its about to be pulled down */
+ 	mmu_notifier_release(mm);
+ 
++	/*
++	 * Taking write lock on mmap_sem does not harm others,
++	 * but it's crucial for uksm to avoid races.
++	 */
++	down_write(&mm->mmap_sem);
++
+ 	if (mm->locked_vm) {
+ 		vma = mm->mmap;
+ 		while (vma) {
+@@ -2952,6 +2982,11 @@ void exit_mmap(struct mm_struct *mm)
+ 		vma = remove_vma(vma);
+ 	}
+ 	vm_unacct_memory(nr_accounted);
++
++	mm->mmap = NULL;
++	mm->mm_rb = RB_ROOT;
++	vmacache_invalidate(mm);
++	up_write(&mm->mmap_sem);
+ }
+ 
+ /* Insert vm structure into process list sorted by address
+@@ -3061,6 +3096,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
+ 			new_vma->vm_ops->open(new_vma);
+ 		vma_link(mm, new_vma, prev, rb_link, rb_parent);
+ 		*need_rmap_locks = false;
++		uksm_vma_add_new(new_vma);
+ 	}
+ 	return new_vma;
+ 
+@@ -3208,6 +3244,7 @@ static struct vm_area_struct *__install_special_mapping(
+ 	vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT);
+ 
+ 	perf_event_mmap(vma);
++	uksm_vma_add_new(vma);
+ 
+ 	return vma;
+ 
+diff --git a/mm/rmap.c b/mm/rmap.c
+index 91619fd..3619420 100644
+--- a/mm/rmap.c
++++ b/mm/rmap.c
+@@ -1109,9 +1109,9 @@ void page_move_anon_rmap(struct page *page, struct vm_area_struct *vma)
+ 
+ /**
+  * __page_set_anon_rmap - set up new anonymous rmap
+- * @page:	Page to add to rmap	
++ * @page:	Page to add to rmap
+  * @vma:	VM area to add page to.
+- * @address:	User virtual address of the mapping	
++ * @address:	User virtual address of the mapping
+  * @exclusive:	the page is exclusively owned by the current process
+  */
+ static void __page_set_anon_rmap(struct page *page,
+diff --git a/mm/uksm.c b/mm/uksm.c
+new file mode 100644
+index 0000000..d4596e1
+--- /dev/null
++++ b/mm/uksm.c
+@@ -0,0 +1,5580 @@
++/*
++ * Ultra KSM. Copyright (C) 2011-2012 Nai Xia
++ *
++ * This is an improvement upon KSM. Some basic data structures and routines
++ * are borrowed from ksm.c .
++ *
++ * Its new features:
++ * 1. Full system scan:
++ *      It automatically scans all user processes' anonymous VMAs. Kernel-user
++ *      interaction to submit a memory area to KSM is no longer needed.
++ *
++ * 2. Rich area detection:
++ *      It automatically detects rich areas containing abundant duplicated
++ *      pages based. Rich areas are given a full scan speed. Poor areas are
++ *      sampled at a reasonable speed with very low CPU consumption.
++ *
++ * 3. Ultra Per-page scan speed improvement:
++ *      A new hash algorithm is proposed. As a result, on a machine with
++ *      Core(TM)2 Quad Q9300 CPU in 32-bit mode and 800MHZ DDR2 main memory, it
++ *      can scan memory areas that does not contain duplicated pages at speed of
++ *      627MB/sec ~ 2445MB/sec and can merge duplicated areas at speed of
++ *      477MB/sec ~ 923MB/sec.
++ *
++ * 4. Thrashing area avoidance:
++ *      Thrashing area(an VMA that has frequent Ksm page break-out) can be
++ *      filtered out. My benchmark shows it's more efficient than KSM's per-page
++ *      hash value based volatile page detection.
++ *
++ *
++ * 5. Misc changes upon KSM:
++ *      * It has a fully x86-opitmized memcmp dedicated for 4-byte-aligned page
++ *        comparison. It's much faster than default C version on x86.
++ *      * rmap_item now has an struct *page member to loosely cache a
++ *        address-->page mapping, which reduces too much time-costly
++ *        follow_page().
++ *      * The VMA creation/exit procedures are hooked to let the Ultra KSM know.
++ *      * try_to_merge_two_pages() now can revert a pte if it fails. No break_
++ *        ksm is needed for this case.
++ *
++ * 6. Full Zero Page consideration(contributed by Figo Zhang)
++ *    Now uksmd consider full zero pages as special pages and merge them to an
++ *    special unswappable uksm zero page.
++ */
++
++#include <linux/errno.h>
++#include <linux/mm.h>
++#include <linux/fs.h>
++#include <linux/mman.h>
++#include <linux/sched.h>
++#include <linux/rwsem.h>
++#include <linux/pagemap.h>
++#include <linux/rmap.h>
++#include <linux/spinlock.h>
++#include <linux/jhash.h>
++#include <linux/delay.h>
++#include <linux/kthread.h>
++#include <linux/wait.h>
++#include <linux/slab.h>
++#include <linux/rbtree.h>
++#include <linux/memory.h>
++#include <linux/mmu_notifier.h>
++#include <linux/swap.h>
++#include <linux/ksm.h>
++#include <linux/crypto.h>
++#include <linux/scatterlist.h>
++#include <crypto/hash.h>
++#include <linux/random.h>
++#include <linux/math64.h>
++#include <linux/gcd.h>
++#include <linux/freezer.h>
++#include <linux/sradix-tree.h>
++
++#include <asm/tlbflush.h>
++#include "internal.h"
++
++#ifdef CONFIG_X86
++#undef memcmp
++
++#ifdef CONFIG_X86_32
++#define memcmp memcmpx86_32
++/*
++ * Compare 4-byte-aligned address s1 and s2, with length n
++ */
++int memcmpx86_32(void *s1, void *s2, size_t n)
++{
++	size_t num = n / 4;
++	register int res;
++
++	__asm__ __volatile__
++	(
++	 "testl %3,%3\n\t"
++	 "repe; cmpsd\n\t"
++	 "je        1f\n\t"
++	 "sbbl      %0,%0\n\t"
++	 "orl       $1,%0\n"
++	 "1:"
++	 : "=&a" (res), "+&S" (s1), "+&D" (s2), "+&c" (num)
++	 : "0" (0)
++	 : "cc");
++
++	return res;
++}
++
++/*
++ * Check the page is all zero ?
++ */
++static int is_full_zero(const void *s1, size_t len)
++{
++	unsigned char same;
++
++	len /= 4;
++
++	__asm__ __volatile__
++	("repe; scasl;"
++	 "sete %0"
++	 : "=qm" (same), "+D" (s1), "+c" (len)
++	 : "a" (0)
++	 : "cc");
++
++	return same;
++}
++
++
++#elif defined(CONFIG_X86_64)
++#define memcmp memcmpx86_64
++/*
++ * Compare 8-byte-aligned address s1 and s2, with length n
++ */
++int memcmpx86_64(void *s1, void *s2, size_t n)
++{
++	size_t num = n / 8;
++	register int res;
++
++	__asm__ __volatile__
++	(
++	 "testq %q3,%q3\n\t"
++	 "repe; cmpsq\n\t"
++	 "je        1f\n\t"
++	 "sbbq      %q0,%q0\n\t"
++	 "orq       $1,%q0\n"
++	 "1:"
++	 : "=&a" (res), "+&S" (s1), "+&D" (s2), "+&c" (num)
++	 : "0" (0)
++	 : "cc");
++
++	return res;
++}
++
++static int is_full_zero(const void *s1, size_t len)
++{
++	unsigned char same;
++
++	len /= 8;
++
++	__asm__ __volatile__
++	("repe; scasq;"
++	 "sete %0"
++	 : "=qm" (same), "+D" (s1), "+c" (len)
++	 : "a" (0)
++	 : "cc");
++
++	return same;
++}
++
++#endif
++#else
++static int is_full_zero(const void *s1, size_t len)
++{
++	unsigned long *src = s1;
++	int i;
++
++	len /= sizeof(*src);
++
++	for (i = 0; i < len; i++) {
++		if (src[i])
++			return 0;
++	}
++
++	return 1;
++}
++#endif
++
++#define UKSM_RUNG_ROUND_FINISHED  (1 << 0)
++#define TIME_RATIO_SCALE	10000
++
++#define SLOT_TREE_NODE_SHIFT	8
++#define SLOT_TREE_NODE_STORE_SIZE	(1UL << SLOT_TREE_NODE_SHIFT)
++struct slot_tree_node {
++	unsigned long size;
++	struct sradix_tree_node snode;
++	void *stores[SLOT_TREE_NODE_STORE_SIZE];
++};
++
++static struct kmem_cache *slot_tree_node_cachep;
++
++static struct sradix_tree_node *slot_tree_node_alloc(void)
++{
++	struct slot_tree_node *p;
++	p = kmem_cache_zalloc(slot_tree_node_cachep, GFP_KERNEL |
++			      __GFP_NORETRY | __GFP_NOWARN);
++	if (!p)
++		return NULL;
++
++	return &p->snode;
++}
++
++static void slot_tree_node_free(struct sradix_tree_node *node)
++{
++	struct slot_tree_node *p;
++
++	p = container_of(node, struct slot_tree_node, snode);
++	kmem_cache_free(slot_tree_node_cachep, p);
++}
++
++static void slot_tree_node_extend(struct sradix_tree_node *parent,
++				  struct sradix_tree_node *child)
++{
++	struct slot_tree_node *p, *c;
++
++	p = container_of(parent, struct slot_tree_node, snode);
++	c = container_of(child, struct slot_tree_node, snode);
++
++	p->size += c->size;
++}
++
++void slot_tree_node_assign(struct sradix_tree_node *node,
++			   unsigned index, void *item)
++{
++	struct vma_slot *slot = item;
++	struct slot_tree_node *cur;
++
++	slot->snode = node;
++	slot->sindex = index;
++
++	while (node) {
++		cur = container_of(node, struct slot_tree_node, snode);
++		cur->size += slot->pages;
++		node = node->parent;
++	}
++}
++
++void slot_tree_node_rm(struct sradix_tree_node *node, unsigned offset)
++{
++	struct vma_slot *slot;
++	struct slot_tree_node *cur;
++	unsigned long pages;
++
++	if (node->height == 1) {
++		slot = node->stores[offset];
++		pages = slot->pages;
++	} else {
++		cur = container_of(node->stores[offset],
++				   struct slot_tree_node, snode);
++		pages = cur->size;
++	}
++
++	while (node) {
++		cur = container_of(node, struct slot_tree_node, snode);
++		cur->size -= pages;
++		node = node->parent;
++	}
++}
++
++unsigned long slot_iter_index;
++int slot_iter(void *item,  unsigned long height)
++{
++	struct slot_tree_node *node;
++	struct vma_slot *slot;
++
++	if (height == 1) {
++		slot = item;
++		if (slot_iter_index < slot->pages) {
++			/*in this one*/
++			return 1;
++		} else {
++			slot_iter_index -= slot->pages;
++			return 0;
++		}
++
++	} else {
++		node = container_of(item, struct slot_tree_node, snode);
++		if (slot_iter_index < node->size) {
++			/*in this one*/
++			return 1;
++		} else {
++			slot_iter_index -= node->size;
++			return 0;
++		}
++	}
++}
++
++
++static inline void slot_tree_init_root(struct sradix_tree_root *root)
++{
++	init_sradix_tree_root(root, SLOT_TREE_NODE_SHIFT);
++	root->alloc = slot_tree_node_alloc;
++	root->free = slot_tree_node_free;
++	root->extend = slot_tree_node_extend;
++	root->assign = slot_tree_node_assign;
++	root->rm = slot_tree_node_rm;
++}
++
++void slot_tree_init(void)
++{
++	slot_tree_node_cachep = kmem_cache_create("slot_tree_node",
++				sizeof(struct slot_tree_node), 0,
++				SLAB_PANIC | SLAB_RECLAIM_ACCOUNT,
++				NULL);
++}
++
++
++/* Each rung of this ladder is a list of VMAs having a same scan ratio */
++struct scan_rung {
++	//struct list_head scanned_list;
++	struct sradix_tree_root vma_root;
++	struct sradix_tree_root vma_root2;
++
++	struct vma_slot *current_scan;
++	unsigned long current_offset;
++
++	/*
++	 * The initial value for current_offset, it should loop over
++	 * [0~ step - 1] to let all slot have its chance to be scanned.
++	 */
++	unsigned long offset_init;
++	unsigned long step; /* dynamic step for current_offset */
++	unsigned int flags;
++	unsigned long pages_to_scan;
++	//unsigned long fully_scanned_slots;
++	/*
++	 * a little bit tricky - if cpu_time_ratio > 0, then the value is the
++	 * the cpu time ratio it can spend in rung_i for every scan
++	 * period. if < 0, then it is the cpu time ratio relative to the
++	 * max cpu percentage user specified. Both in unit of
++	 * 1/TIME_RATIO_SCALE
++	 */
++	int cpu_ratio;
++
++	/*
++	 * How long it will take for all slots in this rung to be fully
++	 * scanned? If it's zero, we don't care about the cover time:
++	 * it's fully scanned.
++	 */
++	unsigned int cover_msecs;
++	//unsigned long vma_num;
++	//unsigned long pages; /* Sum of all slot's pages in rung */
++};
++
++/**
++ * node of either the stable or unstale rbtree
++ *
++ */
++struct tree_node {
++	struct rb_node node; /* link in the main (un)stable rbtree */
++	struct rb_root sub_root; /* rb_root for sublevel collision rbtree */
++	u32 hash;
++	unsigned long count; /* TODO: merged with sub_root */
++	struct list_head all_list; /* all tree nodes in stable/unstable tree */
++};
++
++/**
++ * struct stable_node - node of the stable rbtree
++ * @node: rb node of this ksm page in the stable tree
++ * @hlist: hlist head of rmap_items using this ksm page
++ * @kpfn: page frame number of this ksm page
++ */
++struct stable_node {
++	struct rb_node node; /* link in sub-rbtree */
++	struct tree_node *tree_node; /* it's tree node root in stable tree, NULL if it's in hell list */
++	struct hlist_head hlist;
++	unsigned long kpfn;
++	u32 hash_max; /* if ==0 then it's not been calculated yet */
++	struct list_head all_list; /* in a list for all stable nodes */
++};
++
++/**
++ * struct node_vma - group rmap_items linked in a same stable
++ * node together.
++ */
++struct node_vma {
++	union {
++		struct vma_slot *slot;
++		unsigned long key;  /* slot is used as key sorted on hlist */
++	};
++	struct hlist_node hlist;
++	struct hlist_head rmap_hlist;
++	struct stable_node *head;
++};
++
++/**
++ * struct rmap_item - reverse mapping item for virtual addresses
++ * @rmap_list: next rmap_item in mm_slot's singly-linked rmap_list
++ * @anon_vma: pointer to anon_vma for this mm,address, when in stable tree
++ * @mm: the memory structure this rmap_item is pointing into
++ * @address: the virtual address this rmap_item tracks (+ flags in low bits)
++ * @node: rb node of this rmap_item in the unstable tree
++ * @head: pointer to stable_node heading this list in the stable tree
++ * @hlist: link into hlist of rmap_items hanging off that stable_node
++ */
++struct rmap_item {
++	struct vma_slot *slot;
++	struct page *page;
++	unsigned long address;	/* + low bits used for flags below */
++	unsigned long hash_round;
++	unsigned long entry_index;
++	union {
++		struct {/* when in unstable tree */
++			struct rb_node node;
++			struct tree_node *tree_node;
++			u32 hash_max;
++		};
++		struct { /* when in stable tree */
++			struct node_vma *head;
++			struct hlist_node hlist;
++			struct anon_vma *anon_vma;
++		};
++	};
++} __attribute__((aligned(4)));
++
++struct rmap_list_entry {
++	union {
++		struct rmap_item *item;
++		unsigned long addr;
++	};
++	/* lowest bit is used for is_addr tag */
++} __attribute__((aligned(4))); /* 4 aligned to fit in to pages*/
++
++
++/* Basic data structure definition ends */
++
++
++/*
++ * Flags for rmap_item to judge if it's listed in the stable/unstable tree.
++ * The flags use the low bits of rmap_item.address
++ */
++#define UNSTABLE_FLAG	0x1
++#define STABLE_FLAG	0x2
++#define get_rmap_addr(x)	((x)->address & PAGE_MASK)
++
++/*
++ * rmap_list_entry helpers
++ */
++#define IS_ADDR_FLAG	1
++#define is_addr(ptr)		((unsigned long)(ptr) & IS_ADDR_FLAG)
++#define set_is_addr(ptr)	((ptr) |= IS_ADDR_FLAG)
++#define get_clean_addr(ptr)	(((ptr) & ~(__typeof__(ptr))IS_ADDR_FLAG))
++
++
++/*
++ * High speed caches for frequently allocated and freed structs
++ */
++static struct kmem_cache *rmap_item_cache;
++static struct kmem_cache *stable_node_cache;
++static struct kmem_cache *node_vma_cache;
++static struct kmem_cache *vma_slot_cache;
++static struct kmem_cache *tree_node_cache;
++#define UKSM_KMEM_CACHE(__struct, __flags) kmem_cache_create("uksm_"#__struct,\
++		sizeof(struct __struct), __alignof__(struct __struct),\
++		(__flags), NULL)
++
++/* Array of all scan_rung, uksm_scan_ladder[0] having the minimum scan ratio */
++#define SCAN_LADDER_SIZE 4
++static struct scan_rung uksm_scan_ladder[SCAN_LADDER_SIZE];
++
++/* The evaluation rounds uksmd has finished */
++static unsigned long long uksm_eval_round = 1;
++
++/*
++ * we add 1 to this var when we consider we should rebuild the whole
++ * unstable tree.
++ */
++static unsigned long uksm_hash_round = 1;
++
++/*
++ * How many times the whole memory is scanned.
++ */
++static unsigned long long fully_scanned_round = 1;
++
++/* The total number of virtual pages of all vma slots */
++static u64 uksm_pages_total;
++
++/* The number of pages has been scanned since the start up */
++static u64 uksm_pages_scanned;
++
++static u64 scanned_virtual_pages;
++
++/* The number of pages has been scanned since last encode_benefit call */
++static u64 uksm_pages_scanned_last;
++
++/* If the scanned number is tooo large, we encode it here */
++static u64 pages_scanned_stored;
++
++static unsigned long pages_scanned_base;
++
++/* The number of nodes in the stable tree */
++static unsigned long uksm_pages_shared;
++
++/* The number of page slots additionally sharing those nodes */
++static unsigned long uksm_pages_sharing;
++
++/* The number of nodes in the unstable tree */
++static unsigned long uksm_pages_unshared;
++
++/*
++ * Milliseconds ksmd should sleep between scans,
++ * >= 100ms to be consistent with
++ * scan_time_to_sleep_msec()
++ */
++static unsigned int uksm_sleep_jiffies;
++
++/* The real value for the uksmd next sleep */
++static unsigned int uksm_sleep_real;
++
++/* Saved value for user input uksm_sleep_jiffies when it's enlarged */
++static unsigned int uksm_sleep_saved;
++
++/* Max percentage of cpu utilization ksmd can take to scan in one batch */
++static unsigned int uksm_max_cpu_percentage;
++
++static int uksm_cpu_governor;
++
++static char *uksm_cpu_governor_str[4] = { "full", "medium", "low", "quiet" };
++
++struct uksm_cpu_preset_s {
++	int cpu_ratio[SCAN_LADDER_SIZE];
++	unsigned int cover_msecs[SCAN_LADDER_SIZE];
++	unsigned int max_cpu; /* percentage */
++};
++
++struct uksm_cpu_preset_s uksm_cpu_preset[4] = {
++	{ {20, 40, -2500, -10000}, {1000, 500, 200, 50}, 95},
++	{ {20, 30, -2500, -10000}, {1000, 500, 400, 100}, 50},
++	{ {10, 20, -5000, -10000}, {1500, 1000, 1000, 250}, 20},
++	{ {10, 20, 40, 75}, {2000, 1000, 1000, 1000}, 1},
++};
++
++/* The default value for uksm_ema_page_time if it's not initialized */
++#define UKSM_PAGE_TIME_DEFAULT	500
++
++/*cost to scan one page by expotional moving average in nsecs */
++static unsigned long uksm_ema_page_time = UKSM_PAGE_TIME_DEFAULT;
++
++/* The expotional moving average alpha weight, in percentage. */
++#define EMA_ALPHA	20
++
++/*
++ * The threshold used to filter out thrashing areas,
++ * If it == 0, filtering is disabled, otherwise it's the percentage up-bound
++ * of the thrashing ratio of all areas. Any area with a bigger thrashing ratio
++ * will be considered as having a zero duplication ratio.
++ */
++static unsigned int uksm_thrash_threshold = 50;
++
++/* How much dedup ratio is considered to be abundant*/
++static unsigned int uksm_abundant_threshold = 10;
++
++/* All slots having merged pages in this eval round. */
++struct list_head vma_slot_dedup = LIST_HEAD_INIT(vma_slot_dedup);
++
++/* How many times the ksmd has slept since startup */
++static unsigned long long uksm_sleep_times;
++
++#define UKSM_RUN_STOP	0
++#define UKSM_RUN_MERGE	1
++static unsigned int uksm_run = 1;
++
++static DECLARE_WAIT_QUEUE_HEAD(uksm_thread_wait);
++static DEFINE_MUTEX(uksm_thread_mutex);
++
++/*
++ * List vma_slot_new is for newly created vma_slot waiting to be added by
++ * ksmd. If one cannot be added(e.g. due to it's too small), it's moved to
++ * vma_slot_noadd. vma_slot_del is the list for vma_slot whose corresponding
++ * VMA has been removed/freed.
++ */
++struct list_head vma_slot_new = LIST_HEAD_INIT(vma_slot_new);
++struct list_head vma_slot_noadd = LIST_HEAD_INIT(vma_slot_noadd);
++struct list_head vma_slot_del = LIST_HEAD_INIT(vma_slot_del);
++static DEFINE_SPINLOCK(vma_slot_list_lock);
++
++/* The unstable tree heads */
++static struct rb_root root_unstable_tree = RB_ROOT;
++
++/*
++ * All tree_nodes are in a list to be freed at once when unstable tree is
++ * freed after each scan round.
++ */
++static struct list_head unstable_tree_node_list =
++				LIST_HEAD_INIT(unstable_tree_node_list);
++
++/* List contains all stable nodes */
++static struct list_head stable_node_list = LIST_HEAD_INIT(stable_node_list);
++
++/*
++ * When the hash strength is changed, the stable tree must be delta_hashed and
++ * re-structured. We use two set of below structs to speed up the
++ * re-structuring of stable tree.
++ */
++static struct list_head
++stable_tree_node_list[2] = {LIST_HEAD_INIT(stable_tree_node_list[0]),
++			    LIST_HEAD_INIT(stable_tree_node_list[1])};
++
++static struct list_head *stable_tree_node_listp = &stable_tree_node_list[0];
++static struct rb_root root_stable_tree[2] = {RB_ROOT, RB_ROOT};
++static struct rb_root *root_stable_treep = &root_stable_tree[0];
++static unsigned long stable_tree_index;
++
++/* The hash strength needed to hash a full page */
++#define HASH_STRENGTH_FULL		(PAGE_SIZE / sizeof(u32))
++
++/* The hash strength needed for loop-back hashing */
++#define HASH_STRENGTH_MAX		(HASH_STRENGTH_FULL + 10)
++
++/* The random offsets in a page */
++static u32 *random_nums;
++
++/* The hash strength */
++static unsigned long hash_strength = HASH_STRENGTH_FULL >> 4;
++
++/* The delta value each time the hash strength increases or decreases */
++static unsigned long hash_strength_delta;
++#define HASH_STRENGTH_DELTA_MAX	5
++
++/* The time we have saved due to random_sample_hash */
++static u64 rshash_pos;
++
++/* The time we have wasted due to hash collision */
++static u64 rshash_neg;
++
++struct uksm_benefit {
++	u64 pos;
++	u64 neg;
++	u64 scanned;
++	unsigned long base;
++} benefit;
++
++/*
++ * The relative cost of memcmp, compared to 1 time unit of random sample
++ * hash, this value is tested when ksm module is initialized
++ */
++static unsigned long memcmp_cost;
++
++static unsigned long  rshash_neg_cont_zero;
++static unsigned long  rshash_cont_obscure;
++
++/* The possible states of hash strength adjustment heuristic */
++enum rshash_states {
++		RSHASH_STILL,
++		RSHASH_TRYUP,
++		RSHASH_TRYDOWN,
++		RSHASH_NEW,
++		RSHASH_PRE_STILL,
++};
++
++/* The possible direction we are about to adjust hash strength */
++enum rshash_direct {
++	GO_UP,
++	GO_DOWN,
++	OBSCURE,
++	STILL,
++};
++
++/* random sampling hash state machine */
++static struct {
++	enum rshash_states state;
++	enum rshash_direct pre_direct;
++	u8 below_count;
++	/* Keep a lookup window of size 5, iff above_count/below_count > 3
++	 * in this window we stop trying.
++	 */
++	u8 lookup_window_index;
++	u64 stable_benefit;
++	unsigned long turn_point_down;
++	unsigned long turn_benefit_down;
++	unsigned long turn_point_up;
++	unsigned long turn_benefit_up;
++	unsigned long stable_point;
++} rshash_state;
++
++/*zero page hash table, hash_strength [0 ~ HASH_STRENGTH_MAX]*/
++static u32 *zero_hash_table;
++
++static inline struct node_vma *alloc_node_vma(void)
++{
++	struct node_vma *node_vma;
++	node_vma = kmem_cache_zalloc(node_vma_cache, GFP_KERNEL |
++				     __GFP_NORETRY | __GFP_NOWARN);
++	if (node_vma) {
++		INIT_HLIST_HEAD(&node_vma->rmap_hlist);
++		INIT_HLIST_NODE(&node_vma->hlist);
++	}
++	return node_vma;
++}
++
++static inline void free_node_vma(struct node_vma *node_vma)
++{
++	kmem_cache_free(node_vma_cache, node_vma);
++}
++
++
++static inline struct vma_slot *alloc_vma_slot(void)
++{
++	struct vma_slot *slot;
++
++	/*
++	 * In case ksm is not initialized by now.
++	 * Oops, we need to consider the call site of uksm_init() in the future.
++	 */
++	if (!vma_slot_cache)
++		return NULL;
++
++	slot = kmem_cache_zalloc(vma_slot_cache, GFP_KERNEL |
++				 __GFP_NORETRY | __GFP_NOWARN);
++	if (slot) {
++		INIT_LIST_HEAD(&slot->slot_list);
++		INIT_LIST_HEAD(&slot->dedup_list);
++		slot->flags |= UKSM_SLOT_NEED_RERAND;
++	}
++	return slot;
++}
++
++static inline void free_vma_slot(struct vma_slot *vma_slot)
++{
++	kmem_cache_free(vma_slot_cache, vma_slot);
++}
++
++
++
++static inline struct rmap_item *alloc_rmap_item(void)
++{
++	struct rmap_item *rmap_item;
++
++	rmap_item = kmem_cache_zalloc(rmap_item_cache, GFP_KERNEL |
++				      __GFP_NORETRY | __GFP_NOWARN);
++	if (rmap_item) {
++		/* bug on lowest bit is not clear for flag use */
++		BUG_ON(is_addr(rmap_item));
++	}
++	return rmap_item;
++}
++
++static inline void free_rmap_item(struct rmap_item *rmap_item)
++{
++	rmap_item->slot = NULL;	/* debug safety */
++	kmem_cache_free(rmap_item_cache, rmap_item);
++}
++
++static inline struct stable_node *alloc_stable_node(void)
++{
++	struct stable_node *node;
++	node = kmem_cache_alloc(stable_node_cache, GFP_KERNEL |
++				__GFP_NORETRY | __GFP_NOWARN);
++	if (!node)
++		return NULL;
++
++	INIT_HLIST_HEAD(&node->hlist);
++	list_add(&node->all_list, &stable_node_list);
++	return node;
++}
++
++static inline void free_stable_node(struct stable_node *stable_node)
++{
++	list_del(&stable_node->all_list);
++	kmem_cache_free(stable_node_cache, stable_node);
++}
++
++static inline struct tree_node *alloc_tree_node(struct list_head *list)
++{
++	struct tree_node *node;
++	node = kmem_cache_zalloc(tree_node_cache, GFP_KERNEL |
++				 __GFP_NORETRY | __GFP_NOWARN);
++	if (!node)
++		return NULL;
++
++	list_add(&node->all_list, list);
++	return node;
++}
++
++static inline void free_tree_node(struct tree_node *node)
++{
++	list_del(&node->all_list);
++	kmem_cache_free(tree_node_cache, node);
++}
++
++static void uksm_drop_anon_vma(struct rmap_item *rmap_item)
++{
++	struct anon_vma *anon_vma = rmap_item->anon_vma;
++
++	put_anon_vma(anon_vma);
++}
++
++
++/**
++ * Remove a stable node from stable_tree, may unlink from its tree_node and
++ * may remove its parent tree_node if no other stable node is pending.
++ *
++ * @stable_node 	The node need to be removed
++ * @unlink_rb 		Will this node be unlinked from the rbtree?
++ * @remove_tree_	node Will its tree_node be removed if empty?
++ */
++static void remove_node_from_stable_tree(struct stable_node *stable_node,
++					 int unlink_rb,  int remove_tree_node)
++{
++	struct node_vma *node_vma;
++	struct rmap_item *rmap_item;
++	struct hlist_node *n;
++
++	if (!hlist_empty(&stable_node->hlist)) {
++		hlist_for_each_entry_safe(node_vma, n,
++					  &stable_node->hlist, hlist) {
++			hlist_for_each_entry(rmap_item, &node_vma->rmap_hlist, hlist) {
++				uksm_pages_sharing--;
++
++				uksm_drop_anon_vma(rmap_item);
++				rmap_item->address &= PAGE_MASK;
++			}
++			free_node_vma(node_vma);
++			cond_resched();
++		}
++
++		/* the last one is counted as shared */
++		uksm_pages_shared--;
++		uksm_pages_sharing++;
++	}
++
++	if (stable_node->tree_node && unlink_rb) {
++		rb_erase(&stable_node->node,
++			 &stable_node->tree_node->sub_root);
++
++		if (RB_EMPTY_ROOT(&stable_node->tree_node->sub_root) &&
++		    remove_tree_node) {
++			rb_erase(&stable_node->tree_node->node,
++				 root_stable_treep);
++			free_tree_node(stable_node->tree_node);
++		} else {
++			stable_node->tree_node->count--;
++		}
++	}
++
++	free_stable_node(stable_node);
++}
++
++
++/*
++ * get_uksm_page: checks if the page indicated by the stable node
++ * is still its ksm page, despite having held no reference to it.
++ * In which case we can trust the content of the page, and it
++ * returns the gotten page; but if the page has now been zapped,
++ * remove the stale node from the stable tree and return NULL.
++ *
++ * You would expect the stable_node to hold a reference to the ksm page.
++ * But if it increments the page's count, swapping out has to wait for
++ * ksmd to come around again before it can free the page, which may take
++ * seconds or even minutes: much too unresponsive.  So instead we use a
++ * "keyhole reference": access to the ksm page from the stable node peeps
++ * out through its keyhole to see if that page still holds the right key,
++ * pointing back to this stable node.  This relies on freeing a PageAnon
++ * page to reset its page->mapping to NULL, and relies on no other use of
++ * a page to put something that might look like our key in page->mapping.
++ *
++ * include/linux/pagemap.h page_cache_get_speculative() is a good reference,
++ * but this is different - made simpler by uksm_thread_mutex being held, but
++ * interesting for assuming that no other use of the struct page could ever
++ * put our expected_mapping into page->mapping (or a field of the union which
++ * coincides with page->mapping).  The RCU calls are not for KSM at all, but
++ * to keep the page_count protocol described with page_cache_get_speculative.
++ *
++ * Note: it is possible that get_uksm_page() will return NULL one moment,
++ * then page the next, if the page is in between page_freeze_refs() and
++ * page_unfreeze_refs(): this shouldn't be a problem anywhere, the page
++ * is on its way to being freed; but it is an anomaly to bear in mind.
++ *
++ * @unlink_rb: 		if the removal of this node will firstly unlink from
++ * its rbtree. stable_node_reinsert will prevent this when restructuring the
++ * node from its old tree.
++ *
++ * @remove_tree_node:	if this is the last one of its tree_node, will the
++ * tree_node be freed ? If we are inserting stable node, this tree_node may
++ * be reused, so don't free it.
++ */
++static struct page *get_uksm_page(struct stable_node *stable_node,
++				 int unlink_rb, int remove_tree_node)
++{
++	struct page *page;
++	void *expected_mapping;
++	unsigned long kpfn;
++
++	expected_mapping = (void *)((unsigned long)stable_node |
++				    PAGE_MAPPING_KSM);
++again:
++	kpfn = READ_ONCE(stable_node->kpfn);
++	page = pfn_to_page(kpfn);
++
++	/*
++	 * page is computed from kpfn, so on most architectures reading
++	 * page->mapping is naturally ordered after reading node->kpfn,
++	 * but on Alpha we need to be more careful.
++	 */
++	smp_read_barrier_depends();
++
++	if (READ_ONCE(page->mapping) != expected_mapping)
++		goto stale;
++
++	/*
++	 * We cannot do anything with the page while its refcount is 0.
++	 * Usually 0 means free, or tail of a higher-order page: in which
++	 * case this node is no longer referenced, and should be freed;
++	 * however, it might mean that the page is under page_freeze_refs().
++	 * The __remove_mapping() case is easy, again the node is now stale;
++	 * but if page is swapcache in migrate_page_move_mapping(), it might
++	 * still be our page, in which case it's essential to keep the node.
++	 */
++	while (!get_page_unless_zero(page)) {
++		/*
++		 * Another check for page->mapping != expected_mapping would
++		 * work here too.  We have chosen the !PageSwapCache test to
++		 * optimize the common case, when the page is or is about to
++		 * be freed: PageSwapCache is cleared (under spin_lock_irq)
++		 * in the freeze_refs section of __remove_mapping(); but Anon
++		 * page->mapping reset to NULL later, in free_pages_prepare().
++		 */
++		if (!PageSwapCache(page))
++			goto stale;
++		cpu_relax();
++	}
++
++	if (READ_ONCE(page->mapping) != expected_mapping) {
++		put_page(page);
++		goto stale;
++	}
++
++	lock_page(page);
++	if (READ_ONCE(page->mapping) != expected_mapping) {
++		unlock_page(page);
++		put_page(page);
++		goto stale;
++	}
++	unlock_page(page);
++	return page;
++stale:
++	/*
++	 * We come here from above when page->mapping or !PageSwapCache
++	 * suggests that the node is stale; but it might be under migration.
++	 * We need smp_rmb(), matching the smp_wmb() in ksm_migrate_page(),
++	 * before checking whether node->kpfn has been changed.
++	 */
++	smp_rmb();
++	if (stable_node->kpfn != kpfn)
++		goto again;
++
++	remove_node_from_stable_tree(stable_node, unlink_rb, remove_tree_node);
++
++	return NULL;
++}
++
++/*
++ * Removing rmap_item from stable or unstable tree.
++ * This function will clean the information from the stable/unstable tree.
++ */
++static inline void remove_rmap_item_from_tree(struct rmap_item *rmap_item)
++{
++	if (rmap_item->address & STABLE_FLAG) {
++		struct stable_node *stable_node;
++		struct node_vma *node_vma;
++		struct page *page;
++
++		node_vma = rmap_item->head;
++		stable_node = node_vma->head;
++		page = get_uksm_page(stable_node, 1, 1);
++		if (!page)
++			goto out;
++
++		/*
++		 * page lock is needed because it's racing with
++		 * try_to_unmap_ksm(), etc.
++		 */
++		lock_page(page);
++		hlist_del(&rmap_item->hlist);
++
++		if (hlist_empty(&node_vma->rmap_hlist)) {
++			hlist_del(&node_vma->hlist);
++			free_node_vma(node_vma);
++		}
++		unlock_page(page);
++
++		put_page(page);
++		if (hlist_empty(&stable_node->hlist)) {
++			/* do NOT call remove_node_from_stable_tree() here,
++			 * it's possible for a forked rmap_item not in
++			 * stable tree while the in-tree rmap_items were
++			 * deleted.
++			 */
++			uksm_pages_shared--;
++		} else
++			uksm_pages_sharing--;
++
++
++		uksm_drop_anon_vma(rmap_item);
++	} else if (rmap_item->address & UNSTABLE_FLAG) {
++		if (rmap_item->hash_round == uksm_hash_round) {
++
++			rb_erase(&rmap_item->node,
++				 &rmap_item->tree_node->sub_root);
++			if (RB_EMPTY_ROOT(&rmap_item->tree_node->sub_root)) {
++				rb_erase(&rmap_item->tree_node->node,
++					 &root_unstable_tree);
++
++				free_tree_node(rmap_item->tree_node);
++			} else
++				rmap_item->tree_node->count--;
++		}
++		uksm_pages_unshared--;
++	}
++
++	rmap_item->address &= PAGE_MASK;
++	rmap_item->hash_max = 0;
++
++out:
++	cond_resched();		/* we're called from many long loops */
++}
++
++static inline int slot_in_uksm(struct vma_slot *slot)
++{
++	return list_empty(&slot->slot_list);
++}
++
++/*
++ * Test if the mm is exiting
++ */
++static inline bool uksm_test_exit(struct mm_struct *mm)
++{
++	return atomic_read(&mm->mm_users) == 0;
++}
++
++static inline unsigned long vma_pool_size(struct vma_slot *slot)
++{
++	return round_up(sizeof(struct rmap_list_entry) * slot->pages,
++			PAGE_SIZE) >> PAGE_SHIFT;
++}
++
++#define CAN_OVERFLOW_U64(x, delta) (U64_MAX - (x) < (delta))
++
++/* must be done with sem locked */
++static int slot_pool_alloc(struct vma_slot *slot)
++{
++	unsigned long pool_size;
++
++	if (slot->rmap_list_pool)
++		return 0;
++
++	pool_size = vma_pool_size(slot);
++	slot->rmap_list_pool = kzalloc(sizeof(struct page *) *
++				       pool_size, GFP_KERNEL);
++	if (!slot->rmap_list_pool)
++		return -ENOMEM;
++
++	slot->pool_counts = kzalloc(sizeof(unsigned int) * pool_size,
++				    GFP_KERNEL);
++	if (!slot->pool_counts) {
++		kfree(slot->rmap_list_pool);
++		return -ENOMEM;
++	}
++
++	slot->pool_size = pool_size;
++	BUG_ON(CAN_OVERFLOW_U64(uksm_pages_total, slot->pages));
++	slot->flags |= UKSM_SLOT_IN_UKSM;
++	uksm_pages_total += slot->pages;
++
++	return 0;
++}
++
++/*
++ * Called after vma is unlinked from its mm
++ */
++void uksm_remove_vma(struct vm_area_struct *vma)
++{
++	struct vma_slot *slot;
++
++	if (!vma->uksm_vma_slot)
++		return;
++
++	spin_lock(&vma_slot_list_lock);
++	slot = vma->uksm_vma_slot;
++	if (!slot)
++		goto out;
++
++	if (slot_in_uksm(slot)) {
++		/**
++		 * This slot has been added by ksmd, so move to the del list
++		 * waiting ksmd to free it.
++		 */
++		list_add_tail(&slot->slot_list, &vma_slot_del);
++	} else {
++		/**
++		 * It's still on new list. It's ok to free slot directly.
++		 */
++		list_del(&slot->slot_list);
++		free_vma_slot(slot);
++	}
++out:
++	vma->uksm_vma_slot = NULL;
++	spin_unlock(&vma_slot_list_lock);
++}
++
++/**
++ * Need to do two things:
++ * 1. check if slot was moved to del list
++ * 2. make sure the mmap_sem is manipulated under valid vma.
++ *
++ * My concern here is that in some cases, this may make
++ * vma_slot_list_lock() waiters to serialized further by some
++ * sem->wait_lock, can this really be expensive?
++ *
++ *
++ * @return
++ * 0: if successfully locked mmap_sem
++ * -ENOENT: this slot was moved to del list
++ * -EBUSY: vma lock failed
++ */
++static int try_down_read_slot_mmap_sem(struct vma_slot *slot)
++{
++	struct vm_area_struct *vma;
++	struct mm_struct *mm;
++	struct rw_semaphore *sem;
++
++	spin_lock(&vma_slot_list_lock);
++
++	/* the slot_list was removed and inited from new list, when it enters
++	 * uksm_list. If now it's not empty, then it must be moved to del list
++	 */
++	if (!slot_in_uksm(slot)) {
++		spin_unlock(&vma_slot_list_lock);
++		return -ENOENT;
++	}
++
++	BUG_ON(slot->pages != vma_pages(slot->vma));
++	/* Ok, vma still valid */
++	vma = slot->vma;
++	mm = vma->vm_mm;
++	sem = &mm->mmap_sem;
++
++	if (uksm_test_exit(mm)) {
++		spin_unlock(&vma_slot_list_lock);
++		return -ENOENT;
++	}
++
++	if (down_read_trylock(sem)) {
++		spin_unlock(&vma_slot_list_lock);
++		if (slot_pool_alloc(slot)) {
++			uksm_remove_vma(vma);
++			up_read(sem);
++			return -ENOENT;
++		}
++		return 0;
++	}
++
++	spin_unlock(&vma_slot_list_lock);
++	return -EBUSY;
++}
++
++static inline unsigned long
++vma_page_address(struct page *page, struct vm_area_struct *vma)
++{
++	pgoff_t pgoff = page->index;
++	unsigned long address;
++
++	address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
++	if (unlikely(address < vma->vm_start || address >= vma->vm_end)) {
++		/* page should be within @vma mapping range */
++		return -EFAULT;
++	}
++	return address;
++}
++
++
++/* return 0 on success with the item's mmap_sem locked */
++static inline int get_mergeable_page_lock_mmap(struct rmap_item *item)
++{
++	struct mm_struct *mm;
++	struct vma_slot *slot = item->slot;
++	int err = -EINVAL;
++
++	struct page *page;
++
++	/*
++	 * try_down_read_slot_mmap_sem() returns non-zero if the slot
++	 * has been removed by uksm_remove_vma().
++	 */
++	if (try_down_read_slot_mmap_sem(slot))
++		return -EBUSY;
++
++	mm = slot->vma->vm_mm;
++
++	if (uksm_test_exit(mm))
++		goto failout_up;
++
++	page = item->page;
++	rcu_read_lock();
++	if (!get_page_unless_zero(page)) {
++		rcu_read_unlock();
++		goto failout_up;
++	}
++
++	/* No need to consider huge page here. */
++	if (item->slot->vma->anon_vma != page_anon_vma(page) ||
++	    vma_page_address(page, item->slot->vma) != get_rmap_addr(item)) {
++		/*
++		 * TODO:
++		 * should we release this item becase of its stale page
++		 * mapping?
++		 */
++		put_page(page);
++		rcu_read_unlock();
++		goto failout_up;
++	}
++	rcu_read_unlock();
++	return 0;
++
++failout_up:
++	up_read(&mm->mmap_sem);
++	return err;
++}
++
++/*
++ * What kind of VMA is considered ?
++ */
++static inline int vma_can_enter(struct vm_area_struct *vma)
++{
++	return uksm_flags_can_scan(vma->vm_flags);
++}
++
++/*
++ * Called whenever a fresh new vma is created A new vma_slot.
++ * is created and inserted into a global list Must be called.
++ * after vma is inserted to its mm      		    .
++ */
++void uksm_vma_add_new(struct vm_area_struct *vma)
++{
++	struct vma_slot *slot;
++
++	if (!vma_can_enter(vma)) {
++		vma->uksm_vma_slot = NULL;
++		return;
++	}
++
++	slot = alloc_vma_slot();
++	if (!slot) {
++		vma->uksm_vma_slot = NULL;
++		return;
++	}
++
++	vma->uksm_vma_slot = slot;
++	vma->vm_flags |= VM_MERGEABLE;
++	slot->vma = vma;
++	slot->mm = vma->vm_mm;
++	slot->ctime_j = jiffies;
++	slot->pages = vma_pages(vma);
++	spin_lock(&vma_slot_list_lock);
++	list_add_tail(&slot->slot_list, &vma_slot_new);
++	spin_unlock(&vma_slot_list_lock);
++}
++
++/*   32/3 < they < 32/2 */
++#define shiftl	8
++#define shiftr	12
++
++#define HASH_FROM_TO(from, to) 				\
++for (index = from; index < to; index++) {		\
++	pos = random_nums[index];			\
++	hash += key[pos];				\
++	hash += (hash << shiftl);			\
++	hash ^= (hash >> shiftr);			\
++}
++
++
++#define HASH_FROM_DOWN_TO(from, to) 			\
++for (index = from - 1; index >= to; index--) {		\
++	hash ^= (hash >> shiftr);			\
++	hash ^= (hash >> (shiftr*2));			\
++	hash -= (hash << shiftl);			\
++	hash += (hash << (shiftl*2));			\
++	pos = random_nums[index];			\
++	hash -= key[pos];				\
++}
++
++/*
++ * The main random sample hash function.
++ */
++static u32 random_sample_hash(void *addr, u32 hash_strength)
++{
++	u32 hash = 0xdeadbeef;
++	int index, pos, loop = hash_strength;
++	u32 *key = (u32 *)addr;
++
++	if (loop > HASH_STRENGTH_FULL)
++		loop = HASH_STRENGTH_FULL;
++
++	HASH_FROM_TO(0, loop);
++
++	if (hash_strength > HASH_STRENGTH_FULL) {
++		loop = hash_strength - HASH_STRENGTH_FULL;
++		HASH_FROM_TO(0, loop);
++	}
++
++	return hash;
++}
++
++
++/**
++ * It's used when hash strength is adjusted
++ *
++ * @addr The page's virtual address
++ * @from The original hash strength
++ * @to   The hash strength changed to
++ * @hash The hash value generated with "from" hash value
++ *
++ * return the hash value
++ */
++static u32 delta_hash(void *addr, int from, int to, u32 hash)
++{
++	u32 *key = (u32 *)addr;
++	int index, pos; /* make sure they are int type */
++
++	if (to > from) {
++		if (from >= HASH_STRENGTH_FULL) {
++			from -= HASH_STRENGTH_FULL;
++			to -= HASH_STRENGTH_FULL;
++			HASH_FROM_TO(from, to);
++		} else if (to <= HASH_STRENGTH_FULL) {
++			HASH_FROM_TO(from, to);
++		} else {
++			HASH_FROM_TO(from, HASH_STRENGTH_FULL);
++			HASH_FROM_TO(0, to - HASH_STRENGTH_FULL);
++		}
++	} else {
++		if (from <= HASH_STRENGTH_FULL) {
++			HASH_FROM_DOWN_TO(from, to);
++		} else if (to >= HASH_STRENGTH_FULL) {
++			from -= HASH_STRENGTH_FULL;
++			to -= HASH_STRENGTH_FULL;
++			HASH_FROM_DOWN_TO(from, to);
++		} else {
++			HASH_FROM_DOWN_TO(from - HASH_STRENGTH_FULL, 0);
++			HASH_FROM_DOWN_TO(HASH_STRENGTH_FULL, to);
++		}
++	}
++
++	return hash;
++}
++
++/**
++ *
++ * Called when: rshash_pos or rshash_neg is about to overflow or a scan round
++ * has finished.
++ *
++ * return 0 if no page has been scanned since last call, 1 otherwise.
++ */
++static inline int encode_benefit(void)
++{
++	u64 scanned_delta, pos_delta, neg_delta;
++	unsigned long base = benefit.base;
++
++	scanned_delta = uksm_pages_scanned - uksm_pages_scanned_last;
++
++	if (!scanned_delta)
++		return 0;
++
++	scanned_delta >>= base;
++	pos_delta = rshash_pos >> base;
++	neg_delta = rshash_neg >> base;
++
++	if (CAN_OVERFLOW_U64(benefit.pos, pos_delta) ||
++	    CAN_OVERFLOW_U64(benefit.neg, neg_delta) ||
++	    CAN_OVERFLOW_U64(benefit.scanned, scanned_delta)) {
++		benefit.scanned >>= 1;
++		benefit.neg >>= 1;
++		benefit.pos >>= 1;
++		benefit.base++;
++		scanned_delta >>= 1;
++		pos_delta >>= 1;
++		neg_delta >>= 1;
++	}
++
++	benefit.pos += pos_delta;
++	benefit.neg += neg_delta;
++	benefit.scanned += scanned_delta;
++
++	BUG_ON(!benefit.scanned);
++
++	rshash_pos = rshash_neg = 0;
++	uksm_pages_scanned_last = uksm_pages_scanned;
++
++	return 1;
++}
++
++static inline void reset_benefit(void)
++{
++	benefit.pos = 0;
++	benefit.neg = 0;
++	benefit.base = 0;
++	benefit.scanned = 0;
++}
++
++static inline void inc_rshash_pos(unsigned long delta)
++{
++	if (CAN_OVERFLOW_U64(rshash_pos, delta))
++		encode_benefit();
++
++	rshash_pos += delta;
++}
++
++static inline void inc_rshash_neg(unsigned long delta)
++{
++	if (CAN_OVERFLOW_U64(rshash_neg, delta))
++		encode_benefit();
++
++	rshash_neg += delta;
++}
++
++
++static inline u32 page_hash(struct page *page, unsigned long hash_strength,
++			    int cost_accounting)
++{
++	u32 val;
++	unsigned long delta;
++
++	void *addr = kmap_atomic(page);
++
++	val = random_sample_hash(addr, hash_strength);
++	kunmap_atomic(addr);
++
++	if (cost_accounting) {
++		if (HASH_STRENGTH_FULL > hash_strength)
++			delta = HASH_STRENGTH_FULL - hash_strength;
++		else
++			delta = 0;
++
++		inc_rshash_pos(delta);
++	}
++
++	return val;
++}
++
++static int memcmp_pages(struct page *page1, struct page *page2,
++			int cost_accounting)
++{
++	char *addr1, *addr2;
++	int ret;
++
++	addr1 = kmap_atomic(page1);
++	addr2 = kmap_atomic(page2);
++	ret = memcmp(addr1, addr2, PAGE_SIZE);
++	kunmap_atomic(addr2);
++	kunmap_atomic(addr1);
++
++	if (cost_accounting)
++		inc_rshash_neg(memcmp_cost);
++
++	return ret;
++}
++
++static inline int pages_identical(struct page *page1, struct page *page2)
++{
++	return !memcmp_pages(page1, page2, 0);
++}
++
++static inline int is_page_full_zero(struct page *page)
++{
++	char *addr;
++	int ret;
++
++	addr = kmap_atomic(page);
++	ret = is_full_zero(addr, PAGE_SIZE);
++	kunmap_atomic(addr);
++
++	return ret;
++}
++
++static int write_protect_page(struct vm_area_struct *vma, struct page *page,
++			      pte_t *orig_pte, pte_t *old_pte)
++{
++	struct mm_struct *mm = vma->vm_mm;
++	unsigned long addr;
++	pte_t *ptep;
++	spinlock_t *ptl;
++	int swapped;
++	int err = -EFAULT;
++	unsigned long mmun_start;	/* For mmu_notifiers */
++	unsigned long mmun_end;		/* For mmu_notifiers */
++
++	addr = page_address_in_vma(page, vma);
++	if (addr == -EFAULT)
++		goto out;
++
++	BUG_ON(PageTransCompound(page));
++
++	mmun_start = addr;
++	mmun_end   = addr + PAGE_SIZE;
++	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
++
++	ptep = page_check_address(page, mm, addr, &ptl, 0);
++	if (!ptep)
++		goto out_mn;
++
++	if (old_pte)
++		*old_pte = *ptep;
++
++	if (pte_write(*ptep) || pte_dirty(*ptep)) {
++		pte_t entry;
++
++		swapped = PageSwapCache(page);
++		flush_cache_page(vma, addr, page_to_pfn(page));
++		/*
++		 * Ok this is tricky, when get_user_pages_fast() run it doesnt
++		 * take any lock, therefore the check that we are going to make
++		 * with the pagecount against the mapcount is racey and
++		 * O_DIRECT can happen right after the check.
++		 * So we clear the pte and flush the tlb before the check
++		 * this assure us that no O_DIRECT can happen after the check
++		 * or in the middle of the check.
++		 */
++		entry = ptep_clear_flush_notify(vma, addr, ptep);
++		/*
++		 * Check that no O_DIRECT or similar I/O is in progress on the
++		 * page
++		 */
++		if (page_mapcount(page) + 1 + swapped != page_count(page)) {
++			set_pte_at(mm, addr, ptep, entry);
++			goto out_unlock;
++		}
++		if (pte_dirty(entry))
++			set_page_dirty(page);
++		entry = pte_mkclean(pte_wrprotect(entry));
++		set_pte_at_notify(mm, addr, ptep, entry);
++	}
++	*orig_pte = *ptep;
++	err = 0;
++
++out_unlock:
++	pte_unmap_unlock(ptep, ptl);
++out_mn:
++	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
++out:
++	return err;
++}
++
++#define MERGE_ERR_PGERR		1 /* the page is invalid cannot continue */
++#define MERGE_ERR_COLLI		2 /* there is a collision */
++#define MERGE_ERR_COLLI_MAX	3 /* collision at the max hash strength */
++#define MERGE_ERR_CHANGED	4 /* the page has changed since last hash */
++
++
++/**
++ * replace_page - replace page in vma by new ksm page
++ * @vma:      vma that holds the pte pointing to page
++ * @page:     the page we are replacing by kpage
++ * @kpage:    the ksm page we replace page by
++ * @orig_pte: the original value of the pte
++ *
++ * Returns 0 on success, MERGE_ERR_PGERR on failure.
++ */
++static int replace_page(struct vm_area_struct *vma, struct page *page,
++			struct page *kpage, pte_t orig_pte)
++{
++	struct mm_struct *mm = vma->vm_mm;
++	pgd_t *pgd;
++	pud_t *pud;
++	pmd_t *pmd;
++	pte_t *ptep;
++	spinlock_t *ptl;
++	pte_t entry;
++
++	unsigned long addr;
++	int err = MERGE_ERR_PGERR;
++	unsigned long mmun_start;	/* For mmu_notifiers */
++	unsigned long mmun_end;		/* For mmu_notifiers */
++
++	addr = page_address_in_vma(page, vma);
++	if (addr == -EFAULT)
++		goto out;
++
++	pgd = pgd_offset(mm, addr);
++	if (!pgd_present(*pgd))
++		goto out;
++
++	pud = pud_offset(pgd, addr);
++	if (!pud_present(*pud))
++		goto out;
++
++	pmd = pmd_offset(pud, addr);
++	BUG_ON(pmd_trans_huge(*pmd));
++	if (!pmd_present(*pmd))
++		goto out;
++
++	mmun_start = addr;
++	mmun_end   = addr + PAGE_SIZE;
++	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
++
++	ptep = pte_offset_map_lock(mm, pmd, addr, &ptl);
++	if (!pte_same(*ptep, orig_pte)) {
++		pte_unmap_unlock(ptep, ptl);
++		goto out_mn;
++	}
++
++	flush_cache_page(vma, addr, pte_pfn(*ptep));
++	ptep_clear_flush_notify(vma, addr, ptep);
++	entry = mk_pte(kpage, vma->vm_page_prot);
++
++	/* special treatment is needed for zero_page */
++	if ((page_to_pfn(kpage) == uksm_zero_pfn) ||
++				(page_to_pfn(kpage) == zero_pfn)) {
++		entry = pte_mkspecial(entry);
++		dec_mm_counter(mm, MM_ANONPAGES);
++		inc_zone_page_state(page, NR_UKSM_ZERO_PAGES);
++	} else {
++		get_page(kpage);
++		page_add_anon_rmap(kpage, vma, addr, false);
++	}
++
++	set_pte_at_notify(mm, addr, ptep, entry);
++
++	page_remove_rmap(page, false);
++	if (!page_mapped(page))
++		try_to_free_swap(page);
++	put_page(page);
++
++	pte_unmap_unlock(ptep, ptl);
++	err = 0;
++out_mn:
++	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
++out:
++	return err;
++}
++
++
++/**
++ *  Fully hash a page with HASH_STRENGTH_MAX return a non-zero hash value. The
++ *  zero hash value at HASH_STRENGTH_MAX is used to indicated that its
++ *  hash_max member has not been calculated.
++ *
++ * @page The page needs to be hashed
++ * @hash_old The hash value calculated with current hash strength
++ *
++ * return the new hash value calculated at HASH_STRENGTH_MAX
++ */
++static inline u32 page_hash_max(struct page *page, u32 hash_old)
++{
++	u32 hash_max = 0;
++	void *addr;
++
++	addr = kmap_atomic(page);
++	hash_max = delta_hash(addr, hash_strength,
++			      HASH_STRENGTH_MAX, hash_old);
++
++	kunmap_atomic(addr);
++
++	if (!hash_max)
++		hash_max = 1;
++
++	inc_rshash_neg(HASH_STRENGTH_MAX - hash_strength);
++	return hash_max;
++}
++
++/*
++ * We compare the hash again, to ensure that it is really a hash collision
++ * instead of being caused by page write.
++ */
++static inline int check_collision(struct rmap_item *rmap_item,
++				  u32 hash)
++{
++	int err;
++	struct page *page = rmap_item->page;
++
++	/* if this rmap_item has already been hash_maxed, then the collision
++	 * must appears in the second-level rbtree search. In this case we check
++	 * if its hash_max value has been changed. Otherwise, the collision
++	 * happens in the first-level rbtree search, so we check against it's
++	 * current hash value.
++	 */
++	if (rmap_item->hash_max) {
++		inc_rshash_neg(memcmp_cost);
++		inc_rshash_neg(HASH_STRENGTH_MAX - hash_strength);
++
++		if (rmap_item->hash_max == page_hash_max(page, hash))
++			err = MERGE_ERR_COLLI;
++		else
++			err = MERGE_ERR_CHANGED;
++	} else {
++		inc_rshash_neg(memcmp_cost + hash_strength);
++
++		if (page_hash(page, hash_strength, 0) == hash)
++			err = MERGE_ERR_COLLI;
++		else
++			err = MERGE_ERR_CHANGED;
++	}
++
++	return err;
++}
++
++/**
++ * Try to merge a rmap_item.page with a kpage in stable node. kpage must
++ * already be a ksm page.
++ *
++ * @return 0 if the pages were merged, -EFAULT otherwise.
++ */
++static int try_to_merge_with_uksm_page(struct rmap_item *rmap_item,
++				      struct page *kpage, u32 hash)
++{
++	struct vm_area_struct *vma = rmap_item->slot->vma;
++	struct mm_struct *mm = vma->vm_mm;
++	pte_t orig_pte = __pte(0);
++	int err = MERGE_ERR_PGERR;
++	struct page *page;
++
++	if (uksm_test_exit(mm))
++		goto out;
++
++	page = rmap_item->page;
++
++	if (page == kpage) { /* ksm page forked */
++		err = 0;
++		goto out;
++	}
++
++	/*
++	 * We need the page lock to read a stable PageSwapCache in
++	 * write_protect_page().  We use trylock_page() instead of
++	 * lock_page() because we don't want to wait here - we
++	 * prefer to continue scanning and merging different pages,
++	 * then come back to this page when it is unlocked.
++	 */
++	if (!trylock_page(page))
++		goto out;
++
++	if (!PageAnon(page) || !PageKsm(kpage))
++		goto out_unlock;
++
++	if (PageTransCompound(page)) {
++		err = split_huge_page(page);
++		if (err)
++			goto out_unlock;
++	}
++
++	/*
++	 * If this anonymous page is mapped only here, its pte may need
++	 * to be write-protected.  If it's mapped elsewhere, all of its
++	 * ptes are necessarily already write-protected.  But in either
++	 * case, we need to lock and check page_count is not raised.
++	 */
++	if (write_protect_page(vma, page, &orig_pte, NULL) == 0) {
++		if (pages_identical(page, kpage))
++			err = replace_page(vma, page, kpage, orig_pte);
++		else
++			err = check_collision(rmap_item, hash);
++	}
++
++	if ((vma->vm_flags & VM_LOCKED) && kpage && !err) {
++		munlock_vma_page(page);
++		if (!PageMlocked(kpage)) {
++			unlock_page(page);
++			lock_page(kpage);
++			mlock_vma_page(kpage);
++			page = kpage;		/* for final unlock */
++		}
++	}
++
++out_unlock:
++	unlock_page(page);
++out:
++	return err;
++}
++
++
++
++/**
++ * If two pages fail to merge in try_to_merge_two_pages, then we have a chance
++ * to restore a page mapping that has been changed in try_to_merge_two_pages.
++ *
++ * @return 0 on success.
++ */
++static int restore_uksm_page_pte(struct vm_area_struct *vma, unsigned long addr,
++			     pte_t orig_pte, pte_t wprt_pte)
++{
++	struct mm_struct *mm = vma->vm_mm;
++	pgd_t *pgd;
++	pud_t *pud;
++	pmd_t *pmd;
++	pte_t *ptep;
++	spinlock_t *ptl;
++
++	int err = -EFAULT;
++
++	pgd = pgd_offset(mm, addr);
++	if (!pgd_present(*pgd))
++		goto out;
++
++	pud = pud_offset(pgd, addr);
++	if (!pud_present(*pud))
++		goto out;
++
++	pmd = pmd_offset(pud, addr);
++	if (!pmd_present(*pmd))
++		goto out;
++
++	ptep = pte_offset_map_lock(mm, pmd, addr, &ptl);
++	if (!pte_same(*ptep, wprt_pte)) {
++		/* already copied, let it be */
++		pte_unmap_unlock(ptep, ptl);
++		goto out;
++	}
++
++	/*
++	 * Good boy, still here. When we still get the ksm page, it does not
++	 * return to the free page pool, there is no way that a pte was changed
++	 * to other page and gets back to this page. And remind that ksm page
++	 * do not reuse in do_wp_page(). So it's safe to restore the original
++	 * pte.
++	 */
++	flush_cache_page(vma, addr, pte_pfn(*ptep));
++	ptep_clear_flush_notify(vma, addr, ptep);
++	set_pte_at_notify(mm, addr, ptep, orig_pte);
++
++	pte_unmap_unlock(ptep, ptl);
++	err = 0;
++out:
++	return err;
++}
++
++/**
++ * try_to_merge_two_pages() - take two identical pages and prepare
++ * them to be merged into one page(rmap_item->page)
++ *
++ * @return 0 if we successfully merged two identical pages into
++ *         one ksm page. MERGE_ERR_COLLI if it's only a hash collision
++ *         search in rbtree. MERGE_ERR_CHANGED if rmap_item has been
++ *         changed since it's hashed. MERGE_ERR_PGERR otherwise.
++ *
++ */
++static int try_to_merge_two_pages(struct rmap_item *rmap_item,
++				  struct rmap_item *tree_rmap_item,
++				  u32 hash)
++{
++	pte_t orig_pte1 = __pte(0), orig_pte2 = __pte(0);
++	pte_t wprt_pte1 = __pte(0), wprt_pte2 = __pte(0);
++	struct vm_area_struct *vma1 = rmap_item->slot->vma;
++	struct vm_area_struct *vma2 = tree_rmap_item->slot->vma;
++	struct page *page = rmap_item->page;
++	struct page *tree_page = tree_rmap_item->page;
++	int err = MERGE_ERR_PGERR;
++	struct address_space *saved_mapping;
++
++
++	if (rmap_item->page == tree_rmap_item->page)
++		goto out;
++
++	if (!trylock_page(page))
++		goto out;
++
++	if (!PageAnon(page))
++		goto out_unlock;
++
++	if (PageTransCompound(page)) {
++		err = split_huge_page(page);
++		if (err)
++			goto out_unlock;
++	}
++
++	if (write_protect_page(vma1, page, &wprt_pte1, &orig_pte1) != 0) {
++		unlock_page(page);
++		goto out;
++	}
++
++	/*
++	 * While we hold page lock, upgrade page from
++	 * PageAnon+anon_vma to PageKsm+NULL stable_node:
++	 * stable_tree_insert() will update stable_node.
++	 */
++	saved_mapping = page->mapping;
++	set_page_stable_node(page, NULL);
++	mark_page_accessed(page);
++	if (!PageDirty(page))
++		SetPageDirty(page);
++
++	unlock_page(page);
++
++	if (!trylock_page(tree_page))
++		goto restore_out;
++
++	if (!PageAnon(tree_page)) {
++		unlock_page(tree_page);
++		goto restore_out;
++	}
++
++	if (PageTransCompound(tree_page)) {
++		err = split_huge_page(tree_page);
++		if (err) {
++			unlock_page(tree_page);
++			goto restore_out;
++		}
++	}
++
++	if (write_protect_page(vma2, tree_page, &wprt_pte2, &orig_pte2) != 0) {
++		unlock_page(tree_page);
++		goto restore_out;
++	}
++
++	if (pages_identical(page, tree_page)) {
++		err = replace_page(vma2, tree_page, page, wprt_pte2);
++		if (err) {
++			unlock_page(tree_page);
++			goto restore_out;
++		}
++
++		if ((vma2->vm_flags & VM_LOCKED)) {
++			munlock_vma_page(tree_page);
++			if (!PageMlocked(page)) {
++				unlock_page(tree_page);
++				lock_page(page);
++				mlock_vma_page(page);
++				tree_page = page; /* for final unlock */
++			}
++		}
++
++		unlock_page(tree_page);
++
++		goto out; /* success */
++
++	} else {
++		if (tree_rmap_item->hash_max &&
++		    tree_rmap_item->hash_max == rmap_item->hash_max) {
++			err = MERGE_ERR_COLLI_MAX;
++		} else if (page_hash(page, hash_strength, 0) ==
++		    page_hash(tree_page, hash_strength, 0)) {
++			inc_rshash_neg(memcmp_cost + hash_strength * 2);
++			err = MERGE_ERR_COLLI;
++		} else {
++			err = MERGE_ERR_CHANGED;
++		}
++
++		unlock_page(tree_page);
++	}
++
++restore_out:
++	lock_page(page);
++	if (!restore_uksm_page_pte(vma1, get_rmap_addr(rmap_item),
++				  orig_pte1, wprt_pte1))
++		page->mapping = saved_mapping;
++
++out_unlock:
++	unlock_page(page);
++out:
++	return err;
++}
++
++static inline int hash_cmp(u32 new_val, u32 node_val)
++{
++	if (new_val > node_val)
++		return 1;
++	else if (new_val < node_val)
++		return -1;
++	else
++		return 0;
++}
++
++static inline u32 rmap_item_hash_max(struct rmap_item *item, u32 hash)
++{
++	u32 hash_max = item->hash_max;
++
++	if (!hash_max) {
++		hash_max = page_hash_max(item->page, hash);
++
++		item->hash_max = hash_max;
++	}
++
++	return hash_max;
++}
++
++
++
++/**
++ * stable_tree_search() - search the stable tree for a page
++ *
++ * @item: 	the rmap_item we are comparing with
++ * @hash: 	the hash value of this item->page already calculated
++ *
++ * @return 	the page we have found, NULL otherwise. The page returned has
++ *         	been gotten.
++ */
++static struct page *stable_tree_search(struct rmap_item *item, u32 hash)
++{
++	struct rb_node *node = root_stable_treep->rb_node;
++	struct tree_node *tree_node;
++	unsigned long hash_max;
++	struct page *page = item->page;
++	struct stable_node *stable_node;
++
++	stable_node = page_stable_node(page);
++	if (stable_node) {
++		/* ksm page forked, that is
++		 * if (PageKsm(page) && !in_stable_tree(rmap_item))
++		 * it's actually gotten once outside.
++		 */
++		get_page(page);
++		return page;
++	}
++
++	while (node) {
++		int cmp;
++
++		tree_node = rb_entry(node, struct tree_node, node);
++
++		cmp = hash_cmp(hash, tree_node->hash);
++
++		if (cmp < 0)
++			node = node->rb_left;
++		else if (cmp > 0)
++			node = node->rb_right;
++		else
++			break;
++	}
++
++	if (!node)
++		return NULL;
++
++	if (tree_node->count == 1) {
++		stable_node = rb_entry(tree_node->sub_root.rb_node,
++				       struct stable_node, node);
++		BUG_ON(!stable_node);
++
++		goto get_page_out;
++	}
++
++	/*
++	 * ok, we have to search the second
++	 * level subtree, hash the page to a
++	 * full strength.
++	 */
++	node = tree_node->sub_root.rb_node;
++	BUG_ON(!node);
++	hash_max = rmap_item_hash_max(item, hash);
++
++	while (node) {
++		int cmp;
++
++		stable_node = rb_entry(node, struct stable_node, node);
++
++		cmp = hash_cmp(hash_max, stable_node->hash_max);
++
++		if (cmp < 0)
++			node = node->rb_left;
++		else if (cmp > 0)
++			node = node->rb_right;
++		else
++			goto get_page_out;
++	}
++
++	return NULL;
++
++get_page_out:
++	page = get_uksm_page(stable_node, 1, 1);
++	return page;
++}
++
++static int try_merge_rmap_item(struct rmap_item *item,
++			       struct page *kpage,
++			       struct page *tree_page)
++{
++	spinlock_t *ptl;
++	pte_t *ptep;
++	unsigned long addr;
++	struct vm_area_struct *vma = item->slot->vma;
++
++	addr = get_rmap_addr(item);
++	ptep = page_check_address(kpage, vma->vm_mm, addr, &ptl, 0);
++	if (!ptep)
++		return 0;
++
++	if (pte_write(*ptep)) {
++		/* has changed, abort! */
++		pte_unmap_unlock(ptep, ptl);
++		return 0;
++	}
++
++	get_page(tree_page);
++	page_add_anon_rmap(tree_page, vma, addr, false);
++
++	flush_cache_page(vma, addr, pte_pfn(*ptep));
++	ptep_clear_flush_notify(vma, addr, ptep);
++	set_pte_at_notify(vma->vm_mm, addr, ptep,
++			  mk_pte(tree_page, vma->vm_page_prot));
++
++	page_remove_rmap(kpage, false);
++	put_page(kpage);
++
++	pte_unmap_unlock(ptep, ptl);
++
++	return 1;
++}
++
++/**
++ * try_to_merge_with_stable_page() - when two rmap_items need to be inserted
++ * into stable tree, the page was found to be identical to a stable ksm page,
++ * this is the last chance we can merge them into one.
++ *
++ * @item1:	the rmap_item holding the page which we wanted to insert
++ *       	into stable tree.
++ * @item2:	the other rmap_item we found when unstable tree search
++ * @oldpage:	the page currently mapped by the two rmap_items
++ * @tree_page: 	the page we found identical in stable tree node
++ * @success1:	return if item1 is successfully merged
++ * @success2:	return if item2 is successfully merged
++ */
++static void try_merge_with_stable(struct rmap_item *item1,
++				  struct rmap_item *item2,
++				  struct page **kpage,
++				  struct page *tree_page,
++				  int *success1, int *success2)
++{
++	struct vm_area_struct *vma1 = item1->slot->vma;
++	struct vm_area_struct *vma2 = item2->slot->vma;
++	*success1 = 0;
++	*success2 = 0;
++
++	if (unlikely(*kpage == tree_page)) {
++		/* I don't think this can really happen */
++		printk(KERN_WARNING "UKSM: unexpected condition detected in "
++			"try_merge_with_stable() -- *kpage == tree_page !\n");
++		*success1 = 1;
++		*success2 = 1;
++		return;
++	}
++
++	if (!PageAnon(*kpage) || !PageKsm(*kpage))
++		goto failed;
++
++	if (!trylock_page(tree_page))
++		goto failed;
++
++	/* If the oldpage is still ksm and still pointed
++	 * to in the right place, and still write protected,
++	 * we are confident it's not changed, no need to
++	 * memcmp anymore.
++	 * be ware, we cannot take nested pte locks,
++	 * deadlock risk.
++	 */
++	if (!try_merge_rmap_item(item1, *kpage, tree_page))
++		goto unlock_failed;
++
++	/* ok, then vma2, remind that pte1 already set */
++	if (!try_merge_rmap_item(item2, *kpage, tree_page))
++		goto success_1;
++
++	*success2 = 1;
++success_1:
++	*success1 = 1;
++
++
++	if ((*success1 && vma1->vm_flags & VM_LOCKED) ||
++	    (*success2 && vma2->vm_flags & VM_LOCKED)) {
++		munlock_vma_page(*kpage);
++		if (!PageMlocked(tree_page))
++			mlock_vma_page(tree_page);
++	}
++
++	/*
++	 * We do not need oldpage any more in the caller, so can break the lock
++	 * now.
++	 */
++	unlock_page(*kpage);
++	*kpage = tree_page; /* Get unlocked outside. */
++	return;
++
++unlock_failed:
++	unlock_page(tree_page);
++failed:
++	return;
++}
++
++static inline void stable_node_hash_max(struct stable_node *node,
++					 struct page *page, u32 hash)
++{
++	u32 hash_max = node->hash_max;
++
++	if (!hash_max) {
++		hash_max = page_hash_max(page, hash);
++		node->hash_max = hash_max;
++	}
++}
++
++static inline
++struct stable_node *new_stable_node(struct tree_node *tree_node,
++				    struct page *kpage, u32 hash_max)
++{
++	struct stable_node *new_stable_node;
++
++	new_stable_node = alloc_stable_node();
++	if (!new_stable_node)
++		return NULL;
++
++	new_stable_node->kpfn = page_to_pfn(kpage);
++	new_stable_node->hash_max = hash_max;
++	new_stable_node->tree_node = tree_node;
++	set_page_stable_node(kpage, new_stable_node);
++
++	return new_stable_node;
++}
++
++static inline
++struct stable_node *first_level_insert(struct tree_node *tree_node,
++				       struct rmap_item *rmap_item,
++				       struct rmap_item *tree_rmap_item,
++				       struct page **kpage, u32 hash,
++				       int *success1, int *success2)
++{
++	int cmp;
++	struct page *tree_page;
++	u32 hash_max = 0;
++	struct stable_node *stable_node, *new_snode;
++	struct rb_node *parent = NULL, **new;
++
++	/* this tree node contains no sub-tree yet */
++	stable_node = rb_entry(tree_node->sub_root.rb_node,
++			       struct stable_node, node);
++
++	tree_page = get_uksm_page(stable_node, 1, 0);
++	if (tree_page) {
++		cmp = memcmp_pages(*kpage, tree_page, 1);
++		if (!cmp) {
++			try_merge_with_stable(rmap_item, tree_rmap_item, kpage,
++					      tree_page, success1, success2);
++			put_page(tree_page);
++			if (!*success1 && !*success2)
++				goto failed;
++
++			return stable_node;
++
++		} else {
++			/*
++			 * collision in first level try to create a subtree.
++			 * A new node need to be created.
++			 */
++			put_page(tree_page);
++
++			stable_node_hash_max(stable_node, tree_page,
++					     tree_node->hash);
++			hash_max = rmap_item_hash_max(rmap_item, hash);
++			cmp = hash_cmp(hash_max, stable_node->hash_max);
++
++			parent = &stable_node->node;
++			if (cmp < 0) {
++				new = &parent->rb_left;
++			} else if (cmp > 0) {
++				new = &parent->rb_right;
++			} else {
++				goto failed;
++			}
++		}
++
++	} else {
++		/* the only stable_node deleted, we reuse its tree_node.
++		 */
++		parent = NULL;
++		new = &tree_node->sub_root.rb_node;
++	}
++
++	new_snode = new_stable_node(tree_node, *kpage, hash_max);
++	if (!new_snode)
++		goto failed;
++
++	rb_link_node(&new_snode->node, parent, new);
++	rb_insert_color(&new_snode->node, &tree_node->sub_root);
++	tree_node->count++;
++	*success1 = *success2 = 1;
++
++	return new_snode;
++
++failed:
++	return NULL;
++}
++
++static inline
++struct stable_node *stable_subtree_insert(struct tree_node *tree_node,
++					  struct rmap_item *rmap_item,
++					  struct rmap_item *tree_rmap_item,
++					  struct page **kpage, u32 hash,
++					  int *success1, int *success2)
++{
++	struct page *tree_page;
++	u32 hash_max;
++	struct stable_node *stable_node, *new_snode;
++	struct rb_node *parent, **new;
++
++research:
++	parent = NULL;
++	new = &tree_node->sub_root.rb_node;
++	BUG_ON(!*new);
++	hash_max = rmap_item_hash_max(rmap_item, hash);
++	while (*new) {
++		int cmp;
++
++		stable_node = rb_entry(*new, struct stable_node, node);
++
++		cmp = hash_cmp(hash_max, stable_node->hash_max);
++
++		if (cmp < 0) {
++			parent = *new;
++			new = &parent->rb_left;
++		} else if (cmp > 0) {
++			parent = *new;
++			new = &parent->rb_right;
++		} else {
++			tree_page = get_uksm_page(stable_node, 1, 0);
++			if (tree_page) {
++				cmp = memcmp_pages(*kpage, tree_page, 1);
++				if (!cmp) {
++					try_merge_with_stable(rmap_item,
++						tree_rmap_item, kpage,
++						tree_page, success1, success2);
++
++					put_page(tree_page);
++					if (!*success1 && !*success2)
++						goto failed;
++					/*
++					 * successfully merged with a stable
++					 * node
++					 */
++					return stable_node;
++				} else {
++					put_page(tree_page);
++					goto failed;
++				}
++			} else {
++				/*
++				 * stable node may be deleted,
++				 * and subtree maybe
++				 * restructed, cannot
++				 * continue, research it.
++				 */
++				if (tree_node->count) {
++					goto research;
++				} else {
++					/* reuse the tree node*/
++					parent = NULL;
++					new = &tree_node->sub_root.rb_node;
++				}
++			}
++		}
++	}
++
++	new_snode = new_stable_node(tree_node, *kpage, hash_max);
++	if (!new_snode)
++		goto failed;
++
++	rb_link_node(&new_snode->node, parent, new);
++	rb_insert_color(&new_snode->node, &tree_node->sub_root);
++	tree_node->count++;
++	*success1 = *success2 = 1;
++
++	return new_snode;
++
++failed:
++	return NULL;
++}
++
++
++/**
++ * stable_tree_insert() - try to insert a merged page in unstable tree to
++ * the stable tree
++ *
++ * @kpage:		the page need to be inserted
++ * @hash:		the current hash of this page
++ * @rmap_item:		the rmap_item being scanned
++ * @tree_rmap_item:	the rmap_item found on unstable tree
++ * @success1:		return if rmap_item is merged
++ * @success2:		return if tree_rmap_item is merged
++ *
++ * @return 		the stable_node on stable tree if at least one
++ *      		rmap_item is inserted into stable tree, NULL
++ *      		otherwise.
++ */
++static struct stable_node *
++stable_tree_insert(struct page **kpage, u32 hash,
++		   struct rmap_item *rmap_item,
++		   struct rmap_item *tree_rmap_item,
++		   int *success1, int *success2)
++{
++	struct rb_node **new = &root_stable_treep->rb_node;
++	struct rb_node *parent = NULL;
++	struct stable_node *stable_node;
++	struct tree_node *tree_node;
++	u32 hash_max = 0;
++
++	*success1 = *success2 = 0;
++
++	while (*new) {
++		int cmp;
++
++		tree_node = rb_entry(*new, struct tree_node, node);
++
++		cmp = hash_cmp(hash, tree_node->hash);
++
++		if (cmp < 0) {
++			parent = *new;
++			new = &parent->rb_left;
++		} else if (cmp > 0) {
++			parent = *new;
++			new = &parent->rb_right;
++		} else
++			break;
++	}
++
++	if (*new) {
++		if (tree_node->count == 1) {
++			stable_node = first_level_insert(tree_node, rmap_item,
++						tree_rmap_item, kpage,
++						hash, success1, success2);
++		} else {
++			stable_node = stable_subtree_insert(tree_node,
++					rmap_item, tree_rmap_item, kpage,
++					hash, success1, success2);
++		}
++	} else {
++
++		/* no tree node found */
++		tree_node = alloc_tree_node(stable_tree_node_listp);
++		if (!tree_node) {
++			stable_node = NULL;
++			goto out;
++		}
++
++		stable_node = new_stable_node(tree_node, *kpage, hash_max);
++		if (!stable_node) {
++			free_tree_node(tree_node);
++			goto out;
++		}
++
++		tree_node->hash = hash;
++		rb_link_node(&tree_node->node, parent, new);
++		rb_insert_color(&tree_node->node, root_stable_treep);
++		parent = NULL;
++		new = &tree_node->sub_root.rb_node;
++
++		rb_link_node(&stable_node->node, parent, new);
++		rb_insert_color(&stable_node->node, &tree_node->sub_root);
++		tree_node->count++;
++		*success1 = *success2 = 1;
++	}
++
++out:
++	return stable_node;
++}
++
++
++/**
++ * get_tree_rmap_item_page() - try to get the page and lock the mmap_sem
++ *
++ * @return 	0 on success, -EBUSY if unable to lock the mmap_sem,
++ *         	-EINVAL if the page mapping has been changed.
++ */
++static inline int get_tree_rmap_item_page(struct rmap_item *tree_rmap_item)
++{
++	int err;
++
++	err = get_mergeable_page_lock_mmap(tree_rmap_item);
++
++	if (err == -EINVAL) {
++		/* its page map has been changed, remove it */
++		remove_rmap_item_from_tree(tree_rmap_item);
++	}
++
++	/* The page is gotten and mmap_sem is locked now. */
++	return err;
++}
++
++
++/**
++ * unstable_tree_search_insert() - search an unstable tree rmap_item with the
++ * same hash value. Get its page and trylock the mmap_sem
++ */
++static inline
++struct rmap_item *unstable_tree_search_insert(struct rmap_item *rmap_item,
++					      u32 hash)
++
++{
++	struct rb_node **new = &root_unstable_tree.rb_node;
++	struct rb_node *parent = NULL;
++	struct tree_node *tree_node;
++	u32 hash_max;
++	struct rmap_item *tree_rmap_item;
++
++	while (*new) {
++		int cmp;
++
++		tree_node = rb_entry(*new, struct tree_node, node);
++
++		cmp = hash_cmp(hash, tree_node->hash);
++
++		if (cmp < 0) {
++			parent = *new;
++			new = &parent->rb_left;
++		} else if (cmp > 0) {
++			parent = *new;
++			new = &parent->rb_right;
++		} else
++			break;
++	}
++
++	if (*new) {
++		/* got the tree_node */
++		if (tree_node->count == 1) {
++			tree_rmap_item = rb_entry(tree_node->sub_root.rb_node,
++						  struct rmap_item, node);
++			BUG_ON(!tree_rmap_item);
++
++			goto get_page_out;
++		}
++
++		/* well, search the collision subtree */
++		new = &tree_node->sub_root.rb_node;
++		BUG_ON(!*new);
++		hash_max = rmap_item_hash_max(rmap_item, hash);
++
++		while (*new) {
++			int cmp;
++
++			tree_rmap_item = rb_entry(*new, struct rmap_item,
++						  node);
++
++			cmp = hash_cmp(hash_max, tree_rmap_item->hash_max);
++			parent = *new;
++			if (cmp < 0)
++				new = &parent->rb_left;
++			else if (cmp > 0)
++				new = &parent->rb_right;
++			else
++				goto get_page_out;
++		}
++	} else {
++		/* alloc a new tree_node */
++		tree_node = alloc_tree_node(&unstable_tree_node_list);
++		if (!tree_node)
++			return NULL;
++
++		tree_node->hash = hash;
++		rb_link_node(&tree_node->node, parent, new);
++		rb_insert_color(&tree_node->node, &root_unstable_tree);
++		parent = NULL;
++		new = &tree_node->sub_root.rb_node;
++	}
++
++	/* did not found even in sub-tree */
++	rmap_item->tree_node = tree_node;
++	rmap_item->address |= UNSTABLE_FLAG;
++	rmap_item->hash_round = uksm_hash_round;
++	rb_link_node(&rmap_item->node, parent, new);
++	rb_insert_color(&rmap_item->node, &tree_node->sub_root);
++
++	uksm_pages_unshared++;
++	return NULL;
++
++get_page_out:
++	if (tree_rmap_item->page == rmap_item->page)
++		return NULL;
++
++	if (get_tree_rmap_item_page(tree_rmap_item))
++		return NULL;
++
++	return tree_rmap_item;
++}
++
++static void hold_anon_vma(struct rmap_item *rmap_item,
++			  struct anon_vma *anon_vma)
++{
++	rmap_item->anon_vma = anon_vma;
++	get_anon_vma(anon_vma);
++}
++
++
++/**
++ * stable_tree_append() - append a rmap_item to a stable node. Deduplication
++ * ratio statistics is done in this function.
++ *
++ */
++static void stable_tree_append(struct rmap_item *rmap_item,
++			       struct stable_node *stable_node, int logdedup)
++{
++	struct node_vma *node_vma = NULL, *new_node_vma, *node_vma_cont = NULL;
++	unsigned long key = (unsigned long)rmap_item->slot;
++	unsigned long factor = rmap_item->slot->rung->step;
++
++	BUG_ON(!stable_node);
++	rmap_item->address |= STABLE_FLAG;
++
++	if (hlist_empty(&stable_node->hlist)) {
++		uksm_pages_shared++;
++		goto node_vma_new;
++	} else {
++		uksm_pages_sharing++;
++	}
++
++	hlist_for_each_entry(node_vma, &stable_node->hlist, hlist) {
++		if (node_vma->key >= key)
++			break;
++
++		if (logdedup) {
++			node_vma->slot->pages_bemerged += factor;
++			if (list_empty(&node_vma->slot->dedup_list))
++				list_add(&node_vma->slot->dedup_list,
++					 &vma_slot_dedup);
++		}
++	}
++
++	if (node_vma) {
++		if (node_vma->key == key) {
++			node_vma_cont = hlist_entry_safe(node_vma->hlist.next, struct node_vma, hlist);
++			goto node_vma_ok;
++		} else if (node_vma->key > key) {
++			node_vma_cont = node_vma;
++		}
++	}
++
++node_vma_new:
++	/* no same vma already in node, alloc a new node_vma */
++	new_node_vma = alloc_node_vma();
++	BUG_ON(!new_node_vma);
++	new_node_vma->head = stable_node;
++	new_node_vma->slot = rmap_item->slot;
++
++	if (!node_vma) {
++		hlist_add_head(&new_node_vma->hlist, &stable_node->hlist);
++	} else if (node_vma->key != key) {
++		if (node_vma->key < key)
++			hlist_add_behind(&new_node_vma->hlist, &node_vma->hlist);
++		else {
++			hlist_add_before(&new_node_vma->hlist,
++					 &node_vma->hlist);
++		}
++
++	}
++	node_vma = new_node_vma;
++
++node_vma_ok: /* ok, ready to add to the list */
++	rmap_item->head = node_vma;
++	hlist_add_head(&rmap_item->hlist, &node_vma->rmap_hlist);
++	hold_anon_vma(rmap_item, rmap_item->slot->vma->anon_vma);
++	if (logdedup) {
++		rmap_item->slot->pages_merged++;
++		if (node_vma_cont) {
++			node_vma = node_vma_cont;
++			hlist_for_each_entry_continue(node_vma, hlist) {
++				node_vma->slot->pages_bemerged += factor;
++				if (list_empty(&node_vma->slot->dedup_list))
++					list_add(&node_vma->slot->dedup_list,
++						 &vma_slot_dedup);
++			}
++		}
++	}
++}
++
++/*
++ * We use break_ksm to break COW on a ksm page: it's a stripped down
++ *
++ *	if (get_user_pages(addr, 1, 1, 1, &page, NULL) == 1)
++ *		put_page(page);
++ *
++ * but taking great care only to touch a ksm page, in a VM_MERGEABLE vma,
++ * in case the application has unmapped and remapped mm,addr meanwhile.
++ * Could a ksm page appear anywhere else?  Actually yes, in a VM_PFNMAP
++ * mmap of /dev/mem or /dev/kmem, where we would not want to touch it.
++ */
++static int break_ksm(struct vm_area_struct *vma, unsigned long addr)
++{
++	struct page *page;
++	int ret = 0;
++
++	do {
++		cond_resched();
++		page = follow_page(vma, addr, FOLL_GET | FOLL_MIGRATION | FOLL_REMOTE);
++		if (IS_ERR_OR_NULL(page))
++			break;
++		if (PageKsm(page)) {
++			ret = handle_mm_fault(vma, addr,
++					      FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE);
++		} else
++			ret = VM_FAULT_WRITE;
++		put_page(page);
++	} while (!(ret & (VM_FAULT_WRITE | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | VM_FAULT_OOM)));
++	/*
++	 * We must loop because handle_mm_fault() may back out if there's
++	 * any difficulty e.g. if pte accessed bit gets updated concurrently.
++	 *
++	 * VM_FAULT_WRITE is what we have been hoping for: it indicates that
++	 * COW has been broken, even if the vma does not permit VM_WRITE;
++	 * but note that a concurrent fault might break PageKsm for us.
++	 *
++	 * VM_FAULT_SIGBUS could occur if we race with truncation of the
++	 * backing file, which also invalidates anonymous pages: that's
++	 * okay, that truncation will have unmapped the PageKsm for us.
++	 *
++	 * VM_FAULT_OOM: at the time of writing (late July 2009), setting
++	 * aside mem_cgroup limits, VM_FAULT_OOM would only be set if the
++	 * current task has TIF_MEMDIE set, and will be OOM killed on return
++	 * to user; and ksmd, having no mm, would never be chosen for that.
++	 *
++	 * But if the mm is in a limited mem_cgroup, then the fault may fail
++	 * with VM_FAULT_OOM even if the current task is not TIF_MEMDIE; and
++	 * even ksmd can fail in this way - though it's usually breaking ksm
++	 * just to undo a merge it made a moment before, so unlikely to oom.
++	 *
++	 * That's a pity: we might therefore have more kernel pages allocated
++	 * than we're counting as nodes in the stable tree; but uksm_do_scan
++	 * will retry to break_cow on each pass, so should recover the page
++	 * in due course.  The important thing is to not let VM_MERGEABLE
++	 * be cleared while any such pages might remain in the area.
++	 */
++	return (ret & VM_FAULT_OOM) ? -ENOMEM : 0;
++}
++
++static void break_cow(struct rmap_item *rmap_item)
++{
++	struct vm_area_struct *vma = rmap_item->slot->vma;
++	struct mm_struct *mm = vma->vm_mm;
++	unsigned long addr = get_rmap_addr(rmap_item);
++
++	if (uksm_test_exit(mm))
++		goto out;
++
++	break_ksm(vma, addr);
++out:
++	return;
++}
++
++/*
++ * Though it's very tempting to unmerge in_stable_tree(rmap_item)s rather
++ * than check every pte of a given vma, the locking doesn't quite work for
++ * that - an rmap_item is assigned to the stable tree after inserting ksm
++ * page and upping mmap_sem.  Nor does it fit with the way we skip dup'ing
++ * rmap_items from parent to child at fork time (so as not to waste time
++ * if exit comes before the next scan reaches it).
++ *
++ * Similarly, although we'd like to remove rmap_items (so updating counts
++ * and freeing memory) when unmerging an area, it's easier to leave that
++ * to the next pass of ksmd - consider, for example, how ksmd might be
++ * in cmp_and_merge_page on one of the rmap_items we would be removing.
++ */
++inline int unmerge_uksm_pages(struct vm_area_struct *vma,
++		      unsigned long start, unsigned long end)
++{
++	unsigned long addr;
++	int err = 0;
++
++	for (addr = start; addr < end && !err; addr += PAGE_SIZE) {
++		if (uksm_test_exit(vma->vm_mm))
++			break;
++		if (signal_pending(current))
++			err = -ERESTARTSYS;
++		else
++			err = break_ksm(vma, addr);
++	}
++	return err;
++}
++
++static inline void inc_uksm_pages_scanned(void)
++{
++	u64 delta;
++
++
++	if (uksm_pages_scanned == U64_MAX) {
++		encode_benefit();
++
++		delta = uksm_pages_scanned >> pages_scanned_base;
++
++		if (CAN_OVERFLOW_U64(pages_scanned_stored, delta)) {
++			pages_scanned_stored >>= 1;
++			delta >>= 1;
++			pages_scanned_base++;
++		}
++
++		pages_scanned_stored += delta;
++
++		uksm_pages_scanned = uksm_pages_scanned_last = 0;
++	}
++
++	uksm_pages_scanned++;
++}
++
++static inline int find_zero_page_hash(int strength, u32 hash)
++{
++	return (zero_hash_table[strength] == hash);
++}
++
++static
++int cmp_and_merge_zero_page(struct vm_area_struct *vma, struct page *page)
++{
++	struct page *zero_page = empty_uksm_zero_page;
++	struct mm_struct *mm = vma->vm_mm;
++	pte_t orig_pte = __pte(0);
++	int err = -EFAULT;
++
++	if (uksm_test_exit(mm))
++		goto out;
++
++	if (!trylock_page(page))
++		goto out;
++
++	if (!PageAnon(page))
++		goto out_unlock;
++
++	if (PageTransCompound(page)) {
++		err = split_huge_page(page);
++		if (err)
++			goto out_unlock;
++	}
++
++	if (write_protect_page(vma, page, &orig_pte, 0) == 0) {
++		if (is_page_full_zero(page))
++			err = replace_page(vma, page, zero_page, orig_pte);
++	}
++
++out_unlock:
++	unlock_page(page);
++out:
++	return err;
++}
++
++/*
++ * cmp_and_merge_page() - first see if page can be merged into the stable
++ * tree; if not, compare hash to previous and if it's the same, see if page
++ * can be inserted into the unstable tree, or merged with a page already there
++ * and both transferred to the stable tree.
++ *
++ * @page: the page that we are searching identical page to.
++ * @rmap_item: the reverse mapping into the virtual address of this page
++ */
++static void cmp_and_merge_page(struct rmap_item *rmap_item, u32 hash)
++{
++	struct rmap_item *tree_rmap_item;
++	struct page *page;
++	struct page *kpage = NULL;
++	u32 hash_max;
++	int err;
++	unsigned int success1, success2;
++	struct stable_node *snode;
++	int cmp;
++	struct rb_node *parent = NULL, **new;
++
++	remove_rmap_item_from_tree(rmap_item);
++	page = rmap_item->page;
++
++	/* We first start with searching the page inside the stable tree */
++	kpage = stable_tree_search(rmap_item, hash);
++	if (kpage) {
++		err = try_to_merge_with_uksm_page(rmap_item, kpage,
++						 hash);
++		if (!err) {
++			/*
++			 * The page was successfully merged, add
++			 * its rmap_item to the stable tree.
++			 * page lock is needed because it's
++			 * racing with try_to_unmap_ksm(), etc.
++			 */
++			lock_page(kpage);
++			snode = page_stable_node(kpage);
++			stable_tree_append(rmap_item, snode, 1);
++			unlock_page(kpage);
++			put_page(kpage);
++			return; /* success */
++		}
++		put_page(kpage);
++
++		/*
++		 * if it's a collision and it has been search in sub-rbtree
++		 * (hash_max != 0), we want to abort, because if it is
++		 * successfully merged in unstable tree, the collision trends to
++		 * happen again.
++		 */
++		if (err == MERGE_ERR_COLLI && rmap_item->hash_max)
++			return;
++	}
++
++	tree_rmap_item =
++		unstable_tree_search_insert(rmap_item, hash);
++	if (tree_rmap_item) {
++		err = try_to_merge_two_pages(rmap_item, tree_rmap_item, hash);
++		/*
++		 * As soon as we merge this page, we want to remove the
++		 * rmap_item of the page we have merged with from the unstable
++		 * tree, and insert it instead as new node in the stable tree.
++		 */
++		if (!err) {
++			kpage = page;
++			remove_rmap_item_from_tree(tree_rmap_item);
++			lock_page(kpage);
++			snode = stable_tree_insert(&kpage, hash,
++						   rmap_item, tree_rmap_item,
++						   &success1, &success2);
++
++			/*
++			 * Do not log dedup for tree item, it's not counted as
++			 * scanned in this round.
++			 */
++			if (success2)
++				stable_tree_append(tree_rmap_item, snode, 0);
++
++			/*
++			 * The order of these two stable append is important:
++			 * we are scanning rmap_item.
++			 */
++			if (success1)
++				stable_tree_append(rmap_item, snode, 1);
++
++			/*
++			 * The original kpage may be unlocked inside
++			 * stable_tree_insert() already. This page
++			 * should be unlocked before doing
++			 * break_cow().
++			 */
++			unlock_page(kpage);
++
++			if (!success1)
++				break_cow(rmap_item);
++
++			if (!success2)
++				break_cow(tree_rmap_item);
++
++		} else if (err == MERGE_ERR_COLLI) {
++			BUG_ON(tree_rmap_item->tree_node->count > 1);
++
++			rmap_item_hash_max(tree_rmap_item,
++					   tree_rmap_item->tree_node->hash);
++
++			hash_max = rmap_item_hash_max(rmap_item, hash);
++			cmp = hash_cmp(hash_max, tree_rmap_item->hash_max);
++			parent = &tree_rmap_item->node;
++			if (cmp < 0)
++				new = &parent->rb_left;
++			else if (cmp > 0)
++				new = &parent->rb_right;
++			else
++				goto put_up_out;
++
++			rmap_item->tree_node = tree_rmap_item->tree_node;
++			rmap_item->address |= UNSTABLE_FLAG;
++			rmap_item->hash_round = uksm_hash_round;
++			rb_link_node(&rmap_item->node, parent, new);
++			rb_insert_color(&rmap_item->node,
++					&tree_rmap_item->tree_node->sub_root);
++			rmap_item->tree_node->count++;
++		} else {
++			/*
++			 * either one of the page has changed or they collide
++			 * at the max hash, we consider them as ill items.
++			 */
++			remove_rmap_item_from_tree(tree_rmap_item);
++		}
++put_up_out:
++		put_page(tree_rmap_item->page);
++		up_read(&tree_rmap_item->slot->vma->vm_mm->mmap_sem);
++	}
++}
++
++
++
++
++static inline unsigned long get_pool_index(struct vma_slot *slot,
++					   unsigned long index)
++{
++	unsigned long pool_index;
++
++	pool_index = (sizeof(struct rmap_list_entry *) * index) >> PAGE_SHIFT;
++	if (pool_index >= slot->pool_size)
++		BUG();
++	return pool_index;
++}
++
++static inline unsigned long index_page_offset(unsigned long index)
++{
++	return offset_in_page(sizeof(struct rmap_list_entry *) * index);
++}
++
++static inline
++struct rmap_list_entry *get_rmap_list_entry(struct vma_slot *slot,
++					    unsigned long index, int need_alloc)
++{
++	unsigned long pool_index;
++	struct page *page;
++	void *addr;
++
++
++	pool_index = get_pool_index(slot, index);
++	if (!slot->rmap_list_pool[pool_index]) {
++		if (!need_alloc)
++			return NULL;
++
++		page = alloc_page(GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN);
++		if (!page)
++			return NULL;
++
++		slot->rmap_list_pool[pool_index] = page;
++	}
++
++	addr = kmap(slot->rmap_list_pool[pool_index]);
++	addr += index_page_offset(index);
++
++	return addr;
++}
++
++static inline void put_rmap_list_entry(struct vma_slot *slot,
++				       unsigned long index)
++{
++	unsigned long pool_index;
++
++	pool_index = get_pool_index(slot, index);
++	BUG_ON(!slot->rmap_list_pool[pool_index]);
++	kunmap(slot->rmap_list_pool[pool_index]);
++}
++
++static inline int entry_is_new(struct rmap_list_entry *entry)
++{
++	return !entry->item;
++}
++
++static inline unsigned long get_index_orig_addr(struct vma_slot *slot,
++						unsigned long index)
++{
++	return slot->vma->vm_start + (index << PAGE_SHIFT);
++}
++
++static inline unsigned long get_entry_address(struct rmap_list_entry *entry)
++{
++	unsigned long addr;
++
++	if (is_addr(entry->addr))
++		addr = get_clean_addr(entry->addr);
++	else if (entry->item)
++		addr = get_rmap_addr(entry->item);
++	else
++		BUG();
++
++	return addr;
++}
++
++static inline struct rmap_item *get_entry_item(struct rmap_list_entry *entry)
++{
++	if (is_addr(entry->addr))
++		return NULL;
++
++	return entry->item;
++}
++
++static inline void inc_rmap_list_pool_count(struct vma_slot *slot,
++					    unsigned long index)
++{
++	unsigned long pool_index;
++
++	pool_index = get_pool_index(slot, index);
++	BUG_ON(!slot->rmap_list_pool[pool_index]);
++	slot->pool_counts[pool_index]++;
++}
++
++static inline void dec_rmap_list_pool_count(struct vma_slot *slot,
++					    unsigned long index)
++{
++	unsigned long pool_index;
++
++	pool_index = get_pool_index(slot, index);
++	BUG_ON(!slot->rmap_list_pool[pool_index]);
++	BUG_ON(!slot->pool_counts[pool_index]);
++	slot->pool_counts[pool_index]--;
++}
++
++static inline int entry_has_rmap(struct rmap_list_entry *entry)
++{
++	return !is_addr(entry->addr) && entry->item;
++}
++
++static inline void swap_entries(struct rmap_list_entry *entry1,
++				unsigned long index1,
++				struct rmap_list_entry *entry2,
++				unsigned long index2)
++{
++	struct rmap_list_entry tmp;
++
++	/* swapping two new entries is meaningless */
++	BUG_ON(entry_is_new(entry1) && entry_is_new(entry2));
++
++	tmp = *entry1;
++	*entry1 = *entry2;
++	*entry2 = tmp;
++
++	if (entry_has_rmap(entry1))
++		entry1->item->entry_index = index1;
++
++	if (entry_has_rmap(entry2))
++		entry2->item->entry_index = index2;
++
++	if (entry_has_rmap(entry1) && !entry_has_rmap(entry2)) {
++		inc_rmap_list_pool_count(entry1->item->slot, index1);
++		dec_rmap_list_pool_count(entry1->item->slot, index2);
++	} else if (!entry_has_rmap(entry1) && entry_has_rmap(entry2)) {
++		inc_rmap_list_pool_count(entry2->item->slot, index2);
++		dec_rmap_list_pool_count(entry2->item->slot, index1);
++	}
++}
++
++static inline void free_entry_item(struct rmap_list_entry *entry)
++{
++	unsigned long index;
++	struct rmap_item *item;
++
++	if (!is_addr(entry->addr)) {
++		BUG_ON(!entry->item);
++		item = entry->item;
++		entry->addr = get_rmap_addr(item);
++		set_is_addr(entry->addr);
++		index = item->entry_index;
++		remove_rmap_item_from_tree(item);
++		dec_rmap_list_pool_count(item->slot, index);
++		free_rmap_item(item);
++	}
++}
++
++static inline int pool_entry_boundary(unsigned long index)
++{
++	unsigned long linear_addr;
++
++	linear_addr = sizeof(struct rmap_list_entry *) * index;
++	return index && !offset_in_page(linear_addr);
++}
++
++static inline void try_free_last_pool(struct vma_slot *slot,
++				      unsigned long index)
++{
++	unsigned long pool_index;
++
++	pool_index = get_pool_index(slot, index);
++	if (slot->rmap_list_pool[pool_index] &&
++	    !slot->pool_counts[pool_index]) {
++		__free_page(slot->rmap_list_pool[pool_index]);
++		slot->rmap_list_pool[pool_index] = NULL;
++		slot->flags |= UKSM_SLOT_NEED_SORT;
++	}
++
++}
++
++static inline unsigned long vma_item_index(struct vm_area_struct *vma,
++					   struct rmap_item *item)
++{
++	return (get_rmap_addr(item) - vma->vm_start) >> PAGE_SHIFT;
++}
++
++static int within_same_pool(struct vma_slot *slot,
++			    unsigned long i, unsigned long j)
++{
++	unsigned long pool_i, pool_j;
++
++	pool_i = get_pool_index(slot, i);
++	pool_j = get_pool_index(slot, j);
++
++	return (pool_i == pool_j);
++}
++
++static void sort_rmap_entry_list(struct vma_slot *slot)
++{
++	unsigned long i, j;
++	struct rmap_list_entry *entry, *swap_entry;
++
++	entry = get_rmap_list_entry(slot, 0, 0);
++	for (i = 0; i < slot->pages; ) {
++
++		if (!entry)
++			goto skip_whole_pool;
++
++		if (entry_is_new(entry))
++			goto next_entry;
++
++		if (is_addr(entry->addr)) {
++			entry->addr = 0;
++			goto next_entry;
++		}
++
++		j = vma_item_index(slot->vma, entry->item);
++		if (j == i)
++			goto next_entry;
++
++		if (within_same_pool(slot, i, j))
++			swap_entry = entry + j - i;
++		else
++			swap_entry = get_rmap_list_entry(slot, j, 1);
++
++		swap_entries(entry, i, swap_entry, j);
++		if (!within_same_pool(slot, i, j))
++			put_rmap_list_entry(slot, j);
++		continue;
++
++skip_whole_pool:
++		i += PAGE_SIZE / sizeof(*entry);
++		if (i < slot->pages)
++			entry = get_rmap_list_entry(slot, i, 0);
++		continue;
++
++next_entry:
++		if (i >= slot->pages - 1 ||
++		    !within_same_pool(slot, i, i + 1)) {
++			put_rmap_list_entry(slot, i);
++			if (i + 1 < slot->pages)
++				entry = get_rmap_list_entry(slot, i + 1, 0);
++		} else
++			entry++;
++		i++;
++		continue;
++	}
++
++	/* free empty pool entries which contain no rmap_item */
++	/* CAN be simplied to based on only pool_counts when bug freed !!!!! */
++	for (i = 0; i < slot->pool_size; i++) {
++		unsigned char has_rmap;
++		void *addr;
++
++		if (!slot->rmap_list_pool[i])
++			continue;
++
++		has_rmap = 0;
++		addr = kmap(slot->rmap_list_pool[i]);
++		BUG_ON(!addr);
++		for (j = 0; j < PAGE_SIZE / sizeof(*entry); j++) {
++			entry = (struct rmap_list_entry *)addr + j;
++			if (is_addr(entry->addr))
++				continue;
++			if (!entry->item)
++				continue;
++			has_rmap = 1;
++		}
++		kunmap(slot->rmap_list_pool[i]);
++		if (!has_rmap) {
++			BUG_ON(slot->pool_counts[i]);
++			__free_page(slot->rmap_list_pool[i]);
++			slot->rmap_list_pool[i] = NULL;
++		}
++	}
++
++	slot->flags &= ~UKSM_SLOT_NEED_SORT;
++}
++
++/*
++ * vma_fully_scanned() - if all the pages in this slot have been scanned.
++ */
++static inline int vma_fully_scanned(struct vma_slot *slot)
++{
++	return slot->pages_scanned == slot->pages;
++}
++
++/**
++ * get_next_rmap_item() - Get the next rmap_item in a vma_slot according to
++ * its random permutation. This function is embedded with the random
++ * permutation index management code.
++ */
++static struct rmap_item *get_next_rmap_item(struct vma_slot *slot, u32 *hash)
++{
++	unsigned long rand_range, addr, swap_index, scan_index;
++	struct rmap_item *item = NULL;
++	struct rmap_list_entry *scan_entry, *swap_entry = NULL;
++	struct page *page;
++
++	scan_index = swap_index = slot->pages_scanned % slot->pages;
++
++	if (pool_entry_boundary(scan_index))
++		try_free_last_pool(slot, scan_index - 1);
++
++	if (vma_fully_scanned(slot)) {
++		if (slot->flags & UKSM_SLOT_NEED_SORT)
++			slot->flags |= UKSM_SLOT_NEED_RERAND;
++		else
++			slot->flags &= ~UKSM_SLOT_NEED_RERAND;
++		if (slot->flags & UKSM_SLOT_NEED_SORT)
++			sort_rmap_entry_list(slot);
++	}
++
++	scan_entry = get_rmap_list_entry(slot, scan_index, 1);
++	if (!scan_entry)
++		return NULL;
++
++	if (entry_is_new(scan_entry)) {
++		scan_entry->addr = get_index_orig_addr(slot, scan_index);
++		set_is_addr(scan_entry->addr);
++	}
++
++	if (slot->flags & UKSM_SLOT_NEED_RERAND) {
++		rand_range = slot->pages - scan_index;
++		BUG_ON(!rand_range);
++		swap_index = scan_index + (prandom_u32() % rand_range);
++	}
++
++	if (swap_index != scan_index) {
++		swap_entry = get_rmap_list_entry(slot, swap_index, 1);
++		if (entry_is_new(swap_entry)) {
++			swap_entry->addr = get_index_orig_addr(slot,
++							       swap_index);
++			set_is_addr(swap_entry->addr);
++		}
++		swap_entries(scan_entry, scan_index, swap_entry, swap_index);
++	}
++
++	addr = get_entry_address(scan_entry);
++	item = get_entry_item(scan_entry);
++	BUG_ON(addr > slot->vma->vm_end || addr < slot->vma->vm_start);
++
++	page = follow_page(slot->vma, addr, FOLL_GET);
++	if (IS_ERR_OR_NULL(page))
++		goto nopage;
++
++	if (!PageAnon(page))
++		goto putpage;
++
++	/*check is zero_page pfn or uksm_zero_page*/
++	if ((page_to_pfn(page) == zero_pfn)
++			|| (page_to_pfn(page) == uksm_zero_pfn))
++		goto putpage;
++
++	flush_anon_page(slot->vma, page, addr);
++	flush_dcache_page(page);
++
++
++	*hash = page_hash(page, hash_strength, 1);
++	inc_uksm_pages_scanned();
++	/*if the page content all zero, re-map to zero-page*/
++	if (find_zero_page_hash(hash_strength, *hash)) {
++		if (!cmp_and_merge_zero_page(slot->vma, page)) {
++			slot->pages_merged++;
++
++			/* For full-zero pages, no need to create rmap item */
++			goto putpage;
++		} else {
++			inc_rshash_neg(memcmp_cost / 2);
++		}
++	}
++
++	if (!item) {
++		item = alloc_rmap_item();
++		if (item) {
++			/* It has already been zeroed */
++			item->slot = slot;
++			item->address = addr;
++			item->entry_index = scan_index;
++			scan_entry->item = item;
++			inc_rmap_list_pool_count(slot, scan_index);
++		} else
++			goto putpage;
++	}
++
++	BUG_ON(item->slot != slot);
++	/* the page may have changed */
++	item->page = page;
++	put_rmap_list_entry(slot, scan_index);
++	if (swap_entry)
++		put_rmap_list_entry(slot, swap_index);
++	return item;
++
++putpage:
++	put_page(page);
++	page = NULL;
++nopage:
++	/* no page, store addr back and free rmap_item if possible */
++	free_entry_item(scan_entry);
++	put_rmap_list_entry(slot, scan_index);
++	if (swap_entry)
++		put_rmap_list_entry(slot, swap_index);
++	return NULL;
++}
++
++static inline int in_stable_tree(struct rmap_item *rmap_item)
++{
++	return rmap_item->address & STABLE_FLAG;
++}
++
++/**
++ * scan_vma_one_page() - scan the next page in a vma_slot. Called with
++ * mmap_sem locked.
++ */
++static noinline void scan_vma_one_page(struct vma_slot *slot)
++{
++	u32 hash;
++	struct mm_struct *mm;
++	struct rmap_item *rmap_item = NULL;
++	struct vm_area_struct *vma = slot->vma;
++
++	mm = vma->vm_mm;
++	BUG_ON(!mm);
++	BUG_ON(!slot);
++
++	rmap_item = get_next_rmap_item(slot, &hash);
++	if (!rmap_item)
++		goto out1;
++
++	if (PageKsm(rmap_item->page) && in_stable_tree(rmap_item))
++		goto out2;
++
++	cmp_and_merge_page(rmap_item, hash);
++out2:
++	put_page(rmap_item->page);
++out1:
++	slot->pages_scanned++;
++	slot->this_sampled++;
++	if (slot->fully_scanned_round != fully_scanned_round)
++		scanned_virtual_pages++;
++
++	if (vma_fully_scanned(slot))
++		slot->fully_scanned_round = fully_scanned_round;
++}
++
++static inline unsigned long rung_get_pages(struct scan_rung *rung)
++{
++	struct slot_tree_node *node;
++
++	if (!rung->vma_root.rnode)
++		return 0;
++
++	node = container_of(rung->vma_root.rnode, struct slot_tree_node, snode);
++
++	return node->size;
++}
++
++#define RUNG_SAMPLED_MIN	3
++
++static inline
++void uksm_calc_rung_step(struct scan_rung *rung,
++			 unsigned long page_time, unsigned long ratio)
++{
++	unsigned long sampled, pages;
++
++	/* will be fully scanned ? */
++	if (!rung->cover_msecs) {
++		rung->step = 1;
++		return;
++	}
++
++	sampled = rung->cover_msecs * (NSEC_PER_MSEC / TIME_RATIO_SCALE)
++		  * ratio / page_time;
++
++	/*
++	 *  Before we finsish a scan round and expensive per-round jobs,
++	 *  we need to have a chance to estimate the per page time. So
++	 *  the sampled number can not be too small.
++	 */
++	if (sampled < RUNG_SAMPLED_MIN)
++		sampled = RUNG_SAMPLED_MIN;
++
++	pages = rung_get_pages(rung);
++	if (likely(pages > sampled))
++		rung->step = pages / sampled;
++	else
++		rung->step = 1;
++}
++
++static inline int step_need_recalc(struct scan_rung *rung)
++{
++	unsigned long pages, stepmax;
++
++	pages = rung_get_pages(rung);
++	stepmax = pages / RUNG_SAMPLED_MIN;
++
++	return pages && (rung->step > pages ||
++			 (stepmax && rung->step > stepmax));
++}
++
++static inline
++void reset_current_scan(struct scan_rung *rung, int finished, int step_recalc)
++{
++	struct vma_slot *slot;
++
++	if (finished)
++		rung->flags |= UKSM_RUNG_ROUND_FINISHED;
++
++	if (step_recalc || step_need_recalc(rung)) {
++		uksm_calc_rung_step(rung, uksm_ema_page_time, rung->cpu_ratio);
++		BUG_ON(step_need_recalc(rung));
++	}
++
++	slot_iter_index = prandom_u32() % rung->step;
++	BUG_ON(!rung->vma_root.rnode);
++	slot = sradix_tree_next(&rung->vma_root, NULL, 0, slot_iter);
++	BUG_ON(!slot);
++
++	rung->current_scan = slot;
++	rung->current_offset = slot_iter_index;
++}
++
++static inline struct sradix_tree_root *slot_get_root(struct vma_slot *slot)
++{
++	return &slot->rung->vma_root;
++}
++
++/*
++ * return if resetted.
++ */
++static int advance_current_scan(struct scan_rung *rung)
++{
++	unsigned short n;
++	struct vma_slot *slot, *next = NULL;
++
++	BUG_ON(!rung->vma_root.num);
++
++	slot = rung->current_scan;
++	n = (slot->pages - rung->current_offset) % rung->step;
++	slot_iter_index = rung->step - n;
++	next = sradix_tree_next(&rung->vma_root, slot->snode,
++				slot->sindex, slot_iter);
++
++	if (next) {
++		rung->current_offset = slot_iter_index;
++		rung->current_scan = next;
++		return 0;
++	} else {
++		reset_current_scan(rung, 1, 0);
++		return 1;
++	}
++}
++
++static inline void rung_rm_slot(struct vma_slot *slot)
++{
++	struct scan_rung *rung = slot->rung;
++	struct sradix_tree_root *root;
++
++	if (rung->current_scan == slot)
++		advance_current_scan(rung);
++
++	root = slot_get_root(slot);
++	sradix_tree_delete_from_leaf(root, slot->snode, slot->sindex);
++	slot->snode = NULL;
++	if (step_need_recalc(rung)) {
++		uksm_calc_rung_step(rung, uksm_ema_page_time, rung->cpu_ratio);
++		BUG_ON(step_need_recalc(rung));
++	}
++
++	/* In case advance_current_scan loop back to this slot again */
++	if (rung->vma_root.num && rung->current_scan == slot)
++		reset_current_scan(slot->rung, 1, 0);
++}
++
++static inline void rung_add_new_slots(struct scan_rung *rung,
++			struct vma_slot **slots, unsigned long num)
++{
++	int err;
++	struct vma_slot *slot;
++	unsigned long i;
++	struct sradix_tree_root *root = &rung->vma_root;
++
++	err = sradix_tree_enter(root, (void **)slots, num);
++	BUG_ON(err);
++
++	for (i = 0; i < num; i++) {
++		slot = slots[i];
++		slot->rung = rung;
++		BUG_ON(vma_fully_scanned(slot));
++	}
++
++	if (rung->vma_root.num == num)
++		reset_current_scan(rung, 0, 1);
++}
++
++static inline int rung_add_one_slot(struct scan_rung *rung,
++				     struct vma_slot *slot)
++{
++	int err;
++
++	err = sradix_tree_enter(&rung->vma_root, (void **)&slot, 1);
++	if (err)
++		return err;
++
++	slot->rung = rung;
++	if (rung->vma_root.num == 1)
++		reset_current_scan(rung, 0, 1);
++
++	return 0;
++}
++
++/*
++ * Return true if the slot is deleted from its rung.
++ */
++static inline int vma_rung_enter(struct vma_slot *slot, struct scan_rung *rung)
++{
++	struct scan_rung *old_rung = slot->rung;
++	int err;
++
++	if (old_rung == rung)
++		return 0;
++
++	rung_rm_slot(slot);
++	err = rung_add_one_slot(rung, slot);
++	if (err) {
++		err = rung_add_one_slot(old_rung, slot);
++		WARN_ON(err); /* OOPS, badly OOM, we lost this slot */
++	}
++
++	return 1;
++}
++
++static inline int vma_rung_up(struct vma_slot *slot)
++{
++	struct scan_rung *rung;
++
++	rung = slot->rung;
++	if (slot->rung != &uksm_scan_ladder[SCAN_LADDER_SIZE-1])
++		rung++;
++
++	return vma_rung_enter(slot, rung);
++}
++
++static inline int vma_rung_down(struct vma_slot *slot)
++{
++	struct scan_rung *rung;
++
++	rung = slot->rung;
++	if (slot->rung != &uksm_scan_ladder[0])
++		rung--;
++
++	return vma_rung_enter(slot, rung);
++}
++
++/**
++ * cal_dedup_ratio() - Calculate the deduplication ratio for this slot.
++ */
++static unsigned long cal_dedup_ratio(struct vma_slot *slot)
++{
++	unsigned long ret;
++	unsigned long pages;
++
++	pages = slot->this_sampled;
++	if (!pages)
++		return 0;
++
++	BUG_ON(slot->pages_scanned == slot->last_scanned);
++
++	ret = slot->pages_merged;
++
++	/* Thrashing area filtering */
++	if (ret && uksm_thrash_threshold) {
++		if (slot->pages_cowed * 100 / slot->pages_merged
++		    > uksm_thrash_threshold) {
++			ret = 0;
++		} else {
++			ret = slot->pages_merged - slot->pages_cowed;
++		}
++	}
++
++	return ret * 100 / pages;
++}
++
++/**
++ * cal_dedup_ratio() - Calculate the deduplication ratio for this slot.
++ */
++static unsigned long cal_dedup_ratio_old(struct vma_slot *slot)
++{
++	unsigned long ret;
++	unsigned long pages;
++
++	pages = slot->pages;
++	if (!pages)
++		return 0;
++
++	ret = slot->pages_bemerged;
++
++	/* Thrashing area filtering */
++	if (ret && uksm_thrash_threshold) {
++		if (slot->pages_cowed * 100 / slot->pages_bemerged
++		    > uksm_thrash_threshold) {
++			ret = 0;
++		} else {
++			ret = slot->pages_bemerged - slot->pages_cowed;
++		}
++	}
++
++	return ret * 100 / pages;
++}
++
++/**
++ * stable_node_reinsert() - When the hash_strength has been adjusted, the
++ * stable tree need to be restructured, this is the function re-inserting the
++ * stable node.
++ */
++static inline void stable_node_reinsert(struct stable_node *new_node,
++					struct page *page,
++					struct rb_root *root_treep,
++					struct list_head *tree_node_listp,
++					u32 hash)
++{
++	struct rb_node **new = &root_treep->rb_node;
++	struct rb_node *parent = NULL;
++	struct stable_node *stable_node;
++	struct tree_node *tree_node;
++	struct page *tree_page;
++	int cmp;
++
++	while (*new) {
++		int cmp;
++
++		tree_node = rb_entry(*new, struct tree_node, node);
++
++		cmp = hash_cmp(hash, tree_node->hash);
++
++		if (cmp < 0) {
++			parent = *new;
++			new = &parent->rb_left;
++		} else if (cmp > 0) {
++			parent = *new;
++			new = &parent->rb_right;
++		} else
++			break;
++	}
++
++	if (*new) {
++		/* find a stable tree node with same first level hash value */
++		stable_node_hash_max(new_node, page, hash);
++		if (tree_node->count == 1) {
++			stable_node = rb_entry(tree_node->sub_root.rb_node,
++					       struct stable_node, node);
++			tree_page = get_uksm_page(stable_node, 1, 0);
++			if (tree_page) {
++				stable_node_hash_max(stable_node,
++						      tree_page, hash);
++				put_page(tree_page);
++
++				/* prepare for stable node insertion */
++
++				cmp = hash_cmp(new_node->hash_max,
++						   stable_node->hash_max);
++				parent = &stable_node->node;
++				if (cmp < 0)
++					new = &parent->rb_left;
++				else if (cmp > 0)
++					new = &parent->rb_right;
++				else
++					goto failed;
++
++				goto add_node;
++			} else {
++				/* the only stable_node deleted, the tree node
++				 * was not deleted.
++				 */
++				goto tree_node_reuse;
++			}
++		}
++
++		/* well, search the collision subtree */
++		new = &tree_node->sub_root.rb_node;
++		parent = NULL;
++		BUG_ON(!*new);
++		while (*new) {
++			int cmp;
++
++			stable_node = rb_entry(*new, struct stable_node, node);
++
++			cmp = hash_cmp(new_node->hash_max,
++					   stable_node->hash_max);
++
++			if (cmp < 0) {
++				parent = *new;
++				new = &parent->rb_left;
++			} else if (cmp > 0) {
++				parent = *new;
++				new = &parent->rb_right;
++			} else {
++				/* oh, no, still a collision */
++				goto failed;
++			}
++		}
++
++		goto add_node;
++	}
++
++	/* no tree node found */
++	tree_node = alloc_tree_node(tree_node_listp);
++	if (!tree_node) {
++		printk(KERN_ERR "UKSM: memory allocation error!\n");
++		goto failed;
++	} else {
++		tree_node->hash = hash;
++		rb_link_node(&tree_node->node, parent, new);
++		rb_insert_color(&tree_node->node, root_treep);
++
++tree_node_reuse:
++		/* prepare for stable node insertion */
++		parent = NULL;
++		new = &tree_node->sub_root.rb_node;
++	}
++
++add_node:
++	rb_link_node(&new_node->node, parent, new);
++	rb_insert_color(&new_node->node, &tree_node->sub_root);
++	new_node->tree_node = tree_node;
++	tree_node->count++;
++	return;
++
++failed:
++	/* This can only happen when two nodes have collided
++	 * in two levels.
++	 */
++	new_node->tree_node = NULL;
++	return;
++}
++
++static inline void free_all_tree_nodes(struct list_head *list)
++{
++	struct tree_node *node, *tmp;
++
++	list_for_each_entry_safe(node, tmp, list, all_list) {
++		free_tree_node(node);
++	}
++}
++
++/**
++ * stable_tree_delta_hash() - Delta hash the stable tree from previous hash
++ * strength to the current hash_strength. It re-structures the hole tree.
++ */
++static inline void stable_tree_delta_hash(u32 prev_hash_strength)
++{
++	struct stable_node *node, *tmp;
++	struct rb_root *root_new_treep;
++	struct list_head *new_tree_node_listp;
++
++	stable_tree_index = (stable_tree_index + 1) % 2;
++	root_new_treep = &root_stable_tree[stable_tree_index];
++	new_tree_node_listp = &stable_tree_node_list[stable_tree_index];
++	*root_new_treep = RB_ROOT;
++	BUG_ON(!list_empty(new_tree_node_listp));
++
++	/*
++	 * we need to be safe, the node could be removed by get_uksm_page()
++	 */
++	list_for_each_entry_safe(node, tmp, &stable_node_list, all_list) {
++		void *addr;
++		struct page *node_page;
++		u32 hash;
++
++		/*
++		 * We are completely re-structuring the stable nodes to a new
++		 * stable tree. We don't want to touch the old tree unlinks and
++		 * old tree_nodes. The old tree_nodes will be freed at once.
++		 */
++		node_page = get_uksm_page(node, 0, 0);
++		if (!node_page)
++			continue;
++
++		if (node->tree_node) {
++			hash = node->tree_node->hash;
++
++			addr = kmap_atomic(node_page);
++
++			hash = delta_hash(addr, prev_hash_strength,
++					  hash_strength, hash);
++			kunmap_atomic(addr);
++		} else {
++			/*
++			 *it was not inserted to rbtree due to collision in last
++			 *round scan.
++			 */
++			hash = page_hash(node_page, hash_strength, 0);
++		}
++
++		stable_node_reinsert(node, node_page, root_new_treep,
++				     new_tree_node_listp, hash);
++		put_page(node_page);
++	}
++
++	root_stable_treep = root_new_treep;
++	free_all_tree_nodes(stable_tree_node_listp);
++	BUG_ON(!list_empty(stable_tree_node_listp));
++	stable_tree_node_listp = new_tree_node_listp;
++}
++
++static inline void inc_hash_strength(unsigned long delta)
++{
++	hash_strength += 1 << delta;
++	if (hash_strength > HASH_STRENGTH_MAX)
++		hash_strength = HASH_STRENGTH_MAX;
++}
++
++static inline void dec_hash_strength(unsigned long delta)
++{
++	unsigned long change = 1 << delta;
++
++	if (hash_strength <= change + 1)
++		hash_strength = 1;
++	else
++		hash_strength -= change;
++}
++
++static inline void inc_hash_strength_delta(void)
++{
++	hash_strength_delta++;
++	if (hash_strength_delta > HASH_STRENGTH_DELTA_MAX)
++		hash_strength_delta = HASH_STRENGTH_DELTA_MAX;
++}
++
++/*
++static inline unsigned long get_current_neg_ratio(void)
++{
++	if (!rshash_pos || rshash_neg > rshash_pos)
++		return 100;
++
++	return div64_u64(100 * rshash_neg , rshash_pos);
++}
++*/
++
++static inline unsigned long get_current_neg_ratio(void)
++{
++	u64 pos = benefit.pos;
++	u64 neg = benefit.neg;
++
++	if (!neg)
++		return 0;
++
++	if (!pos || neg > pos)
++		return 100;
++
++	if (neg > div64_u64(U64_MAX, 100))
++		pos = div64_u64(pos, 100);
++	else
++		neg *= 100;
++
++	return div64_u64(neg, pos);
++}
++
++static inline unsigned long get_current_benefit(void)
++{
++	u64 pos = benefit.pos;
++	u64 neg = benefit.neg;
++	u64 scanned = benefit.scanned;
++
++	if (neg > pos)
++		return 0;
++
++	return div64_u64((pos - neg), scanned);
++}
++
++static inline int judge_rshash_direction(void)
++{
++	u64 current_neg_ratio, stable_benefit;
++	u64 current_benefit, delta = 0;
++	int ret = STILL;
++
++	/* Try to probe a value after the boot, and in case the system
++	   are still for a long time. */
++	if ((fully_scanned_round & 0xFFULL) == 10) {
++		ret = OBSCURE;
++		goto out;
++	}
++
++	current_neg_ratio = get_current_neg_ratio();
++
++	if (current_neg_ratio == 0) {
++		rshash_neg_cont_zero++;
++		if (rshash_neg_cont_zero > 2)
++			return GO_DOWN;
++		else
++			return STILL;
++	}
++	rshash_neg_cont_zero = 0;
++
++	if (current_neg_ratio > 90) {
++		ret = GO_UP;
++		goto out;
++	}
++
++	current_benefit = get_current_benefit();
++	stable_benefit = rshash_state.stable_benefit;
++
++	if (!stable_benefit) {
++		ret = OBSCURE;
++		goto out;
++	}
++
++	if (current_benefit > stable_benefit)
++		delta = current_benefit - stable_benefit;
++	else if (current_benefit < stable_benefit)
++		delta = stable_benefit - current_benefit;
++
++	delta = div64_u64(100 * delta , stable_benefit);
++
++	if (delta > 50) {
++		rshash_cont_obscure++;
++		if (rshash_cont_obscure > 2)
++			return OBSCURE;
++		else
++			return STILL;
++	}
++
++out:
++	rshash_cont_obscure = 0;
++	return ret;
++}
++
++/**
++ * rshash_adjust() - The main function to control the random sampling state
++ * machine for hash strength adapting.
++ *
++ * return true if hash_strength has changed.
++ */
++static inline int rshash_adjust(void)
++{
++	unsigned long prev_hash_strength = hash_strength;
++
++	if (!encode_benefit())
++		return 0;
++
++	switch (rshash_state.state) {
++	case RSHASH_STILL:
++		switch (judge_rshash_direction()) {
++		case GO_UP:
++			if (rshash_state.pre_direct == GO_DOWN)
++				hash_strength_delta = 0;
++
++			inc_hash_strength(hash_strength_delta);
++			inc_hash_strength_delta();
++			rshash_state.stable_benefit = get_current_benefit();
++			rshash_state.pre_direct = GO_UP;
++			break;
++
++		case GO_DOWN:
++			if (rshash_state.pre_direct == GO_UP)
++				hash_strength_delta = 0;
++
++			dec_hash_strength(hash_strength_delta);
++			inc_hash_strength_delta();
++			rshash_state.stable_benefit = get_current_benefit();
++			rshash_state.pre_direct = GO_DOWN;
++			break;
++
++		case OBSCURE:
++			rshash_state.stable_point = hash_strength;
++			rshash_state.turn_point_down = hash_strength;
++			rshash_state.turn_point_up = hash_strength;
++			rshash_state.turn_benefit_down = get_current_benefit();
++			rshash_state.turn_benefit_up = get_current_benefit();
++			rshash_state.lookup_window_index = 0;
++			rshash_state.state = RSHASH_TRYDOWN;
++			dec_hash_strength(hash_strength_delta);
++			inc_hash_strength_delta();
++			break;
++
++		case STILL:
++			break;
++		default:
++			BUG();
++		}
++		break;
++
++	case RSHASH_TRYDOWN:
++		if (rshash_state.lookup_window_index++ % 5 == 0)
++			rshash_state.below_count = 0;
++
++		if (get_current_benefit() < rshash_state.stable_benefit)
++			rshash_state.below_count++;
++		else if (get_current_benefit() >
++			 rshash_state.turn_benefit_down) {
++			rshash_state.turn_point_down = hash_strength;
++			rshash_state.turn_benefit_down = get_current_benefit();
++		}
++
++		if (rshash_state.below_count >= 3 ||
++		    judge_rshash_direction() == GO_UP ||
++		    hash_strength == 1) {
++			hash_strength = rshash_state.stable_point;
++			hash_strength_delta = 0;
++			inc_hash_strength(hash_strength_delta);
++			inc_hash_strength_delta();
++			rshash_state.lookup_window_index = 0;
++			rshash_state.state = RSHASH_TRYUP;
++			hash_strength_delta = 0;
++		} else {
++			dec_hash_strength(hash_strength_delta);
++			inc_hash_strength_delta();
++		}
++		break;
++
++	case RSHASH_TRYUP:
++		if (rshash_state.lookup_window_index++ % 5 == 0)
++			rshash_state.below_count = 0;
++
++		if (get_current_benefit() < rshash_state.turn_benefit_down)
++			rshash_state.below_count++;
++		else if (get_current_benefit() > rshash_state.turn_benefit_up) {
++			rshash_state.turn_point_up = hash_strength;
++			rshash_state.turn_benefit_up = get_current_benefit();
++		}
++
++		if (rshash_state.below_count >= 3 ||
++		    judge_rshash_direction() == GO_DOWN ||
++		    hash_strength == HASH_STRENGTH_MAX) {
++			hash_strength = rshash_state.turn_benefit_up >
++				rshash_state.turn_benefit_down ?
++				rshash_state.turn_point_up :
++				rshash_state.turn_point_down;
++
++			rshash_state.state = RSHASH_PRE_STILL;
++		} else {
++			inc_hash_strength(hash_strength_delta);
++			inc_hash_strength_delta();
++		}
++
++		break;
++
++	case RSHASH_NEW:
++	case RSHASH_PRE_STILL:
++		rshash_state.stable_benefit = get_current_benefit();
++		rshash_state.state = RSHASH_STILL;
++		hash_strength_delta = 0;
++		break;
++	default:
++		BUG();
++	}
++
++	/* rshash_neg = rshash_pos = 0; */
++	reset_benefit();
++
++	if (prev_hash_strength != hash_strength)
++		stable_tree_delta_hash(prev_hash_strength);
++
++	return prev_hash_strength != hash_strength;
++}
++
++/**
++ * round_update_ladder() - The main function to do update of all the
++ * adjustments whenever a scan round is finished.
++ */
++static noinline void round_update_ladder(void)
++{
++	int i;
++	unsigned long dedup;
++	struct vma_slot *slot, *tmp_slot;
++
++	for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++		uksm_scan_ladder[i].flags &= ~UKSM_RUNG_ROUND_FINISHED;
++	}
++
++	list_for_each_entry_safe(slot, tmp_slot, &vma_slot_dedup, dedup_list) {
++
++		/* slot may be rung_rm_slot() when mm exits */
++		if (slot->snode) {
++			dedup = cal_dedup_ratio_old(slot);
++			if (dedup && dedup >= uksm_abundant_threshold)
++				vma_rung_up(slot);
++		}
++
++		slot->pages_bemerged = 0;
++		slot->pages_cowed = 0;
++
++		list_del_init(&slot->dedup_list);
++	}
++}
++
++static void uksm_del_vma_slot(struct vma_slot *slot)
++{
++	int i, j;
++	struct rmap_list_entry *entry;
++
++	if (slot->snode) {
++		/*
++		 * In case it just failed when entering the rung, it's not
++		 * necessary.
++		 */
++		rung_rm_slot(slot);
++	}
++
++	if (!list_empty(&slot->dedup_list))
++		list_del(&slot->dedup_list);
++
++	if (!slot->rmap_list_pool || !slot->pool_counts) {
++		/* In case it OOMed in uksm_vma_enter() */
++		goto out;
++	}
++
++	for (i = 0; i < slot->pool_size; i++) {
++		void *addr;
++
++		if (!slot->rmap_list_pool[i])
++			continue;
++
++		addr = kmap(slot->rmap_list_pool[i]);
++		for (j = 0; j < PAGE_SIZE / sizeof(*entry); j++) {
++			entry = (struct rmap_list_entry *)addr + j;
++			if (is_addr(entry->addr))
++				continue;
++			if (!entry->item)
++				continue;
++
++			remove_rmap_item_from_tree(entry->item);
++			free_rmap_item(entry->item);
++			slot->pool_counts[i]--;
++		}
++		BUG_ON(slot->pool_counts[i]);
++		kunmap(slot->rmap_list_pool[i]);
++		__free_page(slot->rmap_list_pool[i]);
++	}
++	kfree(slot->rmap_list_pool);
++	kfree(slot->pool_counts);
++
++out:
++	slot->rung = NULL;
++	if (slot->flags & UKSM_SLOT_IN_UKSM) {
++		BUG_ON(uksm_pages_total < slot->pages);
++		uksm_pages_total -= slot->pages;
++	}
++
++	if (slot->fully_scanned_round == fully_scanned_round)
++		scanned_virtual_pages -= slot->pages;
++	else
++		scanned_virtual_pages -= slot->pages_scanned;
++	free_vma_slot(slot);
++}
++
++
++#define SPIN_LOCK_PERIOD	32
++static struct vma_slot *cleanup_slots[SPIN_LOCK_PERIOD];
++static inline void cleanup_vma_slots(void)
++{
++	struct vma_slot *slot;
++	int i;
++
++	i = 0;
++	spin_lock(&vma_slot_list_lock);
++	while (!list_empty(&vma_slot_del)) {
++		slot = list_entry(vma_slot_del.next,
++				  struct vma_slot, slot_list);
++		list_del(&slot->slot_list);
++		cleanup_slots[i++] = slot;
++		if (i == SPIN_LOCK_PERIOD) {
++			spin_unlock(&vma_slot_list_lock);
++			while (--i >= 0)
++				uksm_del_vma_slot(cleanup_slots[i]);
++			i = 0;
++			spin_lock(&vma_slot_list_lock);
++		}
++	}
++	spin_unlock(&vma_slot_list_lock);
++
++	while (--i >= 0)
++		uksm_del_vma_slot(cleanup_slots[i]);
++}
++
++/*
++*expotional moving average formula
++*/
++static inline unsigned long ema(unsigned long curr, unsigned long last_ema)
++{
++	/*
++	 * For a very high burst, even the ema cannot work well, a false very
++	 * high per-page time estimation can result in feedback in very high
++	 * overhead of context swith and rung update -- this will then lead
++	 * to higher per-paper time, this may not converge.
++	 *
++	 * Instead, we try to approach this value in a binary manner.
++	 */
++	if (curr > last_ema * 10)
++		return last_ema * 2;
++
++	return (EMA_ALPHA * curr + (100 - EMA_ALPHA) * last_ema) / 100;
++}
++
++/*
++ * convert cpu ratio in 1/TIME_RATIO_SCALE configured by user to
++ * nanoseconds based on current uksm_sleep_jiffies.
++ */
++static inline unsigned long cpu_ratio_to_nsec(unsigned int ratio)
++{
++	return NSEC_PER_USEC * jiffies_to_usecs(uksm_sleep_jiffies) /
++		(TIME_RATIO_SCALE - ratio) * ratio;
++}
++
++
++static inline unsigned long rung_real_ratio(int cpu_time_ratio)
++{
++	unsigned long ret;
++
++	BUG_ON(!cpu_time_ratio);
++
++	if (cpu_time_ratio > 0)
++		ret = cpu_time_ratio;
++	else
++		ret = (unsigned long)(-cpu_time_ratio) *
++			uksm_max_cpu_percentage / 100UL;
++
++	return ret ? ret : 1;
++}
++
++static noinline void uksm_calc_scan_pages(void)
++{
++	struct scan_rung *ladder = uksm_scan_ladder;
++	unsigned long sleep_usecs, nsecs;
++	unsigned long ratio;
++	int i;
++	unsigned long per_page;
++
++	if (uksm_ema_page_time > 100000 ||
++	    (((unsigned long) uksm_eval_round & (256UL - 1)) == 0UL))
++		uksm_ema_page_time = UKSM_PAGE_TIME_DEFAULT;
++
++	per_page = uksm_ema_page_time;
++	BUG_ON(!per_page);
++
++	/*
++	 * For every 8 eval round, we try to probe a uksm_sleep_jiffies value
++	 * based on saved user input.
++	 */
++	if (((unsigned long) uksm_eval_round & (8UL - 1)) == 0UL)
++		uksm_sleep_jiffies = uksm_sleep_saved;
++
++	/* We require a rung scan at least 1 page in a period. */
++	nsecs = per_page;
++	ratio = rung_real_ratio(ladder[0].cpu_ratio);
++	if (cpu_ratio_to_nsec(ratio) < nsecs) {
++		sleep_usecs = nsecs * (TIME_RATIO_SCALE - ratio) / ratio
++				/ NSEC_PER_USEC;
++		uksm_sleep_jiffies = usecs_to_jiffies(sleep_usecs) + 1;
++	}
++
++	for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++		ratio = rung_real_ratio(ladder[i].cpu_ratio);
++		ladder[i].pages_to_scan = cpu_ratio_to_nsec(ratio) /
++					per_page;
++		BUG_ON(!ladder[i].pages_to_scan);
++		uksm_calc_rung_step(&ladder[i], per_page, ratio);
++	}
++}
++
++/*
++ * From the scan time of this round (ns) to next expected min sleep time
++ * (ms), be careful of the possible overflows. ratio is taken from
++ * rung_real_ratio()
++ */
++static inline
++unsigned int scan_time_to_sleep(unsigned long long scan_time, unsigned long ratio)
++{
++	scan_time >>= 20; /* to msec level now */
++	BUG_ON(scan_time > (ULONG_MAX / TIME_RATIO_SCALE));
++
++	return (unsigned int) ((unsigned long) scan_time *
++			       (TIME_RATIO_SCALE - ratio) / ratio);
++}
++
++#define __round_mask(x, y) ((__typeof__(x))((y)-1))
++#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
++
++static void uksm_vma_enter(struct vma_slot **slots, unsigned long num)
++{
++	struct scan_rung *rung;
++
++	rung = &uksm_scan_ladder[0];
++	rung_add_new_slots(rung, slots, num);
++}
++
++static struct vma_slot *batch_slots[SLOT_TREE_NODE_STORE_SIZE];
++
++static void uksm_enter_all_slots(void)
++{
++	struct vma_slot *slot;
++	unsigned long index;
++	struct list_head empty_vma_list;
++	int i;
++
++	i = 0;
++	index = 0;
++	INIT_LIST_HEAD(&empty_vma_list);
++
++	spin_lock(&vma_slot_list_lock);
++	while (!list_empty(&vma_slot_new)) {
++		slot = list_entry(vma_slot_new.next,
++				  struct vma_slot, slot_list);
++
++		if (!slot->vma->anon_vma) {
++			list_move(&slot->slot_list, &empty_vma_list);
++		} else if (vma_can_enter(slot->vma)) {
++			batch_slots[index++] = slot;
++			list_del_init(&slot->slot_list);
++		} else {
++			list_move(&slot->slot_list, &vma_slot_noadd);
++		}
++
++		if (++i == SPIN_LOCK_PERIOD ||
++		    (index && !(index % SLOT_TREE_NODE_STORE_SIZE))) {
++			spin_unlock(&vma_slot_list_lock);
++
++			if (index && !(index % SLOT_TREE_NODE_STORE_SIZE)) {
++				uksm_vma_enter(batch_slots, index);
++				index = 0;
++			}
++			i = 0;
++			cond_resched();
++			spin_lock(&vma_slot_list_lock);
++		}
++	}
++
++	list_splice(&empty_vma_list, &vma_slot_new);
++
++	spin_unlock(&vma_slot_list_lock);
++
++	if (index)
++		uksm_vma_enter(batch_slots, index);
++
++}
++
++static inline int rung_round_finished(struct scan_rung *rung)
++{
++	return rung->flags & UKSM_RUNG_ROUND_FINISHED;
++}
++
++static inline void judge_slot(struct vma_slot *slot)
++{
++	struct scan_rung *rung = slot->rung;
++	unsigned long dedup;
++	int deleted;
++
++	dedup = cal_dedup_ratio(slot);
++	if (vma_fully_scanned(slot) && uksm_thrash_threshold)
++		deleted = vma_rung_enter(slot, &uksm_scan_ladder[0]);
++	else if (dedup && dedup >= uksm_abundant_threshold)
++		deleted = vma_rung_up(slot);
++	else
++		deleted = vma_rung_down(slot);
++
++	slot->pages_merged = 0;
++	slot->pages_cowed = 0;
++	slot->this_sampled = 0;
++
++	if (vma_fully_scanned(slot)) {
++		slot->pages_scanned = 0;
++	}
++
++	slot->last_scanned = slot->pages_scanned;
++
++	/* If its deleted in above, then rung was already advanced. */
++	if (!deleted)
++		advance_current_scan(rung);
++}
++
++
++static inline int hash_round_finished(void)
++{
++	if (scanned_virtual_pages > (uksm_pages_total >> 2)) {
++		scanned_virtual_pages = 0;
++		if (uksm_pages_scanned)
++			fully_scanned_round++;
++
++		return 1;
++	} else {
++		return 0;
++	}
++}
++
++#define UKSM_MMSEM_BATCH	5
++#define BUSY_RETRY		100
++
++/**
++ * uksm_do_scan()  - the main worker function.
++ */
++static noinline void uksm_do_scan(void)
++{
++	struct vma_slot *slot, *iter;
++	struct mm_struct *busy_mm;
++	unsigned char round_finished, all_rungs_emtpy;
++	int i, err, mmsem_batch;
++	unsigned long pcost;
++	long long delta_exec;
++	unsigned long vpages, max_cpu_ratio;
++	unsigned long long start_time, end_time, scan_time;
++	unsigned int expected_jiffies;
++
++	might_sleep();
++
++	vpages = 0;
++
++	start_time = task_sched_runtime(current);
++	max_cpu_ratio = 0;
++	mmsem_batch = 0;
++
++	for (i = 0; i < SCAN_LADDER_SIZE;) {
++		struct scan_rung *rung = &uksm_scan_ladder[i];
++		unsigned long ratio;
++		int busy_retry;
++
++		if (!rung->pages_to_scan) {
++			i++;
++			continue;
++		}
++
++		if (!rung->vma_root.num) {
++			rung->pages_to_scan = 0;
++			i++;
++			continue;
++		}
++
++		ratio = rung_real_ratio(rung->cpu_ratio);
++		if (ratio > max_cpu_ratio)
++			max_cpu_ratio = ratio;
++
++		busy_retry = BUSY_RETRY;
++		/*
++		 * Do not consider rung_round_finished() here, just used up the
++		 * rung->pages_to_scan quota.
++		 */
++		while (rung->pages_to_scan && rung->vma_root.num &&
++		       likely(!freezing(current))) {
++			int reset = 0;
++
++			slot = rung->current_scan;
++
++			BUG_ON(vma_fully_scanned(slot));
++
++			if (mmsem_batch) {
++				err = 0;
++			} else {
++				err = try_down_read_slot_mmap_sem(slot);
++			}
++
++			if (err == -ENOENT) {
++rm_slot:
++				rung_rm_slot(slot);
++				continue;
++			}
++
++			busy_mm = slot->mm;
++
++			if (err == -EBUSY) {
++				/* skip other vmas on the same mm */
++				do {
++					reset = advance_current_scan(rung);
++					iter = rung->current_scan;
++					busy_retry--;
++					if (iter->vma->vm_mm != busy_mm ||
++					    !busy_retry || reset)
++						break;
++				} while (1);
++
++				if (iter->vma->vm_mm != busy_mm) {
++					continue;
++				} else {
++					/* scan round finsished */
++					break;
++				}
++			}
++
++			BUG_ON(!vma_can_enter(slot->vma));
++			if (uksm_test_exit(slot->vma->vm_mm)) {
++				mmsem_batch = 0;
++				up_read(&slot->vma->vm_mm->mmap_sem);
++				goto rm_slot;
++			}
++
++			if (mmsem_batch)
++				mmsem_batch--;
++			else
++				mmsem_batch = UKSM_MMSEM_BATCH;
++
++			/* Ok, we have take the mmap_sem, ready to scan */
++			scan_vma_one_page(slot);
++			rung->pages_to_scan--;
++			vpages++;
++
++			if (rung->current_offset + rung->step > slot->pages - 1
++			    || vma_fully_scanned(slot)) {
++				up_read(&slot->vma->vm_mm->mmap_sem);
++				judge_slot(slot);
++				mmsem_batch = 0;
++			} else {
++				rung->current_offset += rung->step;
++				if (!mmsem_batch)
++					up_read(&slot->vma->vm_mm->mmap_sem);
++			}
++
++			busy_retry = BUSY_RETRY;
++			cond_resched();
++		}
++
++		if (mmsem_batch) {
++			up_read(&slot->vma->vm_mm->mmap_sem);
++			mmsem_batch = 0;
++		}
++
++		if (freezing(current))
++			break;
++
++		cond_resched();
++	}
++	end_time = task_sched_runtime(current);
++	delta_exec = end_time - start_time;
++
++	if (freezing(current))
++		return;
++
++	cleanup_vma_slots();
++	uksm_enter_all_slots();
++
++	round_finished = 1;
++	all_rungs_emtpy = 1;
++	for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++		struct scan_rung *rung = &uksm_scan_ladder[i];
++
++		if (rung->vma_root.num) {
++			all_rungs_emtpy = 0;
++			if (!rung_round_finished(rung))
++				round_finished = 0;
++		}
++	}
++
++	if (all_rungs_emtpy)
++		round_finished = 0;
++
++	if (round_finished) {
++		round_update_ladder();
++		uksm_eval_round++;
++
++		if (hash_round_finished() && rshash_adjust()) {
++			/* Reset the unstable root iff hash strength changed */
++			uksm_hash_round++;
++			root_unstable_tree = RB_ROOT;
++			free_all_tree_nodes(&unstable_tree_node_list);
++		}
++
++		/*
++		 * A number of pages can hang around indefinitely on per-cpu
++		 * pagevecs, raised page count preventing write_protect_page
++		 * from merging them.  Though it doesn't really matter much,
++		 * it is puzzling to see some stuck in pages_volatile until
++		 * other activity jostles them out, and they also prevented
++		 * LTP's KSM test from succeeding deterministically; so drain
++		 * them here (here rather than on entry to uksm_do_scan(),
++		 * so we don't IPI too often when pages_to_scan is set low).
++		 */
++		lru_add_drain_all();
++	}
++
++
++	if (vpages && delta_exec > 0) {
++		pcost = (unsigned long) delta_exec / vpages;
++		if (likely(uksm_ema_page_time))
++			uksm_ema_page_time = ema(pcost, uksm_ema_page_time);
++		else
++			uksm_ema_page_time = pcost;
++	}
++
++	uksm_calc_scan_pages();
++	uksm_sleep_real = uksm_sleep_jiffies;
++	/* in case of radical cpu bursts, apply the upper bound */
++	end_time = task_sched_runtime(current);
++	if (max_cpu_ratio && end_time > start_time) {
++		scan_time = end_time - start_time;
++		expected_jiffies = msecs_to_jiffies(
++			scan_time_to_sleep(scan_time, max_cpu_ratio));
++
++		if (expected_jiffies > uksm_sleep_real)
++			uksm_sleep_real = expected_jiffies;
++
++		/* We have a 1 second up bound for responsiveness. */
++		if (jiffies_to_msecs(uksm_sleep_real) > MSEC_PER_SEC)
++			uksm_sleep_real = msecs_to_jiffies(1000);
++	}
++
++	return;
++}
++
++static int ksmd_should_run(void)
++{
++	return uksm_run & UKSM_RUN_MERGE;
++}
++
++static int uksm_scan_thread(void *nothing)
++{
++	set_freezable();
++	set_user_nice(current, 5);
++
++	while (!kthread_should_stop()) {
++		mutex_lock(&uksm_thread_mutex);
++		if (ksmd_should_run()) {
++			uksm_do_scan();
++		}
++		mutex_unlock(&uksm_thread_mutex);
++
++		try_to_freeze();
++
++		if (ksmd_should_run()) {
++			schedule_timeout_interruptible(uksm_sleep_real);
++			uksm_sleep_times++;
++		} else {
++			wait_event_freezable(uksm_thread_wait,
++				ksmd_should_run() || kthread_should_stop());
++		}
++	}
++	return 0;
++}
++
++int rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc)
++{
++	struct stable_node *stable_node;
++	struct node_vma *node_vma;
++	struct rmap_item *rmap_item;
++	int ret = SWAP_AGAIN;
++	int search_new_forks = 0;
++	unsigned long address;
++
++	VM_BUG_ON_PAGE(!PageKsm(page), page);
++	VM_BUG_ON_PAGE(!PageLocked(page), page);
++
++	stable_node = page_stable_node(page);
++	if (!stable_node)
++		return ret;
++again:
++	hlist_for_each_entry(node_vma, &stable_node->hlist, hlist) {
++		hlist_for_each_entry(rmap_item, &node_vma->rmap_hlist, hlist) {
++			struct anon_vma *anon_vma = rmap_item->anon_vma;
++			struct anon_vma_chain *vmac;
++			struct vm_area_struct *vma;
++
++			cond_resched();
++			anon_vma_lock_read(anon_vma);
++			anon_vma_interval_tree_foreach(vmac, &anon_vma->rb_root,
++						       0, ULONG_MAX) {
++				cond_resched();
++				vma = vmac->vma;
++				address = get_rmap_addr(rmap_item);
++
++				if (address < vma->vm_start ||
++				    address >= vma->vm_end)
++					continue;
++
++				if ((rmap_item->slot->vma == vma) ==
++				    search_new_forks)
++					continue;
++
++				if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
++					continue;
++
++				ret = rwc->rmap_one(page, vma, address, rwc->arg);
++				if (ret != SWAP_AGAIN) {
++					anon_vma_unlock_read(anon_vma);
++					goto out;
++				}
++
++				if (rwc->done && rwc->done(page)) {
++					anon_vma_unlock_read(anon_vma);
++					goto out;
++				}
++			}
++			anon_vma_unlock_read(anon_vma);
++		}
++	}
++	if (!search_new_forks++)
++		goto again;
++out:
++	return ret;
++}
++
++#ifdef CONFIG_MIGRATION
++/* Common ksm interface but may be specific to uksm */
++void ksm_migrate_page(struct page *newpage, struct page *oldpage)
++{
++	struct stable_node *stable_node;
++
++	VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
++	VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
++	VM_BUG_ON(newpage->mapping != oldpage->mapping);
++
++	stable_node = page_stable_node(newpage);
++	if (stable_node) {
++		VM_BUG_ON(stable_node->kpfn != page_to_pfn(oldpage));
++		stable_node->kpfn = page_to_pfn(newpage);
++		/*
++		 * newpage->mapping was set in advance; now we need smp_wmb()
++		 * to make sure that the new stable_node->kpfn is visible
++		 * to get_ksm_page() before it can see that oldpage->mapping
++		 * has gone stale (or that PageSwapCache has been cleared).
++		 */
++		smp_wmb();
++		set_page_stable_node(oldpage, NULL);
++	}
++}
++#endif /* CONFIG_MIGRATION */
++
++#ifdef CONFIG_MEMORY_HOTREMOVE
++static struct stable_node *uksm_check_stable_tree(unsigned long start_pfn,
++						 unsigned long end_pfn)
++{
++	struct rb_node *node;
++
++	for (node = rb_first(root_stable_treep); node; node = rb_next(node)) {
++		struct stable_node *stable_node;
++
++		stable_node = rb_entry(node, struct stable_node, node);
++		if (stable_node->kpfn >= start_pfn &&
++		    stable_node->kpfn < end_pfn)
++			return stable_node;
++	}
++	return NULL;
++}
++
++static int uksm_memory_callback(struct notifier_block *self,
++			       unsigned long action, void *arg)
++{
++	struct memory_notify *mn = arg;
++	struct stable_node *stable_node;
++
++	switch (action) {
++	case MEM_GOING_OFFLINE:
++		/*
++		 * Keep it very simple for now: just lock out ksmd and
++		 * MADV_UNMERGEABLE while any memory is going offline.
++		 * mutex_lock_nested() is necessary because lockdep was alarmed
++		 * that here we take uksm_thread_mutex inside notifier chain
++		 * mutex, and later take notifier chain mutex inside
++		 * uksm_thread_mutex to unlock it.   But that's safe because both
++		 * are inside mem_hotplug_mutex.
++		 */
++		mutex_lock_nested(&uksm_thread_mutex, SINGLE_DEPTH_NESTING);
++		break;
++
++	case MEM_OFFLINE:
++		/*
++		 * Most of the work is done by page migration; but there might
++		 * be a few stable_nodes left over, still pointing to struct
++		 * pages which have been offlined: prune those from the tree.
++		 */
++		while ((stable_node = uksm_check_stable_tree(mn->start_pfn,
++					mn->start_pfn + mn->nr_pages)) != NULL)
++			remove_node_from_stable_tree(stable_node, 1, 1);
++		/* fallthrough */
++
++	case MEM_CANCEL_OFFLINE:
++		mutex_unlock(&uksm_thread_mutex);
++		break;
++	}
++	return NOTIFY_OK;
++}
++#endif /* CONFIG_MEMORY_HOTREMOVE */
++
++#ifdef CONFIG_SYSFS
++/*
++ * This all compiles without CONFIG_SYSFS, but is a waste of space.
++ */
++
++#define UKSM_ATTR_RO(_name) \
++	static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
++#define UKSM_ATTR(_name) \
++	static struct kobj_attribute _name##_attr = \
++		__ATTR(_name, 0644, _name##_show, _name##_store)
++
++static ssize_t max_cpu_percentage_show(struct kobject *kobj,
++				    struct kobj_attribute *attr, char *buf)
++{
++	return sprintf(buf, "%u\n", uksm_max_cpu_percentage);
++}
++
++static ssize_t max_cpu_percentage_store(struct kobject *kobj,
++				     struct kobj_attribute *attr,
++				     const char *buf, size_t count)
++{
++	unsigned long max_cpu_percentage;
++	int err;
++
++	err = kstrtoul(buf, 10, &max_cpu_percentage);
++	if (err || max_cpu_percentage > 100)
++		return -EINVAL;
++
++	if (max_cpu_percentage == 100)
++		max_cpu_percentage = 99;
++	else if (max_cpu_percentage < 10)
++		max_cpu_percentage = 10;
++
++	uksm_max_cpu_percentage = max_cpu_percentage;
++
++	return count;
++}
++UKSM_ATTR(max_cpu_percentage);
++
++static ssize_t sleep_millisecs_show(struct kobject *kobj,
++				    struct kobj_attribute *attr, char *buf)
++{
++	return sprintf(buf, "%u\n", jiffies_to_msecs(uksm_sleep_jiffies));
++}
++
++static ssize_t sleep_millisecs_store(struct kobject *kobj,
++				     struct kobj_attribute *attr,
++				     const char *buf, size_t count)
++{
++	unsigned long msecs;
++	int err;
++
++	err = kstrtoul(buf, 10, &msecs);
++	if (err || msecs > MSEC_PER_SEC)
++		return -EINVAL;
++
++	uksm_sleep_jiffies = msecs_to_jiffies(msecs);
++	uksm_sleep_saved = uksm_sleep_jiffies;
++
++	return count;
++}
++UKSM_ATTR(sleep_millisecs);
++
++
++static ssize_t cpu_governor_show(struct kobject *kobj,
++				  struct kobj_attribute *attr, char *buf)
++{
++	int n = sizeof(uksm_cpu_governor_str) / sizeof(char *);
++	int i;
++
++	buf[0] = '\0';
++	for (i = 0; i < n ; i++) {
++		if (uksm_cpu_governor == i)
++			strcat(buf, "[");
++
++		strcat(buf, uksm_cpu_governor_str[i]);
++
++		if (uksm_cpu_governor == i)
++			strcat(buf, "]");
++
++		strcat(buf, " ");
++	}
++	strcat(buf, "\n");
++
++	return strlen(buf);
++}
++
++static inline void init_performance_values(void)
++{
++	int i;
++	struct scan_rung *rung;
++	struct uksm_cpu_preset_s *preset = uksm_cpu_preset + uksm_cpu_governor;
++
++
++	for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++		rung = uksm_scan_ladder + i;
++		rung->cpu_ratio = preset->cpu_ratio[i];
++		rung->cover_msecs = preset->cover_msecs[i];
++	}
++
++	uksm_max_cpu_percentage = preset->max_cpu;
++}
++
++static ssize_t cpu_governor_store(struct kobject *kobj,
++				   struct kobj_attribute *attr,
++				   const char *buf, size_t count)
++{
++	int n = sizeof(uksm_cpu_governor_str) / sizeof(char *);
++
++	for (n--; n >=0 ; n--) {
++		if (!strncmp(buf, uksm_cpu_governor_str[n],
++			     strlen(uksm_cpu_governor_str[n])))
++			break;
++	}
++
++	if (n < 0)
++		return -EINVAL;
++	else
++		uksm_cpu_governor = n;
++
++	init_performance_values();
++
++	return count;
++}
++UKSM_ATTR(cpu_governor);
++
++static ssize_t run_show(struct kobject *kobj, struct kobj_attribute *attr,
++			char *buf)
++{
++	return sprintf(buf, "%u\n", uksm_run);
++}
++
++static ssize_t run_store(struct kobject *kobj, struct kobj_attribute *attr,
++			 const char *buf, size_t count)
++{
++	int err;
++	unsigned long flags;
++
++	err = kstrtoul(buf, 10, &flags);
++	if (err || flags > UINT_MAX)
++		return -EINVAL;
++	if (flags > UKSM_RUN_MERGE)
++		return -EINVAL;
++
++	mutex_lock(&uksm_thread_mutex);
++	if (uksm_run != flags) {
++		uksm_run = flags;
++	}
++	mutex_unlock(&uksm_thread_mutex);
++
++	if (flags & UKSM_RUN_MERGE)
++		wake_up_interruptible(&uksm_thread_wait);
++
++	return count;
++}
++UKSM_ATTR(run);
++
++static ssize_t abundant_threshold_show(struct kobject *kobj,
++				     struct kobj_attribute *attr, char *buf)
++{
++	return sprintf(buf, "%u\n", uksm_abundant_threshold);
++}
++
++static ssize_t abundant_threshold_store(struct kobject *kobj,
++				      struct kobj_attribute *attr,
++				      const char *buf, size_t count)
++{
++	int err;
++	unsigned long flags;
++
++	err = kstrtoul(buf, 10, &flags);
++	if (err || flags > 99)
++		return -EINVAL;
++
++	uksm_abundant_threshold = flags;
++
++	return count;
++}
++UKSM_ATTR(abundant_threshold);
++
++static ssize_t thrash_threshold_show(struct kobject *kobj,
++				     struct kobj_attribute *attr, char *buf)
++{
++	return sprintf(buf, "%u\n", uksm_thrash_threshold);
++}
++
++static ssize_t thrash_threshold_store(struct kobject *kobj,
++				      struct kobj_attribute *attr,
++				      const char *buf, size_t count)
++{
++	int err;
++	unsigned long flags;
++
++	err = kstrtoul(buf, 10, &flags);
++	if (err || flags > 99)
++		return -EINVAL;
++
++	uksm_thrash_threshold = flags;
++
++	return count;
++}
++UKSM_ATTR(thrash_threshold);
++
++static ssize_t cpu_ratios_show(struct kobject *kobj,
++			       struct kobj_attribute *attr, char *buf)
++{
++	int i, size;
++	struct scan_rung *rung;
++	char *p = buf;
++
++	for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++		rung = &uksm_scan_ladder[i];
++
++		if (rung->cpu_ratio > 0)
++			size = sprintf(p, "%d ", rung->cpu_ratio);
++		else
++			size = sprintf(p, "MAX/%d ",
++					TIME_RATIO_SCALE / -rung->cpu_ratio);
++
++		p += size;
++	}
++
++	*p++ = '\n';
++	*p = '\0';
++
++	return p - buf;
++}
++
++static ssize_t cpu_ratios_store(struct kobject *kobj,
++				      struct kobj_attribute *attr,
++				      const char *buf, size_t count)
++{
++	int i, cpuratios[SCAN_LADDER_SIZE], err;
++	unsigned long value;
++	struct scan_rung *rung;
++	char *p, *end = NULL;
++
++	p = kzalloc(count, GFP_KERNEL);
++	if (!p)
++		return -ENOMEM;
++
++	memcpy(p, buf, count);
++
++	for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++		if (i != SCAN_LADDER_SIZE -1) {
++			end = strchr(p, ' ');
++			if (!end)
++				return -EINVAL;
++
++			*end = '\0';
++		}
++
++		if (strstr(p, "MAX/")) {
++			p = strchr(p, '/') + 1;
++			err = kstrtoul(p, 10, &value);
++			if (err || value > TIME_RATIO_SCALE || !value)
++				return -EINVAL;
++
++			cpuratios[i] = - (int) (TIME_RATIO_SCALE / value);
++		} else {
++			err = kstrtoul(p, 10, &value);
++			if (err || value > TIME_RATIO_SCALE || !value)
++				return -EINVAL;
++
++			cpuratios[i] = value;
++		}
++
++		p = end + 1;
++	}
++
++	for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++		rung = &uksm_scan_ladder[i];
++
++		rung->cpu_ratio = cpuratios[i];
++	}
++
++	return count;
++}
++UKSM_ATTR(cpu_ratios);
++
++static ssize_t eval_intervals_show(struct kobject *kobj,
++			       struct kobj_attribute *attr, char *buf)
++{
++	int i, size;
++	struct scan_rung *rung;
++	char *p = buf;
++
++	for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++		rung = &uksm_scan_ladder[i];
++		size = sprintf(p, "%u ", rung->cover_msecs);
++		p += size;
++	}
++
++	*p++ = '\n';
++	*p = '\0';
++
++	return p - buf;
++}
++
++static ssize_t eval_intervals_store(struct kobject *kobj,
++				      struct kobj_attribute *attr,
++				      const char *buf, size_t count)
++{
++	int i, err;
++	unsigned long values[SCAN_LADDER_SIZE];
++	struct scan_rung *rung;
++	char *p, *end = NULL;
++	ssize_t ret = count;
++
++	p = kzalloc(count + 2, GFP_KERNEL);
++	if (!p)
++		return -ENOMEM;
++
++	memcpy(p, buf, count);
++
++	for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++		if (i != SCAN_LADDER_SIZE -1) {
++			end = strchr(p, ' ');
++			if (!end) {
++				ret = -EINVAL;
++				goto out;
++			}
++
++			*end = '\0';
++		}
++
++		err = kstrtoul(p, 10, &values[i]);
++		if (err) {
++			ret = -EINVAL;
++			goto out;
++		}
++
++		p = end + 1;
++	}
++
++	for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++		rung = &uksm_scan_ladder[i];
++
++		rung->cover_msecs = values[i];
++	}
++
++out:
++	kfree(p);
++	return ret;
++}
++UKSM_ATTR(eval_intervals);
++
++static ssize_t ema_per_page_time_show(struct kobject *kobj,
++				 struct kobj_attribute *attr, char *buf)
++{
++	return sprintf(buf, "%lu\n", uksm_ema_page_time);
++}
++UKSM_ATTR_RO(ema_per_page_time);
++
++static ssize_t pages_shared_show(struct kobject *kobj,
++				 struct kobj_attribute *attr, char *buf)
++{
++	return sprintf(buf, "%lu\n", uksm_pages_shared);
++}
++UKSM_ATTR_RO(pages_shared);
++
++static ssize_t pages_sharing_show(struct kobject *kobj,
++				  struct kobj_attribute *attr, char *buf)
++{
++	return sprintf(buf, "%lu\n", uksm_pages_sharing);
++}
++UKSM_ATTR_RO(pages_sharing);
++
++static ssize_t pages_unshared_show(struct kobject *kobj,
++				   struct kobj_attribute *attr, char *buf)
++{
++	return sprintf(buf, "%lu\n", uksm_pages_unshared);
++}
++UKSM_ATTR_RO(pages_unshared);
++
++static ssize_t full_scans_show(struct kobject *kobj,
++			       struct kobj_attribute *attr, char *buf)
++{
++	return sprintf(buf, "%llu\n", fully_scanned_round);
++}
++UKSM_ATTR_RO(full_scans);
++
++static ssize_t pages_scanned_show(struct kobject *kobj,
++				  struct kobj_attribute *attr, char *buf)
++{
++	unsigned long base = 0;
++	u64 delta, ret;
++
++	if (pages_scanned_stored) {
++		base = pages_scanned_base;
++		ret = pages_scanned_stored;
++		delta = uksm_pages_scanned >> base;
++		if (CAN_OVERFLOW_U64(ret, delta)) {
++			ret >>= 1;
++			delta >>= 1;
++			base++;
++			ret += delta;
++		}
++	} else {
++		ret = uksm_pages_scanned;
++	}
++
++	while (ret > ULONG_MAX) {
++		ret >>= 1;
++		base++;
++	}
++
++	if (base)
++		return sprintf(buf, "%lu * 2^%lu\n", (unsigned long)ret, base);
++	else
++		return sprintf(buf, "%lu\n", (unsigned long)ret);
++}
++UKSM_ATTR_RO(pages_scanned);
++
++static ssize_t hash_strength_show(struct kobject *kobj,
++				  struct kobj_attribute *attr, char *buf)
++{
++	return sprintf(buf, "%lu\n", hash_strength);
++}
++UKSM_ATTR_RO(hash_strength);
++
++static ssize_t sleep_times_show(struct kobject *kobj,
++				  struct kobj_attribute *attr, char *buf)
++{
++	return sprintf(buf, "%llu\n", uksm_sleep_times);
++}
++UKSM_ATTR_RO(sleep_times);
++
++
++static struct attribute *uksm_attrs[] = {
++	&max_cpu_percentage_attr.attr,
++	&sleep_millisecs_attr.attr,
++	&cpu_governor_attr.attr,
++	&run_attr.attr,
++	&ema_per_page_time_attr.attr,
++	&pages_shared_attr.attr,
++	&pages_sharing_attr.attr,
++	&pages_unshared_attr.attr,
++	&full_scans_attr.attr,
++	&pages_scanned_attr.attr,
++	&hash_strength_attr.attr,
++	&sleep_times_attr.attr,
++	&thrash_threshold_attr.attr,
++	&abundant_threshold_attr.attr,
++	&cpu_ratios_attr.attr,
++	&eval_intervals_attr.attr,
++	NULL,
++};
++
++static struct attribute_group uksm_attr_group = {
++	.attrs = uksm_attrs,
++	.name = "uksm",
++};
++#endif /* CONFIG_SYSFS */
++
++static inline void init_scan_ladder(void)
++{
++	int i;
++	struct scan_rung *rung;
++
++	for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++		rung = uksm_scan_ladder + i;
++		slot_tree_init_root(&rung->vma_root);
++	}
++
++	init_performance_values();
++	uksm_calc_scan_pages();
++}
++
++static inline int cal_positive_negative_costs(void)
++{
++	struct page *p1, *p2;
++	unsigned char *addr1, *addr2;
++	unsigned long i, time_start, hash_cost;
++	unsigned long loopnum = 0;
++
++	/*IMPORTANT: volatile is needed to prevent over-optimization by gcc. */
++	volatile u32 hash;
++	volatile int ret;
++
++	p1 = alloc_page(GFP_KERNEL);
++	if (!p1)
++		return -ENOMEM;
++
++	p2 = alloc_page(GFP_KERNEL);
++	if (!p2)
++		return -ENOMEM;
++
++	addr1 = kmap_atomic(p1);
++	addr2 = kmap_atomic(p2);
++	memset(addr1, prandom_u32(), PAGE_SIZE);
++	memcpy(addr2, addr1, PAGE_SIZE);
++
++	/* make sure that the two pages differ in last byte */
++	addr2[PAGE_SIZE-1] = ~addr2[PAGE_SIZE-1];
++	kunmap_atomic(addr2);
++	kunmap_atomic(addr1);
++
++	time_start = jiffies;
++	while (jiffies - time_start < 100) {
++		for (i = 0; i < 100; i++)
++			hash = page_hash(p1, HASH_STRENGTH_FULL, 0);
++		loopnum += 100;
++	}
++	hash_cost = (jiffies - time_start);
++
++	time_start = jiffies;
++	for (i = 0; i < loopnum; i++)
++		ret = pages_identical(p1, p2);
++	memcmp_cost = HASH_STRENGTH_FULL * (jiffies - time_start);
++	memcmp_cost /= hash_cost;
++	printk(KERN_INFO "UKSM: relative memcmp_cost = %lu "
++			 "hash=%u cmp_ret=%d.\n",
++	       memcmp_cost, hash, ret);
++
++	__free_page(p1);
++	__free_page(p2);
++	return 0;
++}
++
++static int init_zeropage_hash_table(void)
++{
++	struct page *page;
++	char *addr;
++	int i;
++
++	page = alloc_page(GFP_KERNEL);
++	if (!page)
++		return -ENOMEM;
++
++	addr = kmap_atomic(page);
++	memset(addr, 0, PAGE_SIZE);
++	kunmap_atomic(addr);
++
++	zero_hash_table = kmalloc(HASH_STRENGTH_MAX * sizeof(u32),
++		GFP_KERNEL);
++	if (!zero_hash_table)
++		return -ENOMEM;
++
++	for (i = 0; i < HASH_STRENGTH_MAX; i++)
++		zero_hash_table[i] = page_hash(page, i, 0);
++
++	__free_page(page);
++
++	return 0;
++}
++
++static inline int init_random_sampling(void)
++{
++	unsigned long i;
++	random_nums = kmalloc(PAGE_SIZE, GFP_KERNEL);
++	if (!random_nums)
++		return -ENOMEM;
++
++	for (i = 0; i < HASH_STRENGTH_FULL; i++)
++		random_nums[i] = i;
++
++	for (i = 0; i < HASH_STRENGTH_FULL; i++) {
++		unsigned long rand_range, swap_index, tmp;
++
++		rand_range = HASH_STRENGTH_FULL - i;
++		swap_index = i + prandom_u32() % rand_range;
++		tmp = random_nums[i];
++		random_nums[i] =  random_nums[swap_index];
++		random_nums[swap_index] = tmp;
++	}
++
++	rshash_state.state = RSHASH_NEW;
++	rshash_state.below_count = 0;
++	rshash_state.lookup_window_index = 0;
++
++	return cal_positive_negative_costs();
++}
++
++static int __init uksm_slab_init(void)
++{
++	rmap_item_cache = UKSM_KMEM_CACHE(rmap_item, 0);
++	if (!rmap_item_cache)
++		goto out;
++
++	stable_node_cache = UKSM_KMEM_CACHE(stable_node, 0);
++	if (!stable_node_cache)
++		goto out_free1;
++
++	node_vma_cache = UKSM_KMEM_CACHE(node_vma, 0);
++	if (!node_vma_cache)
++		goto out_free2;
++
++	vma_slot_cache = UKSM_KMEM_CACHE(vma_slot, 0);
++	if (!vma_slot_cache)
++		goto out_free3;
++
++	tree_node_cache = UKSM_KMEM_CACHE(tree_node, 0);
++	if (!tree_node_cache)
++		goto out_free4;
++
++	return 0;
++
++out_free4:
++	kmem_cache_destroy(vma_slot_cache);
++out_free3:
++	kmem_cache_destroy(node_vma_cache);
++out_free2:
++	kmem_cache_destroy(stable_node_cache);
++out_free1:
++	kmem_cache_destroy(rmap_item_cache);
++out:
++	return -ENOMEM;
++}
++
++static void __init uksm_slab_free(void)
++{
++	kmem_cache_destroy(stable_node_cache);
++	kmem_cache_destroy(rmap_item_cache);
++	kmem_cache_destroy(node_vma_cache);
++	kmem_cache_destroy(vma_slot_cache);
++	kmem_cache_destroy(tree_node_cache);
++}
++
++/* Common interface to ksm, different to it. */
++int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
++		unsigned long end, int advice, unsigned long *vm_flags)
++{
++	int err;
++
++	switch (advice) {
++	case MADV_MERGEABLE:
++		return 0;		/* just ignore the advice */
++
++	case MADV_UNMERGEABLE:
++		if (!(*vm_flags & VM_MERGEABLE) || !uksm_flags_can_scan(*vm_flags))
++			return 0;		/* just ignore the advice */
++
++		if (vma->anon_vma) {
++			err = unmerge_uksm_pages(vma, start, end);
++			if (err)
++				return err;
++		}
++
++		uksm_remove_vma(vma);
++		*vm_flags &= ~VM_MERGEABLE;
++		break;
++	}
++
++	return 0;
++}
++
++/* Common interface to ksm, actually the same. */
++struct page *ksm_might_need_to_copy(struct page *page,
++			struct vm_area_struct *vma, unsigned long address)
++{
++	struct anon_vma *anon_vma = page_anon_vma(page);
++	struct page *new_page;
++
++	if (PageKsm(page)) {
++		if (page_stable_node(page))
++			return page;	/* no need to copy it */
++	} else if (!anon_vma) {
++		return page;		/* no need to copy it */
++	} else if (anon_vma->root == vma->anon_vma->root &&
++		 page->index == linear_page_index(vma, address)) {
++		return page;		/* still no need to copy it */
++	}
++	if (!PageUptodate(page))
++		return page;		/* let do_swap_page report the error */
++
++	new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
++	if (new_page) {
++		copy_user_highpage(new_page, page, address, vma);
++
++		SetPageDirty(new_page);
++		__SetPageUptodate(new_page);
++		__SetPageLocked(new_page);
++	}
++
++	return new_page;
++}
++
++static int __init uksm_init(void)
++{
++	struct task_struct *uksm_thread;
++	int err;
++
++	uksm_sleep_jiffies = msecs_to_jiffies(100);
++	uksm_sleep_saved = uksm_sleep_jiffies;
++
++	slot_tree_init();
++	init_scan_ladder();
++
++
++	err = init_random_sampling();
++	if (err)
++		goto out_free2;
++
++	err = uksm_slab_init();
++	if (err)
++		goto out_free1;
++
++	err = init_zeropage_hash_table();
++	if (err)
++		goto out_free0;
++
++	uksm_thread = kthread_run(uksm_scan_thread, NULL, "uksmd");
++	if (IS_ERR(uksm_thread)) {
++		printk(KERN_ERR "uksm: creating kthread failed\n");
++		err = PTR_ERR(uksm_thread);
++		goto out_free;
++	}
++
++#ifdef CONFIG_SYSFS
++	err = sysfs_create_group(mm_kobj, &uksm_attr_group);
++	if (err) {
++		printk(KERN_ERR "uksm: register sysfs failed\n");
++		kthread_stop(uksm_thread);
++		goto out_free;
++	}
++#else
++	uksm_run = UKSM_RUN_MERGE;	/* no way for user to start it */
++
++#endif /* CONFIG_SYSFS */
++
++#ifdef CONFIG_MEMORY_HOTREMOVE
++	/*
++	 * Choose a high priority since the callback takes uksm_thread_mutex:
++	 * later callbacks could only be taking locks which nest within that.
++	 */
++	hotplug_memory_notifier(uksm_memory_callback, 100);
++#endif
++	return 0;
++
++out_free:
++	kfree(zero_hash_table);
++out_free0:
++	uksm_slab_free();
++out_free1:
++	kfree(random_nums);
++out_free2:
++	kfree(uksm_scan_ladder);
++	return err;
++}
++
++#ifdef MODULE
++subsys_initcall(ksm_init);
++#else
++late_initcall(uksm_init);
++#endif
++
+diff --git a/mm/vmstat.c b/mm/vmstat.c
+index 7c28df3..b1f783f 100644
+--- a/mm/vmstat.c
++++ b/mm/vmstat.c
+@@ -974,6 +974,9 @@ const char * const vmstat_text[] = {
+ 	"nr_dirtied",
+ 	"nr_written",
+ 
++#ifdef CONFIG_UKSM
++	"nr_uksm_zero_pages",
++#endif
+ 	/* enum writeback_stat_item counters */
+ 	"nr_dirty_threshold",
+ 	"nr_dirty_background_threshold",
diff --git a/patches_disabled/05-intel-pstate-backport.patch b/patches_disabled/05-intel-pstate-backport.patch
new file mode 100644
index 0000000..9e466e1
--- /dev/null
+++ b/patches_disabled/05-intel-pstate-backport.patch
@@ -0,0 +1,148 @@
+--- linux-4.8/drivers/cpufreq/intel_pstate.c.orig	2016-10-02 19:24:33.000000000 -0400
++++ linux-4.8/drivers/cpufreq/intel_pstate.c	2016-10-09 19:32:01.073141319 -0400
+@@ -181,6 +181,8 @@
+  * @cpu:		CPU number for this instance data
+  * @update_util:	CPUFreq utility callback information
+  * @update_util_set:	CPUFreq utility callback is set
++ * @iowait_boost:	iowait-related boost fraction
++ * @last_update:	Time of the last update.
+  * @pstate:		Stores P state limits for this CPU
+  * @vid:		Stores VID limits for this CPU
+  * @pid:		Stores PID parameters for this CPU
+@@ -206,6 +208,7 @@
+ 	struct vid_data vid;
+ 	struct _pid pid;
+
++	u64	last_update;
+ 	u64	last_sample_time;
+ 	u64	prev_aperf;
+ 	u64	prev_mperf;
+@@ -216,6 +219,7 @@
+ 	struct acpi_processor_performance acpi_perf_data;
+ 	bool valid_pss_table;
+ #endif
++	unsigned int iowait_boost;
+ };
+
+ static struct cpudata **all_cpu_data;
+@@ -229,6 +233,7 @@
+  * @p_gain_pct:		PID proportional gain
+  * @i_gain_pct:		PID integral gain
+  * @d_gain_pct:		PID derivative gain
++ * @boost_iowait:	Whether or not to use iowait boosting.
+  *
+  * Stores per CPU model static PID configuration data.
+  */
+@@ -240,6 +245,7 @@
+ 	int p_gain_pct;
+ 	int d_gain_pct;
+ 	int i_gain_pct;
++	bool boost_iowait;
+ };
+
+ /**
+@@ -1029,7 +1035,7 @@
+ 	},
+ };
+
+-static struct cpu_defaults silvermont_params = {
++static const struct cpu_defaults silvermont_params = {
+ 	.pid_policy = {
+ 		.sample_rate_ms = 10,
+ 		.deadband = 0,
+@@ -1037,6 +1043,7 @@
+ 		.p_gain_pct = 14,
+ 		.d_gain_pct = 0,
+ 		.i_gain_pct = 4,
++		.boost_iowait = true,
+ 	},
+ 	.funcs = {
+ 		.get_max = atom_get_max_pstate,
+@@ -1050,7 +1057,7 @@
+ 	},
+ };
+
+-static struct cpu_defaults airmont_params = {
++static const struct cpu_defaults airmont_params = {
+ 	.pid_policy = {
+ 		.sample_rate_ms = 10,
+ 		.deadband = 0,
+@@ -1058,6 +1065,7 @@
+ 		.p_gain_pct = 14,
+ 		.d_gain_pct = 0,
+ 		.i_gain_pct = 4,
++		.boost_iowait = true,
+ 	},
+ 	.funcs = {
+ 		.get_max = atom_get_max_pstate,
+@@ -1071,7 +1079,7 @@
+ 	},
+ };
+
+-static struct cpu_defaults knl_params = {
++static const struct cpu_defaults knl_params = {
+ 	.pid_policy = {
+ 		.sample_rate_ms = 10,
+ 		.deadband = 0,
+@@ -1091,7 +1099,7 @@
+ 	},
+ };
+
+-static struct cpu_defaults bxt_params = {
++static const struct cpu_defaults bxt_params = {
+ 	.pid_policy = {
+ 		.sample_rate_ms = 10,
+ 		.deadband = 0,
+@@ -1099,6 +1107,7 @@
+ 		.p_gain_pct = 14,
+ 		.d_gain_pct = 0,
+ 		.i_gain_pct = 4,
++		.boost_iowait = true,
+ 	},
+ 	.funcs = {
+ 		.get_max = core_get_max_pstate,
+@@ -1222,36 +1231,18 @@
+ static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu)
+ {
+ 	struct sample *sample = &cpu->sample;
+-	u64 cummulative_iowait, delta_iowait_us;
+-	u64 delta_iowait_mperf;
+-	u64 mperf, now;
+-	int32_t cpu_load;
++	int32_t busy_frac, boost;
+
+-	cummulative_iowait = get_cpu_iowait_time_us(cpu->cpu, &now);
++	busy_frac = div_fp(sample->mperf, sample->tsc);
+
+-	/*
+-	 * Convert iowait time into number of IO cycles spent at max_freq.
+-	 * IO is considered as busy only for the cpu_load algorithm. For
+-	 * performance this is not needed since we always try to reach the
+-	 * maximum P-State, so we are already boosting the IOs.
+-	 */
+-	delta_iowait_us = cummulative_iowait - cpu->prev_cummulative_iowait;
+-	delta_iowait_mperf = div64_u64(delta_iowait_us * cpu->pstate.scaling *
+-		cpu->pstate.max_pstate, MSEC_PER_SEC);
+-
+-	mperf = cpu->sample.mperf + delta_iowait_mperf;
+-	cpu->prev_cummulative_iowait = cummulative_iowait;
++	boost = cpu->iowait_boost;
++	cpu->iowait_boost >>= 1;
+
+-	/*
+-	 * The load can be estimated as the ratio of the mperf counter
+-	 * running at a constant frequency during active periods
+-	 * (C0) and the time stamp counter running at the same frequency
+-	 * also during C-states.
+-	 */
+-	cpu_load = div64_u64(int_tofp(100) * mperf, sample->tsc);
+-	cpu->sample.busy_scaled = cpu_load;
++	if (busy_frac < boost)
++		busy_frac = boost;
+
+-	return get_avg_pstate(cpu) - pid_calc(&cpu->pid, cpu_load);
++	sample->busy_scaled = busy_frac * 100;
++	return get_avg_pstate(cpu) - pid_calc(&cpu->pid, sample->busy_scaled);
+ }
+
+ static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu)
diff --git a/patches_disabled/06-dirty_cow.patch b/patches_disabled/06-dirty_cow.patch
new file mode 100644
index 0000000..0b9d611
--- /dev/null
+++ b/patches_disabled/06-dirty_cow.patch
@@ -0,0 +1,94 @@
+From 19be0eaffa3ac7d8eb6784ad9bdbc7d67ed8e619 Mon Sep 17 00:00:00 2001
+From: Linus Torvalds <torvalds@linux-foundation.org>
+Date: Thu, 13 Oct 2016 13:07:36 -0700
+Subject: mm: remove gup_flags FOLL_WRITE games from __get_user_pages()
+
+This is an ancient bug that was actually attempted to be fixed once
+(badly) by me eleven years ago in commit 4ceb5db9757a ("Fix
+get_user_pages() race for write access") but that was then undone due to
+problems on s390 by commit f33ea7f404e5 ("fix get_user_pages bug").
+
+In the meantime, the s390 situation has long been fixed, and we can now
+fix it by checking the pte_dirty() bit properly (and do it better).  The
+s390 dirty bit was implemented in abf09bed3cce ("s390/mm: implement
+software dirty bits") which made it into v3.9.  Earlier kernels will
+have to look at the page state itself.
+
+Also, the VM has become more scalable, and what used a purely
+theoretical race back then has become easier to trigger.
+
+To fix it, we introduce a new internal FOLL_COW flag to mark the "yes,
+we already did a COW" rather than play racy games with FOLL_WRITE that
+is very fundamental, and then use the pte dirty flag to validate that
+the FOLL_COW flag is still valid.
+
+Reported-and-tested-by: Phil "not Paul" Oester <kernel@linuxace.com>
+Acked-by: Hugh Dickins <hughd@google.com>
+Reviewed-by: Michal Hocko <mhocko@suse.com>
+Cc: Andy Lutomirski <luto@kernel.org>
+Cc: Kees Cook <keescook@chromium.org>
+Cc: Oleg Nesterov <oleg@redhat.com>
+Cc: Willy Tarreau <w@1wt.eu>
+Cc: Nick Piggin <npiggin@gmail.com>
+Cc: Greg Thelen <gthelen@google.com>
+Cc: stable@vger.kernel.org
+Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
+---
+ include/linux/mm.h |  1 +
+ mm/gup.c           | 14 ++++++++++++--
+ 2 files changed, 13 insertions(+), 2 deletions(-)
+
+diff --git a/include/linux/mm.h b/include/linux/mm.h
+index e9caec6..ed85879 100644
+--- a/include/linux/mm.h
++++ b/include/linux/mm.h
+@@ -2232,6 +2232,7 @@ static inline struct page *follow_page(struct vm_area_struct *vma,
+ #define FOLL_TRIED	0x800	/* a retry, previous pass started an IO */
+ #define FOLL_MLOCK	0x1000	/* lock present pages */
+ #define FOLL_REMOTE	0x2000	/* we are working on non-current tsk/mm */
++#define FOLL_COW	0x4000	/* internal GUP flag */
+ 
+ typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
+ 			void *data);
+diff --git a/mm/gup.c b/mm/gup.c
+index 96b2b2f..22cc22e 100644
+--- a/mm/gup.c
++++ b/mm/gup.c
+@@ -60,6 +60,16 @@ static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address,
+ 	return -EEXIST;
+ }
+ 
++/*
++ * FOLL_FORCE can write to even unwritable pte's, but only
++ * after we've gone through a COW cycle and they are dirty.
++ */
++static inline bool can_follow_write_pte(pte_t pte, unsigned int flags)
++{
++	return pte_write(pte) ||
++		((flags & FOLL_FORCE) && (flags & FOLL_COW) && pte_dirty(pte));
++}
++
+ static struct page *follow_page_pte(struct vm_area_struct *vma,
+ 		unsigned long address, pmd_t *pmd, unsigned int flags)
+ {
+@@ -95,7 +105,7 @@ retry:
+ 	}
+ 	if ((flags & FOLL_NUMA) && pte_protnone(pte))
+ 		goto no_page;
+-	if ((flags & FOLL_WRITE) && !pte_write(pte)) {
++	if ((flags & FOLL_WRITE) && !can_follow_write_pte(pte, flags)) {
+ 		pte_unmap_unlock(ptep, ptl);
+ 		return NULL;
+ 	}
+@@ -412,7 +422,7 @@ static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma,
+ 	 * reCOWed by userspace write).
+ 	 */
+ 	if ((ret & VM_FAULT_WRITE) && !(vma->vm_flags & VM_WRITE))
+-		*flags &= ~FOLL_WRITE;
++	        *flags |= FOLL_COW;
+ 	return 0;
+ }
+ 
+-- 
+cgit v0.12
+
diff --git a/patches_disabled/4.4-sched-bfs-467.patch b/patches_disabled/4.4-sched-bfs-467.patch
new file mode 100644
index 0000000..a58da98
--- /dev/null
+++ b/patches_disabled/4.4-sched-bfs-467.patch
@@ -0,0 +1,9153 @@
+The Brain Fuck Scheduler v0.467 by Con Kolivas.
+
+A single shared runqueue O(n) strict fairness earliest deadline first design.
+
+Excellent throughput and latency for 1 to many CPUs on desktop and server
+commodity hardware.
+Not recommended for 4096 cpus.
+
+Scalability is optimal when your workload is equal to the number of CPUs on
+bfs. ie you should ONLY do make -j4 on quad core, -j2 on dual core and so on.
+
+Features SCHED_IDLEPRIO and SCHED_ISO scheduling policies as well.
+You do NOT need to use these policies for good performance, they are purely
+optional for even better performance in extreme conditions.
+
+To run something idleprio, use schedtool like so:
+
+schedtool -D -e make -j4
+
+To run something isoprio, use schedtool like so:
+
+schedtool -I -e amarok
+
+Includes configurable SMT-nice support for better nice level and scheduling
+policy support across SMT (aka hyperthread) sibling CPUs.
+
+Includes accurate sub-tick accounting of tasks so userspace reported
+cpu usage may be very different if you have very short lived tasks.
+
+-ck
+
+---
+
+---
+ Documentation/scheduler/sched-BFS.txt     |  347 +
+ Documentation/sysctl/kernel.txt           |   26 
+ arch/powerpc/platforms/cell/spufs/sched.c |    5 
+ arch/x86/Kconfig                          |   22 
+ drivers/cpufreq/cpufreq.c                 |    7 
+ drivers/cpufreq/cpufreq_conservative.c    |    4 
+ drivers/cpufreq/cpufreq_ondemand.c        |    4 
+ drivers/cpufreq/intel_pstate.c            |    9 
+ fs/proc/base.c                            |    2 
+ include/linux/init_task.h                 |   76 
+ include/linux/ioprio.h                    |    2 
+ include/linux/jiffies.h                   |    2 
+ include/linux/sched.h                     |   93 
+ include/linux/sched/prio.h                |   12 
+ include/uapi/linux/sched.h                |    9 
+ init/Kconfig                              |   54 
+ init/main.c                               |    3 
+ kernel/delayacct.c                        |    2 
+ kernel/exit.c                             |    2 
+ kernel/sched/Makefile                     |   10 
+ kernel/sched/bfs.c                        | 7585 ++++++++++++++++++++++++++++++
+ kernel/sched/bfs_sched.h                  |  181 
+ kernel/sched/idle.c                       |    4 
+ kernel/sched/stats.c                      |    4 
+ kernel/sysctl.c                           |   41 
+ kernel/time/Kconfig                       |    2 
+ kernel/time/posix-cpu-timers.c            |   10 
+ kernel/trace/trace_selftest.c             |    5 
+ lib/Kconfig.debug                         |    2 
+ 29 files changed, 8451 insertions(+), 74 deletions(-)
+
+Index: linux-4.4-ck1/arch/powerpc/platforms/cell/spufs/sched.c
+===================================================================
+--- linux-4.4-ck1.orig/arch/powerpc/platforms/cell/spufs/sched.c	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/arch/powerpc/platforms/cell/spufs/sched.c	2016-03-25 16:03:45.288350392 +1100
+@@ -64,11 +64,6 @@ static struct timer_list spusched_timer;
+ static struct timer_list spuloadavg_timer;
+ 
+ /*
+- * Priority of a normal, non-rt, non-niced'd process (aka nice level 0).
+- */
+-#define NORMAL_PRIO		120
+-
+-/*
+  * Frequency of the spu scheduler tick.  By default we do one SPU scheduler
+  * tick for every 10 CPU scheduler ticks.
+  */
+Index: linux-4.4-ck1/Documentation/scheduler/sched-BFS.txt
+===================================================================
+--- /dev/null	1970-01-01 00:00:00.000000000 +0000
++++ linux-4.4-ck1/Documentation/scheduler/sched-BFS.txt	2016-03-25 16:03:45.288350392 +1100
+@@ -0,0 +1,347 @@
++BFS - The Brain Fuck Scheduler by Con Kolivas.
++
++Goals.
++
++The goal of the Brain Fuck Scheduler, referred to as BFS from here on, is to
++completely do away with the complex designs of the past for the cpu process
++scheduler and instead implement one that is very simple in basic design.
++The main focus of BFS is to achieve excellent desktop interactivity and
++responsiveness without heuristics and tuning knobs that are difficult to
++understand, impossible to model and predict the effect of, and when tuned to
++one workload cause massive detriment to another.
++
++
++Design summary.
++
++BFS is best described as a single runqueue, O(n) lookup, earliest effective
++virtual deadline first design, loosely based on EEVDF (earliest eligible virtual
++deadline first) and my previous Staircase Deadline scheduler. Each component
++shall be described in order to understand the significance of, and reasoning for
++it. The codebase when the first stable version was released was approximately
++9000 lines less code than the existing mainline linux kernel scheduler (in
++2.6.31). This does not even take into account the removal of documentation and
++the cgroups code that is not used.
++
++Design reasoning.
++
++The single runqueue refers to the queued but not running processes for the
++entire system, regardless of the number of CPUs. The reason for going back to
++a single runqueue design is that once multiple runqueues are introduced,
++per-CPU or otherwise, there will be complex interactions as each runqueue will
++be responsible for the scheduling latency and fairness of the tasks only on its
++own runqueue, and to achieve fairness and low latency across multiple CPUs, any
++advantage in throughput of having CPU local tasks causes other disadvantages.
++This is due to requiring a very complex balancing system to at best achieve some
++semblance of fairness across CPUs and can only maintain relatively low latency
++for tasks bound to the same CPUs, not across them. To increase said fairness
++and latency across CPUs, the advantage of local runqueue locking, which makes
++for better scalability, is lost due to having to grab multiple locks.
++
++A significant feature of BFS is that all accounting is done purely based on CPU
++used and nowhere is sleep time used in any way to determine entitlement or
++interactivity. Interactivity "estimators" that use some kind of sleep/run
++algorithm are doomed to fail to detect all interactive tasks, and to falsely tag
++tasks that aren't interactive as being so. The reason for this is that it is
++close to impossible to determine that when a task is sleeping, whether it is
++doing it voluntarily, as in a userspace application waiting for input in the
++form of a mouse click or otherwise, or involuntarily, because it is waiting for
++another thread, process, I/O, kernel activity or whatever. Thus, such an
++estimator will introduce corner cases, and more heuristics will be required to
++cope with those corner cases, introducing more corner cases and failed
++interactivity detection and so on. Interactivity in BFS is built into the design
++by virtue of the fact that tasks that are waking up have not used up their quota
++of CPU time, and have earlier effective deadlines, thereby making it very likely
++they will preempt any CPU bound task of equivalent nice level. See below for
++more information on the virtual deadline mechanism. Even if they do not preempt
++a running task, because the rr interval is guaranteed to have a bound upper
++limit on how long a task will wait for, it will be scheduled within a timeframe
++that will not cause visible interface jitter.
++
++
++Design details.
++
++Task insertion.
++
++BFS inserts tasks into each relevant queue as an O(1) insertion into a double
++linked list. On insertion, *every* running queue is checked to see if the newly
++queued task can run on any idle queue, or preempt the lowest running task on the
++system. This is how the cross-CPU scheduling of BFS achieves significantly lower
++latency per extra CPU the system has. In this case the lookup is, in the worst
++case scenario, O(n) where n is the number of CPUs on the system.
++
++Data protection.
++
++BFS has one single lock protecting the process local data of every task in the
++global queue. Thus every insertion, removal and modification of task data in the
++global runqueue needs to grab the global lock. However, once a task is taken by
++a CPU, the CPU has its own local data copy of the running process' accounting
++information which only that CPU accesses and modifies (such as during a
++timer tick) thus allowing the accounting data to be updated lockless. Once a
++CPU has taken a task to run, it removes it from the global queue. Thus the
++global queue only ever has, at most,
++
++	(number of tasks requesting cpu time) - (number of logical CPUs) + 1
++
++tasks in the global queue. This value is relevant for the time taken to look up
++tasks during scheduling. This will increase if many tasks with CPU affinity set
++in their policy to limit which CPUs they're allowed to run on if they outnumber
++the number of CPUs. The +1 is because when rescheduling a task, the CPU's
++currently running task is put back on the queue. Lookup will be described after
++the virtual deadline mechanism is explained.
++
++Virtual deadline.
++
++The key to achieving low latency, scheduling fairness, and "nice level"
++distribution in BFS is entirely in the virtual deadline mechanism. The one
++tunable in BFS is the rr_interval, or "round robin interval". This is the
++maximum time two SCHED_OTHER (or SCHED_NORMAL, the common scheduling policy)
++tasks of the same nice level will be running for, or looking at it the other
++way around, the longest duration two tasks of the same nice level will be
++delayed for. When a task requests cpu time, it is given a quota (time_slice)
++equal to the rr_interval and a virtual deadline. The virtual deadline is
++offset from the current time in jiffies by this equation:
++
++	jiffies + (prio_ratio * rr_interval)
++
++The prio_ratio is determined as a ratio compared to the baseline of nice -20
++and increases by 10% per nice level. The deadline is a virtual one only in that
++no guarantee is placed that a task will actually be scheduled by this time, but
++it is used to compare which task should go next. There are three components to
++how a task is next chosen. First is time_slice expiration. If a task runs out
++of its time_slice, it is descheduled, the time_slice is refilled, and the
++deadline reset to that formula above. Second is sleep, where a task no longer
++is requesting CPU for whatever reason. The time_slice and deadline are _not_
++adjusted in this case and are just carried over for when the task is next
++scheduled. Third is preemption, and that is when a newly waking task is deemed
++higher priority than a currently running task on any cpu by virtue of the fact
++that it has an earlier virtual deadline than the currently running task. The
++earlier deadline is the key to which task is next chosen for the first and
++second cases. Once a task is descheduled, it is put back on the queue, and an
++O(n) lookup of all queued-but-not-running tasks is done to determine which has
++the earliest deadline and that task is chosen to receive CPU next.
++
++The CPU proportion of different nice tasks works out to be approximately the
++
++	(prio_ratio difference)^2
++
++The reason it is squared is that a task's deadline does not change while it is
++running unless it runs out of time_slice. Thus, even if the time actually
++passes the deadline of another task that is queued, it will not get CPU time
++unless the current running task deschedules, and the time "base" (jiffies) is
++constantly moving.
++
++Task lookup.
++
++BFS has 103 priority queues. 100 of these are dedicated to the static priority
++of realtime tasks, and the remaining 3 are, in order of best to worst priority,
++SCHED_ISO (isochronous), SCHED_NORMAL, and SCHED_IDLEPRIO (idle priority
++scheduling). When a task of these priorities is queued, a bitmap of running
++priorities is set showing which of these priorities has tasks waiting for CPU
++time. When a CPU is made to reschedule, the lookup for the next task to get
++CPU time is performed in the following way:
++
++First the bitmap is checked to see what static priority tasks are queued. If
++any realtime priorities are found, the corresponding queue is checked and the
++first task listed there is taken (provided CPU affinity is suitable) and lookup
++is complete. If the priority corresponds to a SCHED_ISO task, they are also
++taken in FIFO order (as they behave like SCHED_RR). If the priority corresponds
++to either SCHED_NORMAL or SCHED_IDLEPRIO, then the lookup becomes O(n). At this
++stage, every task in the runlist that corresponds to that priority is checked
++to see which has the earliest set deadline, and (provided it has suitable CPU
++affinity) it is taken off the runqueue and given the CPU. If a task has an
++expired deadline, it is taken and the rest of the lookup aborted (as they are
++chosen in FIFO order).
++
++Thus, the lookup is O(n) in the worst case only, where n is as described
++earlier, as tasks may be chosen before the whole task list is looked over.
++
++
++Scalability.
++
++The major limitations of BFS will be that of scalability, as the separate
++runqueue designs will have less lock contention as the number of CPUs rises.
++However they do not scale linearly even with separate runqueues as multiple
++runqueues will need to be locked concurrently on such designs to be able to
++achieve fair CPU balancing, to try and achieve some sort of nice-level fairness
++across CPUs, and to achieve low enough latency for tasks on a busy CPU when
++other CPUs would be more suited. BFS has the advantage that it requires no
++balancing algorithm whatsoever, as balancing occurs by proxy simply because
++all CPUs draw off the global runqueue, in priority and deadline order. Despite
++the fact that scalability is _not_ the prime concern of BFS, it both shows very
++good scalability to smaller numbers of CPUs and is likely a more scalable design
++at these numbers of CPUs.
++
++It also has some very low overhead scalability features built into the design
++when it has been deemed their overhead is so marginal that they're worth adding.
++The first is the local copy of the running process' data to the CPU it's running
++on to allow that data to be updated lockless where possible. Then there is
++deference paid to the last CPU a task was running on, by trying that CPU first
++when looking for an idle CPU to use the next time it's scheduled. Finally there
++is the notion of "sticky" tasks that are flagged when they are involuntarily
++descheduled, meaning they still want further CPU time. This sticky flag is
++used to bias heavily against those tasks being scheduled on a different CPU
++unless that CPU would be otherwise idle. When a cpu frequency governor is used
++that scales with CPU load, such as ondemand, sticky tasks are not scheduled
++on a different CPU at all, preferring instead to go idle. This means the CPU
++they were bound to is more likely to increase its speed while the other CPU
++will go idle, thus speeding up total task execution time and likely decreasing
++power usage. This is the only scenario where BFS will allow a CPU to go idle
++in preference to scheduling a task on the earliest available spare CPU.
++
++The real cost of migrating a task from one CPU to another is entirely dependant
++on the cache footprint of the task, how cache intensive the task is, how long
++it's been running on that CPU to take up the bulk of its cache, how big the CPU
++cache is, how fast and how layered the CPU cache is, how fast a context switch
++is... and so on. In other words, it's close to random in the real world where we
++do more than just one sole workload. The only thing we can be sure of is that
++it's not free. So BFS uses the principle that an idle CPU is a wasted CPU and
++utilising idle CPUs is more important than cache locality, and cache locality
++only plays a part after that.
++
++When choosing an idle CPU for a waking task, the cache locality is determined
++according to where the task last ran and then idle CPUs are ranked from best
++to worst to choose the most suitable idle CPU based on cache locality, NUMA
++node locality and hyperthread sibling business. They are chosen in the
++following preference (if idle):
++
++* Same core, idle or busy cache, idle threads
++* Other core, same cache, idle or busy cache, idle threads.
++* Same node, other CPU, idle cache, idle threads.
++* Same node, other CPU, busy cache, idle threads.
++* Same core, busy threads.
++* Other core, same cache, busy threads.
++* Same node, other CPU, busy threads.
++* Other node, other CPU, idle cache, idle threads.
++* Other node, other CPU, busy cache, idle threads.
++* Other node, other CPU, busy threads.
++
++This shows the SMT or "hyperthread" awareness in the design as well which will
++choose a real idle core first before a logical SMT sibling which already has
++tasks on the physical CPU.
++
++Early benchmarking of BFS suggested scalability dropped off at the 16 CPU mark.
++However this benchmarking was performed on an earlier design that was far less
++scalable than the current one so it's hard to know how scalable it is in terms
++of both CPUs (due to the global runqueue) and heavily loaded machines (due to
++O(n) lookup) at this stage. Note that in terms of scalability, the number of
++_logical_ CPUs matters, not the number of _physical_ CPUs. Thus, a dual (2x)
++quad core (4X) hyperthreaded (2X) machine is effectively a 16X. Newer benchmark
++results are very promising indeed, without needing to tweak any knobs, features
++or options. Benchmark contributions are most welcome.
++
++
++Features
++
++As the initial prime target audience for BFS was the average desktop user, it
++was designed to not need tweaking, tuning or have features set to obtain benefit
++from it. Thus the number of knobs and features has been kept to an absolute
++minimum and should not require extra user input for the vast majority of cases.
++There are precisely 2 tunables, and 2 extra scheduling policies. The rr_interval
++and iso_cpu tunables, and the SCHED_ISO and SCHED_IDLEPRIO policies. In addition
++to this, BFS also uses sub-tick accounting. What BFS does _not_ now feature is
++support for CGROUPS. The average user should neither need to know what these
++are, nor should they need to be using them to have good desktop behaviour.
++
++rr_interval
++
++There is only one "scheduler" tunable, the round robin interval. This can be
++accessed in
++
++	/proc/sys/kernel/rr_interval
++
++The value is in milliseconds, and the default value is set to 6ms. Valid values
++are from 1 to 1000. Decreasing the value will decrease latencies at the cost of
++decreasing throughput, while increasing it will improve throughput, but at the
++cost of worsening latencies. The accuracy of the rr interval is limited by HZ
++resolution of the kernel configuration. Thus, the worst case latencies are
++usually slightly higher than this actual value. BFS uses "dithering" to try and
++minimise the effect the Hz limitation has. The default value of 6 is not an
++arbitrary one. It is based on the fact that humans can detect jitter at
++approximately 7ms, so aiming for much lower latencies is pointless under most
++circumstances. It is worth noting this fact when comparing the latency
++performance of BFS to other schedulers. Worst case latencies being higher than
++7ms are far worse than average latencies not being in the microsecond range.
++Experimentation has shown that rr intervals being increased up to 300 can
++improve throughput but beyond that, scheduling noise from elsewhere prevents
++further demonstrable throughput.
++
++Isochronous scheduling.
++
++Isochronous scheduling is a unique scheduling policy designed to provide
++near-real-time performance to unprivileged (ie non-root) users without the
++ability to starve the machine indefinitely. Isochronous tasks (which means
++"same time") are set using, for example, the schedtool application like so:
++
++	schedtool -I -e amarok
++
++This will start the audio application "amarok" as SCHED_ISO. How SCHED_ISO works
++is that it has a priority level between true realtime tasks and SCHED_NORMAL
++which would allow them to preempt all normal tasks, in a SCHED_RR fashion (ie,
++if multiple SCHED_ISO tasks are running, they purely round robin at rr_interval
++rate). However if ISO tasks run for more than a tunable finite amount of time,
++they are then demoted back to SCHED_NORMAL scheduling. This finite amount of
++time is the percentage of _total CPU_ available across the machine, configurable
++as a percentage in the following "resource handling" tunable (as opposed to a
++scheduler tunable):
++
++	/proc/sys/kernel/iso_cpu
++
++and is set to 70% by default. It is calculated over a rolling 5 second average
++Because it is the total CPU available, it means that on a multi CPU machine, it
++is possible to have an ISO task running as realtime scheduling indefinitely on
++just one CPU, as the other CPUs will be available. Setting this to 100 is the
++equivalent of giving all users SCHED_RR access and setting it to 0 removes the
++ability to run any pseudo-realtime tasks.
++
++A feature of BFS is that it detects when an application tries to obtain a
++realtime policy (SCHED_RR or SCHED_FIFO) and the caller does not have the
++appropriate privileges to use those policies. When it detects this, it will
++give the task SCHED_ISO policy instead. Thus it is transparent to the user.
++Because some applications constantly set their policy as well as their nice
++level, there is potential for them to undo the override specified by the user
++on the command line of setting the policy to SCHED_ISO. To counter this, once
++a task has been set to SCHED_ISO policy, it needs superuser privileges to set
++it back to SCHED_NORMAL. This will ensure the task remains ISO and all child
++processes and threads will also inherit the ISO policy.
++
++Idleprio scheduling.
++
++Idleprio scheduling is a scheduling policy designed to give out CPU to a task
++_only_ when the CPU would be otherwise idle. The idea behind this is to allow
++ultra low priority tasks to be run in the background that have virtually no
++effect on the foreground tasks. This is ideally suited to distributed computing
++clients (like setiathome, folding, mprime etc) but can also be used to start
++a video encode or so on without any slowdown of other tasks. To avoid this
++policy from grabbing shared resources and holding them indefinitely, if it
++detects a state where the task is waiting on I/O, the machine is about to
++suspend to ram and so on, it will transiently schedule them as SCHED_NORMAL. As
++per the Isochronous task management, once a task has been scheduled as IDLEPRIO,
++it cannot be put back to SCHED_NORMAL without superuser privileges. Tasks can
++be set to start as SCHED_IDLEPRIO with the schedtool command like so:
++
++	schedtool -D -e ./mprime
++
++Subtick accounting.
++
++It is surprisingly difficult to get accurate CPU accounting, and in many cases,
++the accounting is done by simply determining what is happening at the precise
++moment a timer tick fires off. This becomes increasingly inaccurate as the
++timer tick frequency (HZ) is lowered. It is possible to create an application
++which uses almost 100% CPU, yet by being descheduled at the right time, records
++zero CPU usage. While the main problem with this is that there are possible
++security implications, it is also difficult to determine how much CPU a task
++really does use. BFS tries to use the sub-tick accounting from the TSC clock,
++where possible, to determine real CPU usage. This is not entirely reliable, but
++is far more likely to produce accurate CPU usage data than the existing designs
++and will not show tasks as consuming no CPU usage when they actually are. Thus,
++the amount of CPU reported as being used by BFS will more accurately represent
++how much CPU the task itself is using (as is shown for example by the 'time'
++application), so the reported values may be quite different to other schedulers.
++Values reported as the 'load' are more prone to problems with this design, but
++per process values are closer to real usage. When comparing throughput of BFS
++to other designs, it is important to compare the actual completed work in terms
++of total wall clock time taken and total work done, rather than the reported
++"cpu usage".
++
++
++Con Kolivas <kernel@kolivas.org> Tue, 5 Apr 2011
+Index: linux-4.4-ck1/Documentation/sysctl/kernel.txt
+===================================================================
+--- linux-4.4-ck1.orig/Documentation/sysctl/kernel.txt	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/Documentation/sysctl/kernel.txt	2016-03-25 16:03:45.289350411 +1100
+@@ -39,6 +39,7 @@ show up in /proc/sys/kernel:
+ - hung_task_timeout_secs
+ - hung_task_warnings
+ - kexec_load_disabled
++- iso_cpu
+ - kptr_restrict
+ - kstack_depth_to_print       [ X86 only ]
+ - l2cr                        [ PPC only ]
+@@ -67,6 +68,7 @@ show up in /proc/sys/kernel:
+ - randomize_va_space
+ - real-root-dev               ==> Documentation/initrd.txt
+ - reboot-cmd                  [ SPARC only ]
++- rr_interval
+ - rtsig-max
+ - rtsig-nr
+ - sem
+@@ -396,6 +398,16 @@ kernel stack.
+ 
+ ==============================================================
+ 
++iso_cpu: (BFS CPU scheduler only).
++
++This sets the percentage cpu that the unprivileged SCHED_ISO tasks can
++run effectively at realtime priority, averaged over a rolling five
++seconds over the -whole- system, meaning all cpus.
++
++Set to 70 (percent) by default.
++
++==============================================================
++
+ l2cr: (PPC only)
+ 
+ This flag controls the L2 cache of G3 processor boards. If
+@@ -735,6 +747,20 @@ rebooting. ???
+ 
+ ==============================================================
+ 
++rr_interval: (BFS CPU scheduler only)
++
++This is the smallest duration that any cpu process scheduling unit
++will run for. Increasing this value can increase throughput of cpu
++bound tasks substantially but at the expense of increased latencies
++overall. Conversely decreasing it will decrease average and maximum
++latencies but at the expense of throughput. This value is in
++milliseconds and the default value chosen depends on the number of
++cpus available at scheduler initialisation with a minimum of 6.
++
++Valid values are from 1-1000.
++
++==============================================================
++
+ rtsig-max & rtsig-nr:
+ 
+ The file rtsig-max can be used to tune the maximum number
+Index: linux-4.4-ck1/fs/proc/base.c
+===================================================================
+--- linux-4.4-ck1.orig/fs/proc/base.c	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/fs/proc/base.c	2016-03-25 16:03:45.289350411 +1100
+@@ -504,7 +504,7 @@ static int proc_pid_schedstat(struct seq
+ 		seq_printf(m, "0 0 0\n");
+ 	else
+ 		seq_printf(m, "%llu %llu %lu\n",
+-		   (unsigned long long)task->se.sum_exec_runtime,
++		   (unsigned long long)tsk_seruntime(task),
+ 		   (unsigned long long)task->sched_info.run_delay,
+ 		   task->sched_info.pcount);
+ 
+Index: linux-4.4-ck1/include/linux/init_task.h
+===================================================================
+--- linux-4.4-ck1.orig/include/linux/init_task.h	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/include/linux/init_task.h	2016-03-25 16:03:45.289350411 +1100
+@@ -157,8 +157,6 @@ extern struct task_group root_task_group
+ # define INIT_VTIME(tsk)
+ #endif
+ 
+-#define INIT_TASK_COMM "swapper"
+-
+ #ifdef CONFIG_RT_MUTEXES
+ # define INIT_RT_MUTEXES(tsk)						\
+ 	.pi_waiters = RB_ROOT,						\
+@@ -187,6 +185,78 @@ extern struct task_group root_task_group
+  *  INIT_TASK is used to set up the first task table, touch at
+  * your own risk!. Base=0, limit=0x1fffff (=2MB)
+  */
++#ifdef CONFIG_SCHED_BFS
++#define INIT_TASK_COMM "BFS"
++#define INIT_TASK(tsk)	\
++{									\
++	.state		= 0,						\
++	.stack		= &init_thread_info,				\
++	.usage		= ATOMIC_INIT(2),				\
++	.flags		= PF_KTHREAD,					\
++	.prio		= NORMAL_PRIO,					\
++	.static_prio	= MAX_PRIO-20,					\
++	.normal_prio	= NORMAL_PRIO,					\
++	.deadline	= 0,						\
++	.policy		= SCHED_NORMAL,					\
++	.cpus_allowed	= CPU_MASK_ALL,					\
++	.mm		= NULL,						\
++	.active_mm	= &init_mm,					\
++	.restart_block = {						\
++		.fn = do_no_restart_syscall,				\
++	},								\
++	.run_list	= LIST_HEAD_INIT(tsk.run_list),			\
++	.time_slice	= HZ,					\
++	.tasks		= LIST_HEAD_INIT(tsk.tasks),			\
++	INIT_PUSHABLE_TASKS(tsk)					\
++	.ptraced	= LIST_HEAD_INIT(tsk.ptraced),			\
++	.ptrace_entry	= LIST_HEAD_INIT(tsk.ptrace_entry),		\
++	.real_parent	= &tsk,						\
++	.parent		= &tsk,						\
++	.children	= LIST_HEAD_INIT(tsk.children),			\
++	.sibling	= LIST_HEAD_INIT(tsk.sibling),			\
++	.group_leader	= &tsk,						\
++	RCU_POINTER_INITIALIZER(real_cred, &init_cred),			\
++	RCU_POINTER_INITIALIZER(cred, &init_cred),			\
++	.comm		= INIT_TASK_COMM,				\
++	.thread		= INIT_THREAD,					\
++	.fs		= &init_fs,					\
++	.files		= &init_files,					\
++	.signal		= &init_signals,				\
++	.sighand	= &init_sighand,				\
++	.nsproxy	= &init_nsproxy,				\
++	.pending	= {						\
++		.list = LIST_HEAD_INIT(tsk.pending.list),		\
++		.signal = {{0}}},					\
++	.blocked	= {{0}},					\
++	.alloc_lock	= __SPIN_LOCK_UNLOCKED(tsk.alloc_lock),		\
++	.journal_info	= NULL,						\
++	.cpu_timers	= INIT_CPU_TIMERS(tsk.cpu_timers),		\
++	.pi_lock	= __RAW_SPIN_LOCK_UNLOCKED(tsk.pi_lock),		\
++	.timer_slack_ns = 50000, /* 50 usec default slack */		\
++	.pids = {							\
++		[PIDTYPE_PID]  = INIT_PID_LINK(PIDTYPE_PID),		\
++		[PIDTYPE_PGID] = INIT_PID_LINK(PIDTYPE_PGID),		\
++		[PIDTYPE_SID]  = INIT_PID_LINK(PIDTYPE_SID),		\
++	},								\
++	.thread_group	= LIST_HEAD_INIT(tsk.thread_group),		\
++	.thread_node	= LIST_HEAD_INIT(init_signals.thread_head),	\
++	INIT_IDS							\
++	INIT_PERF_EVENTS(tsk)						\
++	INIT_TRACE_IRQFLAGS						\
++	INIT_LOCKDEP							\
++	INIT_FTRACE_GRAPH						\
++	INIT_TRACE_RECURSION						\
++	INIT_TASK_RCU_PREEMPT(tsk)					\
++	INIT_TASK_RCU_TASKS(tsk)					\
++	INIT_CPUSET_SEQ(tsk)						\
++	INIT_RT_MUTEXES(tsk)						\
++	INIT_PREV_CPUTIME(tsk)						\
++	INIT_VTIME(tsk)							\
++	INIT_NUMA_BALANCING(tsk)					\
++	INIT_KASAN(tsk)							\
++}
++#else /* CONFIG_SCHED_BFS */
++#define INIT_TASK_COMM "swapper"
+ #define INIT_TASK(tsk)	\
+ {									\
+ 	.state		= 0,						\
+@@ -261,7 +331,7 @@ extern struct task_group root_task_group
+ 	INIT_NUMA_BALANCING(tsk)					\
+ 	INIT_KASAN(tsk)							\
+ }
+-
++#endif /* CONFIG_SCHED_BFS */
+ 
+ #define INIT_CPU_TIMERS(cpu_timers)					\
+ {									\
+Index: linux-4.4-ck1/include/linux/ioprio.h
+===================================================================
+--- linux-4.4-ck1.orig/include/linux/ioprio.h	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/include/linux/ioprio.h	2016-03-25 16:03:45.289350411 +1100
+@@ -52,6 +52,8 @@ enum {
+  */
+ static inline int task_nice_ioprio(struct task_struct *task)
+ {
++	if (iso_task(task))
++		return 0;
+ 	return (task_nice(task) + 20) / 5;
+ }
+ 
+Index: linux-4.4-ck1/include/linux/sched.h
+===================================================================
+--- linux-4.4-ck1.orig/include/linux/sched.h	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/include/linux/sched.h	2016-03-25 16:03:45.290350430 +1100
+@@ -176,7 +176,7 @@ extern void get_iowait_load(unsigned lon
+ 
+ extern void calc_global_load(unsigned long ticks);
+ 
+-#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
++#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) && !defined(CONFIG_SCHED_BFS)
+ extern void update_cpu_load_nohz(void);
+ #else
+ static inline void update_cpu_load_nohz(void) { }
+@@ -339,8 +339,6 @@ extern void init_idle_bootup_task(struct
+ 
+ extern cpumask_var_t cpu_isolated_map;
+ 
+-extern int runqueue_is_locked(int cpu);
+-
+ #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
+ extern void nohz_balance_enter_idle(int cpu);
+ extern void set_cpu_sd_state_idle(void);
+@@ -1382,9 +1380,11 @@ struct task_struct {
+ 	unsigned int flags;	/* per process flags, defined below */
+ 	unsigned int ptrace;
+ 
++#if defined(CONFIG_SMP) || defined(CONFIG_SCHED_BFS)
++	int on_cpu;
++#endif
+ #ifdef CONFIG_SMP
+ 	struct llist_node wake_entry;
+-	int on_cpu;
+ 	unsigned int wakee_flips;
+ 	unsigned long wakee_flip_decay_ts;
+ 	struct task_struct *last_wakee;
+@@ -1392,12 +1392,29 @@ struct task_struct {
+ 	int wake_cpu;
+ #endif
+ 	int on_rq;
+-
+ 	int prio, static_prio, normal_prio;
+ 	unsigned int rt_priority;
++#ifdef CONFIG_SCHED_BFS
++	int time_slice;
++	u64 deadline;
++	struct list_head run_list;
++	u64 last_ran;
++	u64 sched_time; /* sched_clock time spent running */
++#ifdef CONFIG_SMT_NICE
++	int smt_bias; /* Policy/nice level bias across smt siblings */
++#endif
++#ifdef CONFIG_SMP
++	bool sticky; /* Soft affined flag */
++#endif
++#ifdef CONFIG_HOTPLUG_CPU
++	bool zerobound; /* Bound to CPU0 for hotplug */
++#endif
++	unsigned long rt_timeout;
++#else /* CONFIG_SCHED_BFS */
+ 	const struct sched_class *sched_class;
+ 	struct sched_entity se;
+ 	struct sched_rt_entity rt;
++#endif
+ #ifdef CONFIG_CGROUP_SCHED
+ 	struct task_group *sched_task_group;
+ #endif
+@@ -1517,6 +1534,9 @@ struct task_struct {
+ 	int __user *clear_child_tid;		/* CLONE_CHILD_CLEARTID */
+ 
+ 	cputime_t utime, stime, utimescaled, stimescaled;
++#ifdef CONFIG_SCHED_BFS
++	unsigned long utime_pc, stime_pc;
++#endif
+ 	cputime_t gtime;
+ 	struct prev_cputime prev_cputime;
+ #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
+@@ -1829,6 +1849,63 @@ extern int arch_task_struct_size __read_
+ # define arch_task_struct_size (sizeof(struct task_struct))
+ #endif
+ 
++#ifdef CONFIG_SCHED_BFS
++bool grunqueue_is_locked(void);
++void grq_unlock_wait(void);
++void cpu_scaling(int cpu);
++void cpu_nonscaling(int cpu);
++#define tsk_seruntime(t)		((t)->sched_time)
++#define tsk_rttimeout(t)		((t)->rt_timeout)
++
++static inline void tsk_cpus_current(struct task_struct *p)
++{
++}
++
++static inline int runqueue_is_locked(int cpu)
++{
++	return grunqueue_is_locked();
++}
++
++void print_scheduler_version(void);
++
++static inline bool iso_task(struct task_struct *p)
++{
++	return (p->policy == SCHED_ISO);
++}
++#else /* CFS */
++extern int runqueue_is_locked(int cpu);
++static inline void cpu_scaling(int cpu)
++{
++}
++
++static inline void cpu_nonscaling(int cpu)
++{
++}
++#define tsk_seruntime(t)	((t)->se.sum_exec_runtime)
++#define tsk_rttimeout(t)	((t)->rt.timeout)
++
++static inline void tsk_cpus_current(struct task_struct *p)
++{
++	p->nr_cpus_allowed = current->nr_cpus_allowed;
++}
++
++static inline void print_scheduler_version(void)
++{
++	printk(KERN_INFO"CFS CPU scheduler.\n");
++}
++
++static inline bool iso_task(struct task_struct *p)
++{
++	return false;
++}
++
++/* Anyone feel like implementing this? */
++static inline bool above_background_load(void)
++{
++	return false;
++}
++#endif /* CONFIG_SCHED_BFS */
++
+ /* Future-safe accessor for struct task_struct's cpus_allowed. */
+ #define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed)
+ 
+@@ -2244,7 +2321,7 @@ static inline int set_cpus_allowed_ptr(s
+ }
+ #endif
+ 
+-#ifdef CONFIG_NO_HZ_COMMON
++#if defined(CONFIG_NO_HZ_COMMON) && !defined(CONFIG_SCHED_BFS)
+ void calc_load_enter_idle(void);
+ void calc_load_exit_idle(void);
+ #else
+@@ -2317,7 +2394,7 @@ extern unsigned long long
+ task_sched_runtime(struct task_struct *task);
+ 
+ /* sched_exec is called by processes performing an exec */
+-#ifdef CONFIG_SMP
++#if defined(CONFIG_SMP) && !defined(CONFIG_SCHED_BFS)
+ extern void sched_exec(void);
+ #else
+ #define sched_exec()   {}
+@@ -3102,7 +3179,7 @@ static inline unsigned int task_cpu(cons
+ 	return 0;
+ }
+ 
+-static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
++static inline void set_task_cpu(struct task_struct *p, int cpu)
+ {
+ }
+ 
+Index: linux-4.4-ck1/init/Kconfig
+===================================================================
+--- linux-4.4-ck1.orig/init/Kconfig	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/init/Kconfig	2016-03-25 16:03:45.290350430 +1100
+@@ -28,6 +28,20 @@ config BUILDTIME_EXTABLE_SORT
+ 
+ menu "General setup"
+ 
++config SCHED_BFS
++	bool "BFS cpu scheduler"
++	---help---
++	  The Brain Fuck CPU Scheduler for excellent interactivity and
++	  responsiveness on the desktop and solid scalability on normal
++          hardware and commodity servers. Not recommended for 4096 CPUs.
++
++	  Currently incompatible with the Group CPU scheduler, and RCU TORTURE
++          TEST so these options are disabled.
++
++          Say Y here.
++	default y
++
++
+ config BROKEN
+ 	bool
+ 
+@@ -340,7 +354,7 @@ choice
+ # Kind of a stub config for the pure tick based cputime accounting
+ config TICK_CPU_ACCOUNTING
+ 	bool "Simple tick based cputime accounting"
+-	depends on !S390 && !NO_HZ_FULL
++	depends on !S390 && !NO_HZ_FULL && !SCHED_BFS
+ 	help
+ 	  This is the basic tick based cputime accounting that maintains
+ 	  statistics about user, system and idle time spent on per jiffies
+@@ -365,6 +379,7 @@ config VIRT_CPU_ACCOUNTING_GEN
+ 	bool "Full dynticks CPU time accounting"
+ 	depends on HAVE_CONTEXT_TRACKING
+ 	depends on HAVE_VIRT_CPU_ACCOUNTING_GEN
++	depends on !SCHED_BFS
+ 	select VIRT_CPU_ACCOUNTING
+ 	select CONTEXT_TRACKING
+ 	help
+@@ -699,6 +714,7 @@ config RCU_NOCB_CPU
+ 	bool "Offload RCU callback processing from boot-selected CPUs"
+ 	depends on TREE_RCU || PREEMPT_RCU
+ 	depends on RCU_EXPERT || NO_HZ_FULL
++	depends on !SCHED_BFS
+ 	default n
+ 	help
+ 	  Use this option to reduce OS jitter for aggressive HPC or
+@@ -909,6 +925,7 @@ config NUMA_BALANCING
+ 	depends on ARCH_SUPPORTS_NUMA_BALANCING
+ 	depends on !ARCH_WANT_NUMA_VARIABLE_LOCALITY
+ 	depends on SMP && NUMA && MIGRATION
++	depends on !SCHED_BFS
+ 	help
+ 	  This option adds support for automatic NUMA aware memory/task placement.
+ 	  The mechanism is quite primitive and is based on migrating memory when
+@@ -995,6 +1012,7 @@ config PROC_PID_CPUSET
+ 
+ config CGROUP_CPUACCT
+ 	bool "Simple CPU accounting cgroup subsystem"
++	depends on !SCHED_BFS
+ 	help
+ 	  Provides a simple Resource Controller for monitoring the
+ 	  total CPU consumed by the tasks in a cgroup.
+@@ -1080,6 +1098,7 @@ config CGROUP_PERF
+ 
+ menuconfig CGROUP_SCHED
+ 	bool "Group CPU scheduler"
++	depends on !SCHED_BFS
+ 	default n
+ 	help
+ 	  This feature lets CPU scheduler recognize task groups and control CPU
+@@ -1227,6 +1246,7 @@ endif # NAMESPACES
+ 
+ config SCHED_AUTOGROUP
+ 	bool "Automatic process group scheduling"
++	depends on !SCHED_BFS
+ 	select CGROUPS
+ 	select CGROUP_SCHED
+ 	select FAIR_GROUP_SCHED
+@@ -1711,38 +1731,8 @@ config COMPAT_BRK
+ 
+ 	  On non-ancient distros (post-2000 ones) N is usually a safe choice.
+ 
+-choice
+-	prompt "Choose SLAB allocator"
+-	default SLUB
+-	help
+-	   This option allows to select a slab allocator.
+-
+-config SLAB
+-	bool "SLAB"
+-	help
+-	  The regular slab allocator that is established and known to work
+-	  well in all environments. It organizes cache hot objects in
+-	  per cpu and per node queues.
+-
+ config SLUB
+-	bool "SLUB (Unqueued Allocator)"
+-	help
+-	   SLUB is a slab allocator that minimizes cache line usage
+-	   instead of managing queues of cached objects (SLAB approach).
+-	   Per cpu caching is realized using slabs of objects instead
+-	   of queues of objects. SLUB can use memory efficiently
+-	   and has enhanced diagnostics. SLUB is the default choice for
+-	   a slab allocator.
+-
+-config SLOB
+-	depends on EXPERT
+-	bool "SLOB (Simple Allocator)"
+-	help
+-	   SLOB replaces the stock allocator with a drastically simpler
+-	   allocator. SLOB is generally more space efficient but
+-	   does not perform as well on large systems.
+-
+-endchoice
++	def_bool y
+ 
+ config SLUB_CPU_PARTIAL
+ 	default y
+Index: linux-4.4-ck1/init/main.c
+===================================================================
+--- linux-4.4-ck1.orig/init/main.c	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/init/main.c	2016-03-25 16:03:45.290350430 +1100
+@@ -808,7 +808,6 @@ int __init_or_module do_one_initcall(ini
+ 	return ret;
+ }
+ 
+-
+ extern initcall_t __initcall_start[];
+ extern initcall_t __initcall0_start[];
+ extern initcall_t __initcall1_start[];
+@@ -943,6 +942,8 @@ static int __ref kernel_init(void *unuse
+ 
+ 	flush_delayed_fput();
+ 
++	print_scheduler_version();
++
+ 	if (ramdisk_execute_command) {
+ 		ret = run_init_process(ramdisk_execute_command);
+ 		if (!ret)
+Index: linux-4.4-ck1/kernel/delayacct.c
+===================================================================
+--- linux-4.4-ck1.orig/kernel/delayacct.c	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/kernel/delayacct.c	2016-03-25 16:03:45.290350430 +1100
+@@ -104,7 +104,7 @@ int __delayacct_add_tsk(struct taskstats
+ 	 */
+ 	t1 = tsk->sched_info.pcount;
+ 	t2 = tsk->sched_info.run_delay;
+-	t3 = tsk->se.sum_exec_runtime;
++	t3 = tsk_seruntime(tsk);
+ 
+ 	d->cpu_count += t1;
+ 
+Index: linux-4.4-ck1/kernel/exit.c
+===================================================================
+--- linux-4.4-ck1.orig/kernel/exit.c	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/kernel/exit.c	2016-03-25 16:03:45.290350430 +1100
+@@ -135,7 +135,7 @@ static void __exit_signal(struct task_st
+ 	sig->inblock += task_io_get_inblock(tsk);
+ 	sig->oublock += task_io_get_oublock(tsk);
+ 	task_io_accounting_add(&sig->ioac, &tsk->ioac);
+-	sig->sum_sched_runtime += tsk->se.sum_exec_runtime;
++	sig->sum_sched_runtime += tsk_seruntime(tsk);
+ 	sig->nr_threads--;
+ 	__unhash_process(tsk, group_dead);
+ 	write_sequnlock(&sig->stats_lock);
+Index: linux-4.4-ck1/kernel/sysctl.c
+===================================================================
+--- linux-4.4-ck1.orig/kernel/sysctl.c	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/kernel/sysctl.c	2016-03-25 16:03:45.291350449 +1100
+@@ -125,7 +125,13 @@ static int __maybe_unused one = 1;
+ static int __maybe_unused two = 2;
+ static int __maybe_unused four = 4;
+ static unsigned long one_ul = 1;
+-static int one_hundred = 100;
++static int __maybe_unused one_hundred = 100;
++#ifdef CONFIG_SCHED_BFS
++extern int rr_interval;
++extern int sched_interactive;
++extern int sched_iso_cpu;
++static int __read_mostly one_thousand = 1000;
++#endif
+ #ifdef CONFIG_PRINTK
+ static int ten_thousand = 10000;
+ #endif
+@@ -260,7 +266,7 @@ static struct ctl_table sysctl_base_tabl
+ 	{ }
+ };
+ 
+-#ifdef CONFIG_SCHED_DEBUG
++#if defined(CONFIG_SCHED_DEBUG) && !defined(CONFIG_SCHED_BFS)
+ static int min_sched_granularity_ns = 100000;		/* 100 usecs */
+ static int max_sched_granularity_ns = NSEC_PER_SEC;	/* 1 second */
+ static int min_wakeup_granularity_ns;			/* 0 usecs */
+@@ -277,6 +283,7 @@ static int max_extfrag_threshold = 1000;
+ #endif
+ 
+ static struct ctl_table kern_table[] = {
++#ifndef CONFIG_SCHED_BFS
+ 	{
+ 		.procname	= "sched_child_runs_first",
+ 		.data		= &sysctl_sched_child_runs_first,
+@@ -434,6 +441,7 @@ static struct ctl_table kern_table[] = {
+ 		.extra1		= &one,
+ 	},
+ #endif
++#endif /* !CONFIG_SCHED_BFS */
+ #ifdef CONFIG_PROVE_LOCKING
+ 	{
+ 		.procname	= "prove_locking",
+@@ -991,6 +999,35 @@ static struct ctl_table kern_table[] = {
+ 		.proc_handler	= proc_dointvec,
+ 	},
+ #endif
++#ifdef CONFIG_SCHED_BFS
++	{
++		.procname	= "rr_interval",
++		.data		= &rr_interval,
++		.maxlen		= sizeof (int),
++		.mode		= 0644,
++		.proc_handler	= &proc_dointvec_minmax,
++		.extra1		= &one,
++		.extra2		= &one_thousand,
++	},
++	{
++		.procname	= "interactive",
++		.data		= &sched_interactive,
++		.maxlen		= sizeof(int),
++		.mode		= 0644,
++		.proc_handler	= &proc_dointvec_minmax,
++		.extra1		= &zero,
++		.extra2		= &one,
++	},
++	{
++		.procname	= "iso_cpu",
++		.data		= &sched_iso_cpu,
++		.maxlen		= sizeof (int),
++		.mode		= 0644,
++		.proc_handler	= &proc_dointvec_minmax,
++		.extra1		= &zero,
++		.extra2		= &one_hundred,
++	},
++#endif
+ #if defined(CONFIG_S390) && defined(CONFIG_SMP)
+ 	{
+ 		.procname	= "spin_retry",
+Index: linux-4.4-ck1/lib/Kconfig.debug
+===================================================================
+--- linux-4.4-ck1.orig/lib/Kconfig.debug	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/lib/Kconfig.debug	2016-03-25 16:03:45.291350449 +1100
+@@ -1239,7 +1239,7 @@ config TORTURE_TEST
+ 
+ config RCU_TORTURE_TEST
+ 	tristate "torture tests for RCU"
+-	depends on DEBUG_KERNEL
++	depends on DEBUG_KERNEL && !SCHED_BFS
+ 	select TORTURE_TEST
+ 	select SRCU
+ 	select TASKS_RCU
+Index: linux-4.4-ck1/include/linux/jiffies.h
+===================================================================
+--- linux-4.4-ck1.orig/include/linux/jiffies.h	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/include/linux/jiffies.h	2016-03-25 16:03:45.291350449 +1100
+@@ -164,7 +164,7 @@ static inline u64 get_jiffies_64(void)
+  * Have the 32 bit jiffies value wrap 5 minutes after boot
+  * so jiffies wrap bugs show up earlier.
+  */
+-#define INITIAL_JIFFIES ((unsigned long)(unsigned int) (-300*HZ))
++#define INITIAL_JIFFIES ((unsigned long)(unsigned int) (-10*HZ))
+ 
+ /*
+  * Change timeval to jiffies, trying to avoid the
+Index: linux-4.4-ck1/drivers/cpufreq/cpufreq.c
+===================================================================
+--- linux-4.4-ck1.orig/drivers/cpufreq/cpufreq.c	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/drivers/cpufreq/cpufreq.c	2016-03-25 16:03:45.291350449 +1100
+@@ -25,6 +25,7 @@
+ #include <linux/kernel_stat.h>
+ #include <linux/module.h>
+ #include <linux/mutex.h>
++#include <linux/sched.h>
+ #include <linux/slab.h>
+ #include <linux/suspend.h>
+ #include <linux/syscore_ops.h>
+@@ -1900,6 +1901,12 @@ int __cpufreq_driver_target(struct cpufr
+ 	}
+ 
+ out:
++	if (likely(retval != -EINVAL)) {
++		if (target_freq == policy->max)
++			cpu_nonscaling(policy->cpu);
++		else
++			cpu_scaling(policy->cpu);
++	}
+ 	return retval;
+ }
+ EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
+Index: linux-4.4-ck1/drivers/cpufreq/cpufreq_ondemand.c
+===================================================================
+--- linux-4.4-ck1.orig/drivers/cpufreq/cpufreq_ondemand.c	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/drivers/cpufreq/cpufreq_ondemand.c	2016-03-25 16:03:45.291350449 +1100
+@@ -19,7 +19,7 @@
+ #include "cpufreq_governor.h"
+ 
+ /* On-demand governor macros */
+-#define DEF_FREQUENCY_UP_THRESHOLD		(80)
++#define DEF_FREQUENCY_UP_THRESHOLD		(63)
+ #define DEF_SAMPLING_DOWN_FACTOR		(1)
+ #define MAX_SAMPLING_DOWN_FACTOR		(100000)
+ #define MICRO_FREQUENCY_UP_THRESHOLD		(95)
+@@ -148,7 +148,7 @@ static void dbs_freq_increase(struct cpu
+ }
+ 
+ /*
+- * Every sampling_rate, we check, if current idle time is less than 20%
++ * Every sampling_rate, we check, if current idle time is less than 37%
+  * (default), then we try to increase frequency. Else, we adjust the frequency
+  * proportional to load.
+  */
+Index: linux-4.4-ck1/kernel/sched/bfs.c
+===================================================================
+--- /dev/null	1970-01-01 00:00:00.000000000 +0000
++++ linux-4.4-ck1/kernel/sched/bfs.c	2016-03-25 16:05:20.525834708 +1100
+@@ -0,0 +1,7585 @@
++/*
++ *  kernel/sched/bfs.c, was kernel/sched.c
++ *
++ *  Kernel scheduler and related syscalls
++ *
++ *  Copyright (C) 1991-2002  Linus Torvalds
++ *
++ *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and
++ *		make semaphores SMP safe
++ *  1998-11-19	Implemented schedule_timeout() and related stuff
++ *		by Andrea Arcangeli
++ *  2002-01-04	New ultra-scalable O(1) scheduler by Ingo Molnar:
++ *		hybrid priority-list and round-robin design with
++ *		an array-switch method of distributing timeslices
++ *		and per-CPU runqueues.  Cleanups and useful suggestions
++ *		by Davide Libenzi, preemptible kernel bits by Robert Love.
++ *  2003-09-03	Interactivity tuning by Con Kolivas.
++ *  2004-04-02	Scheduler domains code by Nick Piggin
++ *  2007-04-15  Work begun on replacing all interactivity tuning with a
++ *              fair scheduling design by Con Kolivas.
++ *  2007-05-05  Load balancing (smp-nice) and other improvements
++ *              by Peter Williams
++ *  2007-05-06  Interactivity improvements to CFS by Mike Galbraith
++ *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri
++ *  2007-11-29  RT balancing improvements by Steven Rostedt, Gregory Haskins,
++ *              Thomas Gleixner, Mike Kravetz
++ *  now		Brainfuck deadline scheduling policy by Con Kolivas deletes
++ *              a whole lot of those previous things.
++ */
++
++#include <linux/mm.h>
++#include <linux/module.h>
++#include <linux/nmi.h>
++#include <linux/init.h>
++#include <asm/uaccess.h>
++#include <linux/highmem.h>
++#include <asm/mmu_context.h>
++#include <linux/interrupt.h>
++#include <linux/capability.h>
++#include <linux/completion.h>
++#include <linux/kernel_stat.h>
++#include <linux/debug_locks.h>
++#include <linux/perf_event.h>
++#include <linux/security.h>
++#include <linux/notifier.h>
++#include <linux/profile.h>
++#include <linux/freezer.h>
++#include <linux/vmalloc.h>
++#include <linux/blkdev.h>
++#include <linux/delay.h>
++#include <linux/smp.h>
++#include <linux/threads.h>
++#include <linux/timer.h>
++#include <linux/rcupdate.h>
++#include <linux/cpu.h>
++#include <linux/cpuset.h>
++#include <linux/cpumask.h>
++#include <linux/percpu.h>
++#include <linux/proc_fs.h>
++#include <linux/seq_file.h>
++#include <linux/syscalls.h>
++#include <linux/sched/sysctl.h>
++#include <linux/times.h>
++#include <linux/tsacct_kern.h>
++#include <linux/kprobes.h>
++#include <linux/delayacct.h>
++#include <linux/log2.h>
++#include <linux/bootmem.h>
++#include <linux/ftrace.h>
++#include <linux/slab.h>
++#include <linux/init_task.h>
++#include <linux/binfmts.h>
++#include <linux/context_tracking.h>
++#include <linux/sched/prio.h>
++#include <linux/tick.h>
++
++#include <asm/irq_regs.h>
++#include <asm/switch_to.h>
++#include <asm/tlb.h>
++#include <asm/unistd.h>
++#include <asm/mutex.h>
++#ifdef CONFIG_PARAVIRT
++#include <asm/paravirt.h>
++#endif
++
++#include "cpupri.h"
++#include "../workqueue_internal.h"
++#include "../smpboot.h"
++
++#define CREATE_TRACE_POINTS
++#include <trace/events/sched.h>
++
++#include "bfs_sched.h"
++
++#define rt_prio(prio)		unlikely((prio) < MAX_RT_PRIO)
++#define rt_task(p)		rt_prio((p)->prio)
++#define rt_queue(rq)		rt_prio((rq)->rq_prio)
++#define batch_task(p)		(unlikely((p)->policy == SCHED_BATCH))
++#define is_rt_policy(policy)	((policy) == SCHED_FIFO || \
++					(policy) == SCHED_RR)
++#define has_rt_policy(p)	unlikely(is_rt_policy((p)->policy))
++
++#define is_idle_policy(policy)	((policy) == SCHED_IDLEPRIO)
++#define idleprio_task(p)	unlikely(is_idle_policy((p)->policy))
++#define task_running_idle(p)	unlikely((p)->prio == IDLE_PRIO)
++#define idle_queue(rq)		(unlikely(is_idle_policy((rq)->rq_policy)))
++
++#define is_iso_policy(policy)	((policy) == SCHED_ISO)
++#define iso_task(p)		unlikely(is_iso_policy((p)->policy))
++#define iso_queue(rq)		unlikely(is_iso_policy((rq)->rq_policy))
++#define task_running_iso(p)	unlikely((p)->prio == ISO_PRIO)
++#define rq_running_iso(rq)	((rq)->rq_prio == ISO_PRIO)
++
++#define rq_idle(rq)		((rq)->rq_prio == PRIO_LIMIT)
++
++#define ISO_PERIOD		((5 * HZ * grq.noc) + 1)
++
++#define SCHED_PRIO(p)		((p) + MAX_RT_PRIO)
++#define STOP_PRIO		(MAX_RT_PRIO - 1)
++
++/*
++ * Some helpers for converting to/from various scales. Use shifts to get
++ * approximate multiples of ten for less overhead.
++ */
++#define JIFFIES_TO_NS(TIME)	((TIME) * (1000000000 / HZ))
++#define JIFFY_NS		(1000000000 / HZ)
++#define HALF_JIFFY_NS		(1000000000 / HZ / 2)
++#define HALF_JIFFY_US		(1000000 / HZ / 2)
++#define MS_TO_NS(TIME)		((TIME) << 20)
++#define MS_TO_US(TIME)		((TIME) << 10)
++#define NS_TO_MS(TIME)		((TIME) >> 20)
++#define NS_TO_US(TIME)		((TIME) >> 10)
++
++#define RESCHED_US	(100) /* Reschedule if less than this many μs left */
++
++void print_scheduler_version(void)
++{
++	printk(KERN_INFO "BFS CPU scheduler v0.467 by Con Kolivas.\n");
++}
++
++/*
++ * This is the time all tasks within the same priority round robin.
++ * Value is in ms and set to a minimum of 6ms. Scales with number of cpus.
++ * Tunable via /proc interface.
++ */
++int rr_interval __read_mostly = 6;
++
++/* Tunable to choose whether to prioritise latency or throughput, simple
++ * binary yes or no */
++
++int sched_interactive __read_mostly = 1;
++
++/*
++ * sched_iso_cpu - sysctl which determines the cpu percentage SCHED_ISO tasks
++ * are allowed to run five seconds as real time tasks. This is the total over
++ * all online cpus.
++ */
++int sched_iso_cpu __read_mostly = 70;
++
++/*
++ * The relative length of deadline for each priority(nice) level.
++ */
++static int prio_ratios[NICE_WIDTH] __read_mostly;
++
++/*
++ * The quota handed out to tasks of all priority levels when refilling their
++ * time_slice.
++ */
++static inline int timeslice(void)
++{
++	return MS_TO_US(rr_interval);
++}
++
++/*
++ * The global runqueue data that all CPUs work off. Data is protected either
++ * by the global grq lock, or the discrete lock that precedes the data in this
++ * struct.
++ */
++struct global_rq {
++	raw_spinlock_t lock;
++	unsigned long nr_running;
++	unsigned long nr_uninterruptible;
++	unsigned long long nr_switches;
++	struct list_head queue[PRIO_LIMIT];
++	DECLARE_BITMAP(prio_bitmap, PRIO_LIMIT + 1);
++	unsigned long qnr; /* queued not running */
++#ifdef CONFIG_SMP
++	cpumask_t cpu_idle_map;
++	bool idle_cpus;
++#endif
++	int noc; /* num_online_cpus stored and updated when it changes */
++	u64 niffies; /* Nanosecond jiffies */
++	unsigned long last_jiffy; /* Last jiffy we updated niffies */
++
++	raw_spinlock_t iso_lock;
++	int iso_ticks;
++	bool iso_refractory;
++};
++
++#ifdef CONFIG_SMP
++/*
++ * We add the notion of a root-domain which will be used to define per-domain
++ * variables. Each exclusive cpuset essentially defines an island domain by
++ * fully partitioning the member cpus from any other cpuset. Whenever a new
++ * exclusive cpuset is created, we also create and attach a new root-domain
++ * object.
++ *
++ */
++struct root_domain {
++	atomic_t refcount;
++	atomic_t rto_count;
++	struct rcu_head rcu;
++	cpumask_var_t span;
++	cpumask_var_t online;
++
++	/*
++	 * The "RT overload" flag: it gets set if a CPU has more than
++	 * one runnable RT task.
++	 */
++	cpumask_var_t rto_mask;
++	struct cpupri cpupri;
++};
++
++/*
++ * By default the system creates a single root-domain with all cpus as
++ * members (mimicking the global state we have today).
++ */
++static struct root_domain def_root_domain;
++
++#endif /* CONFIG_SMP */
++
++/* There can be only one */
++static struct global_rq grq;
++
++static DEFINE_MUTEX(sched_hotcpu_mutex);
++
++/* cpus with isolated domains */
++cpumask_var_t cpu_isolated_map;
++
++DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
++#ifdef CONFIG_SMP
++struct rq *cpu_rq(int cpu)
++{
++	return &per_cpu(runqueues, (cpu));
++}
++#define task_rq(p)		cpu_rq(task_cpu(p))
++#define cpu_curr(cpu)		(cpu_rq(cpu)->curr)
++/*
++ * sched_domains_mutex serialises calls to init_sched_domains,
++ * detach_destroy_domains and partition_sched_domains.
++ */
++DEFINE_MUTEX(sched_domains_mutex);
++
++/*
++ * By default the system creates a single root-domain with all cpus as
++ * members (mimicking the global state we have today).
++ */
++static struct root_domain def_root_domain;
++
++int __weak arch_sd_sibling_asym_packing(void)
++{
++       return 0*SD_ASYM_PACKING;
++}
++#else
++struct rq *uprq;
++#endif /* CONFIG_SMP */
++
++static inline void update_rq_clock(struct rq *rq);
++
++/*
++ * Sanity check should sched_clock return bogus values. We make sure it does
++ * not appear to go backwards, and use jiffies to determine the maximum and
++ * minimum it could possibly have increased, and round down to the nearest
++ * jiffy when it falls outside this.
++ */
++static inline void niffy_diff(s64 *niff_diff, int jiff_diff)
++{
++	unsigned long min_diff, max_diff;
++
++	if (jiff_diff > 1)
++		min_diff = JIFFIES_TO_NS(jiff_diff - 1);
++	else
++		min_diff = 1;
++	/*  Round up to the nearest tick for maximum */
++	max_diff = JIFFIES_TO_NS(jiff_diff + 1);
++
++	if (unlikely(*niff_diff < min_diff || *niff_diff > max_diff))
++		*niff_diff = min_diff;
++}
++
++#ifdef CONFIG_SMP
++static inline int cpu_of(struct rq *rq)
++{
++	return rq->cpu;
++}
++
++/*
++ * Niffies are a globally increasing nanosecond counter. Whenever a runqueue
++ * clock is updated with the grq.lock held, it is an opportunity to update the
++ * niffies value. Any CPU can update it by adding how much its clock has
++ * increased since it last updated niffies, minus any added niffies by other
++ * CPUs.
++ */
++static inline void update_clocks(struct rq *rq)
++{
++	s64 ndiff;
++	long jdiff;
++
++	update_rq_clock(rq);
++	ndiff = rq->clock - rq->old_clock;
++	/* old_clock is only updated when we are updating niffies */
++	rq->old_clock = rq->clock;
++	ndiff -= grq.niffies - rq->last_niffy;
++	jdiff = jiffies - grq.last_jiffy;
++	niffy_diff(&ndiff, jdiff);
++	grq.last_jiffy += jdiff;
++	grq.niffies += ndiff;
++	rq->last_niffy = grq.niffies;
++}
++#else /* CONFIG_SMP */
++static inline int cpu_of(struct rq *rq)
++{
++	return 0;
++}
++
++static inline void update_clocks(struct rq *rq)
++{
++	s64 ndiff;
++	long jdiff;
++
++	update_rq_clock(rq);
++	ndiff = rq->clock - rq->old_clock;
++	rq->old_clock = rq->clock;
++	jdiff = jiffies - grq.last_jiffy;
++	niffy_diff(&ndiff, jdiff);
++	grq.last_jiffy += jdiff;
++	grq.niffies += ndiff;
++}
++#endif
++
++#include "stats.h"
++
++#ifndef prepare_arch_switch
++# define prepare_arch_switch(next)	do { } while (0)
++#endif
++#ifndef finish_arch_switch
++# define finish_arch_switch(prev)	do { } while (0)
++#endif
++#ifndef finish_arch_post_lock_switch
++# define finish_arch_post_lock_switch()	do { } while (0)
++#endif
++
++/*
++ * All common locking functions performed on grq.lock. rq->clock is local to
++ * the CPU accessing it so it can be modified just with interrupts disabled
++ * when we're not updating niffies.
++ * Looking up task_rq must be done under grq.lock to be safe.
++ */
++static void update_rq_clock_task(struct rq *rq, s64 delta);
++
++static inline void update_rq_clock(struct rq *rq)
++{
++	s64 delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
++
++	if (unlikely(delta < 0))
++		return;
++	rq->clock += delta;
++	update_rq_clock_task(rq, delta);
++}
++
++static inline bool task_running(struct task_struct *p)
++{
++	return p->on_cpu;
++}
++
++static inline void grq_lock(void)
++	__acquires(grq.lock)
++{
++	raw_spin_lock(&grq.lock);
++}
++
++static inline void grq_unlock(void)
++	__releases(grq.lock)
++{
++	raw_spin_unlock(&grq.lock);
++}
++
++static inline void grq_lock_irq(void)
++	__acquires(grq.lock)
++{
++	raw_spin_lock_irq(&grq.lock);
++}
++
++static inline void time_lock_grq(struct rq *rq)
++	__acquires(grq.lock)
++{
++	grq_lock();
++	update_clocks(rq);
++}
++
++static inline void grq_unlock_irq(void)
++	__releases(grq.lock)
++{
++	raw_spin_unlock_irq(&grq.lock);
++}
++
++static inline void grq_lock_irqsave(unsigned long *flags)
++	__acquires(grq.lock)
++{
++	raw_spin_lock_irqsave(&grq.lock, *flags);
++}
++
++static inline void grq_unlock_irqrestore(unsigned long *flags)
++	__releases(grq.lock)
++{
++	raw_spin_unlock_irqrestore(&grq.lock, *flags);
++}
++
++static inline struct rq
++*task_grq_lock(struct task_struct *p, unsigned long *flags)
++	__acquires(grq.lock)
++{
++	grq_lock_irqsave(flags);
++	return task_rq(p);
++}
++
++static inline struct rq
++*time_task_grq_lock(struct task_struct *p, unsigned long *flags)
++	__acquires(grq.lock)
++{
++	struct rq *rq = task_grq_lock(p, flags);
++	update_clocks(rq);
++	return rq;
++}
++
++static inline struct rq *task_grq_lock_irq(struct task_struct *p)
++	__acquires(grq.lock)
++{
++	grq_lock_irq();
++	return task_rq(p);
++}
++
++static inline void time_task_grq_lock_irq(struct task_struct *p)
++	__acquires(grq.lock)
++{
++	struct rq *rq = task_grq_lock_irq(p);
++	update_clocks(rq);
++}
++
++static inline void task_grq_unlock_irq(void)
++	__releases(grq.lock)
++{
++	grq_unlock_irq();
++}
++
++static inline void task_grq_unlock(unsigned long *flags)
++	__releases(grq.lock)
++{
++	grq_unlock_irqrestore(flags);
++}
++
++/**
++ * grunqueue_is_locked
++ *
++ * Returns true if the global runqueue is locked.
++ * This interface allows printk to be called with the runqueue lock
++ * held and know whether or not it is OK to wake up the klogd.
++ */
++bool grunqueue_is_locked(void)
++{
++	return raw_spin_is_locked(&grq.lock);
++}
++
++void grq_unlock_wait(void)
++	__releases(grq.lock)
++{
++	smp_mb(); /* spin-unlock-wait is not a full memory barrier */
++	raw_spin_unlock_wait(&grq.lock);
++}
++
++static inline void time_grq_lock(struct rq *rq, unsigned long *flags)
++	__acquires(grq.lock)
++{
++	local_irq_save(*flags);
++	time_lock_grq(rq);
++}
++
++static inline struct rq *__task_grq_lock(struct task_struct *p)
++	__acquires(grq.lock)
++{
++	grq_lock();
++	return task_rq(p);
++}
++
++static inline void __task_grq_unlock(void)
++	__releases(grq.lock)
++{
++	grq_unlock();
++}
++
++static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
++{
++}
++
++static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
++{
++#ifdef CONFIG_DEBUG_SPINLOCK
++	/* this is a valid case when another task releases the spinlock */
++	grq.lock.owner = current;
++#endif
++	/*
++	 * If we are tracking spinlock dependencies then we have to
++	 * fix up the runqueue lock - which gets 'carried over' from
++	 * prev into current:
++	 */
++	spin_acquire(&grq.lock.dep_map, 0, 0, _THIS_IP_);
++
++	grq_unlock_irq();
++}
++
++static inline bool deadline_before(u64 deadline, u64 time)
++{
++	return (deadline < time);
++}
++
++static inline bool deadline_after(u64 deadline, u64 time)
++{
++	return (deadline > time);
++}
++
++/*
++ * A task that is queued but not running will be on the grq run list.
++ * A task that is not running or queued will not be on the grq run list.
++ * A task that is currently running will have ->on_cpu set but not on the
++ * grq run list.
++ */
++static inline bool task_queued(struct task_struct *p)
++{
++	return (!list_empty(&p->run_list));
++}
++
++/*
++ * Removing from the global runqueue. Enter with grq locked.
++ */
++static void dequeue_task(struct task_struct *p)
++{
++	list_del_init(&p->run_list);
++	if (list_empty(grq.queue + p->prio))
++		__clear_bit(p->prio, grq.prio_bitmap);
++	sched_info_dequeued(task_rq(p), p);
++}
++
++/*
++ * To determine if it's safe for a task of SCHED_IDLEPRIO to actually run as
++ * an idle task, we ensure none of the following conditions are met.
++ */
++static bool idleprio_suitable(struct task_struct *p)
++{
++	return (!freezing(p) && !signal_pending(p) &&
++		!(task_contributes_to_load(p)) && !(p->flags & (PF_EXITING)));
++}
++
++/*
++ * To determine if a task of SCHED_ISO can run in pseudo-realtime, we check
++ * that the iso_refractory flag is not set.
++ */
++static bool isoprio_suitable(void)
++{
++	return !grq.iso_refractory;
++}
++
++/*
++ * Adding to the global runqueue. Enter with grq locked.
++ */
++static void enqueue_task(struct task_struct *p, struct rq *rq)
++{
++	if (!rt_task(p)) {
++		/* Check it hasn't gotten rt from PI */
++		if ((idleprio_task(p) && idleprio_suitable(p)) ||
++		   (iso_task(p) && isoprio_suitable()))
++			p->prio = p->normal_prio;
++		else
++			p->prio = NORMAL_PRIO;
++	}
++	__set_bit(p->prio, grq.prio_bitmap);
++	list_add_tail(&p->run_list, grq.queue + p->prio);
++	sched_info_queued(rq, p);
++}
++
++static inline void requeue_task(struct task_struct *p)
++{
++	sched_info_queued(task_rq(p), p);
++}
++
++/*
++ * Returns the relative length of deadline all compared to the shortest
++ * deadline which is that of nice -20.
++ */
++static inline int task_prio_ratio(struct task_struct *p)
++{
++	return prio_ratios[TASK_USER_PRIO(p)];
++}
++
++/*
++ * task_timeslice - all tasks of all priorities get the exact same timeslice
++ * length. CPU distribution is handled by giving different deadlines to
++ * tasks of different priorities. Use 128 as the base value for fast shifts.
++ */
++static inline int task_timeslice(struct task_struct *p)
++{
++	return (rr_interval * task_prio_ratio(p) / 128);
++}
++
++static void resched_task(struct task_struct *p);
++
++static inline void resched_curr(struct rq *rq)
++{
++	resched_task(rq->curr);
++}
++
++/*
++ * qnr is the "queued but not running" count which is the total number of
++ * tasks on the global runqueue list waiting for cpu time but not actually
++ * currently running on a cpu.
++ */
++static inline void inc_qnr(void)
++{
++	grq.qnr++;
++}
++
++static inline void dec_qnr(void)
++{
++	grq.qnr--;
++}
++
++static inline int queued_notrunning(void)
++{
++	return grq.qnr;
++}
++
++#ifdef CONFIG_SMP
++/*
++ * The cpu_idle_map stores a bitmap of all the CPUs currently idle to
++ * allow easy lookup of whether any suitable idle CPUs are available.
++ * It's cheaper to maintain a binary yes/no if there are any idle CPUs on the
++ * idle_cpus variable than to do a full bitmask check when we are busy.
++ */
++static inline void set_cpuidle_map(int cpu)
++{
++	if (likely(cpu_online(cpu))) {
++		cpumask_set_cpu(cpu, &grq.cpu_idle_map);
++		grq.idle_cpus = true;
++	}
++}
++
++static inline void clear_cpuidle_map(int cpu)
++{
++	cpumask_clear_cpu(cpu, &grq.cpu_idle_map);
++	if (cpumask_empty(&grq.cpu_idle_map))
++		grq.idle_cpus = false;
++}
++
++static bool suitable_idle_cpus(struct task_struct *p)
++{
++	if (!grq.idle_cpus)
++		return false;
++	return (cpumask_intersects(&p->cpus_allowed, &grq.cpu_idle_map));
++}
++
++#define CPUIDLE_DIFF_THREAD	(1)
++#define CPUIDLE_DIFF_CORE	(2)
++#define CPUIDLE_CACHE_BUSY	(4)
++#define CPUIDLE_DIFF_CPU	(8)
++#define CPUIDLE_THREAD_BUSY	(16)
++#define CPUIDLE_THROTTLED	(32)
++#define CPUIDLE_DIFF_NODE	(64)
++
++static inline bool scaling_rq(struct rq *rq);
++
++/*
++ * The best idle CPU is chosen according to the CPUIDLE ranking above where the
++ * lowest value would give the most suitable CPU to schedule p onto next. The
++ * order works out to be the following:
++ *
++ * Same core, idle or busy cache, idle or busy threads
++ * Other core, same cache, idle or busy cache, idle threads.
++ * Same node, other CPU, idle cache, idle threads.
++ * Same node, other CPU, busy cache, idle threads.
++ * Other core, same cache, busy threads.
++ * Same node, other CPU, busy threads.
++ * Other node, other CPU, idle cache, idle threads.
++ * Other node, other CPU, busy cache, idle threads.
++ * Other node, other CPU, busy threads.
++ */
++static int best_mask_cpu(int best_cpu, struct rq *rq, cpumask_t *tmpmask)
++{
++	int best_ranking = CPUIDLE_DIFF_NODE | CPUIDLE_THROTTLED |
++		CPUIDLE_THREAD_BUSY | CPUIDLE_DIFF_CPU | CPUIDLE_CACHE_BUSY |
++		CPUIDLE_DIFF_CORE | CPUIDLE_DIFF_THREAD;
++	int cpu_tmp;
++
++	if (cpumask_test_cpu(best_cpu, tmpmask))
++		goto out;
++
++	for_each_cpu(cpu_tmp, tmpmask) {
++		int ranking, locality;
++		struct rq *tmp_rq;
++
++		ranking = 0;
++		tmp_rq = cpu_rq(cpu_tmp);
++
++		locality = rq->cpu_locality[cpu_tmp];
++#ifdef CONFIG_NUMA
++		if (locality > 3)
++			ranking |= CPUIDLE_DIFF_NODE;
++		else
++#endif
++		if (locality > 2)
++			ranking |= CPUIDLE_DIFF_CPU;
++#ifdef CONFIG_SCHED_MC
++		else if (locality == 2)
++			ranking |= CPUIDLE_DIFF_CORE;
++		if (!(tmp_rq->cache_idle(cpu_tmp)))
++			ranking |= CPUIDLE_CACHE_BUSY;
++#endif
++#ifdef CONFIG_SCHED_SMT
++		if (locality == 1)
++			ranking |= CPUIDLE_DIFF_THREAD;
++		if (!(tmp_rq->siblings_idle(cpu_tmp)))
++			ranking |= CPUIDLE_THREAD_BUSY;
++#endif
++		if (scaling_rq(tmp_rq))
++			ranking |= CPUIDLE_THROTTLED;
++
++		if (ranking < best_ranking) {
++			best_cpu = cpu_tmp;
++			best_ranking = ranking;
++		}
++	}
++out:
++	return best_cpu;
++}
++
++static void resched_best_mask(int best_cpu, struct rq *rq, cpumask_t *tmpmask)
++{
++	best_cpu = best_mask_cpu(best_cpu, rq, tmpmask);
++	resched_curr(cpu_rq(best_cpu));
++}
++
++bool cpus_share_cache(int this_cpu, int that_cpu)
++{
++	struct rq *this_rq = cpu_rq(this_cpu);
++
++	return (this_rq->cpu_locality[that_cpu] < 3);
++}
++
++#ifdef CONFIG_SCHED_SMT
++#ifdef CONFIG_SMT_NICE
++static const cpumask_t *thread_cpumask(int cpu);
++
++/* Find the best real time priority running on any SMT siblings of cpu and if
++ * none are running, the static priority of the best deadline task running.
++ * The lookups to the other runqueues is done lockless as the occasional wrong
++ * value would be harmless. */
++static int best_smt_bias(int cpu)
++{
++	int other_cpu, best_bias = 0;
++
++	for_each_cpu(other_cpu, thread_cpumask(cpu)) {
++		struct rq *rq;
++
++		if (other_cpu == cpu)
++			continue;
++		rq = cpu_rq(other_cpu);
++		if (rq_idle(rq))
++			continue;
++		if (!rq->online)
++			continue;
++		if (!rq->rq_mm)
++			continue;
++		if (likely(rq->rq_smt_bias > best_bias))
++			best_bias = rq->rq_smt_bias;
++	}
++	return best_bias;
++}
++
++static int task_prio_bias(struct task_struct *p)
++{
++	if (rt_task(p))
++		return 1 << 30;
++	else if (task_running_iso(p))
++		return 1 << 29;
++	else if (task_running_idle(p))
++		return 0;
++	return MAX_PRIO - p->static_prio;
++}
++
++/* We've already decided p can run on CPU, now test if it shouldn't for SMT
++ * nice reasons. */
++static bool smt_should_schedule(struct task_struct *p, int cpu)
++{
++	int best_bias, task_bias;
++
++	/* Kernel threads always run */
++	if (unlikely(!p->mm))
++		return true;
++	if (rt_task(p))
++		return true;
++	if (!idleprio_suitable(p))
++		return true;
++	best_bias = best_smt_bias(cpu);
++	/* The smt siblings are all idle or running IDLEPRIO */
++	if (best_bias < 1)
++		return true;
++	task_bias = task_prio_bias(p);
++	if (task_bias < 1)
++		return false;
++	if (task_bias >= best_bias)
++		return true;
++	/* Dither 25% cpu of normal tasks regardless of nice difference */
++	if (best_bias % 4 == 1)
++		return true;
++	/* Sorry, you lose */
++	return false;
++}
++#endif
++#endif
++
++static bool resched_best_idle(struct task_struct *p)
++{
++	cpumask_t tmpmask;
++	int best_cpu;
++
++	cpumask_and(&tmpmask, &p->cpus_allowed, &grq.cpu_idle_map);
++	best_cpu = best_mask_cpu(task_cpu(p), task_rq(p), &tmpmask);
++#ifdef CONFIG_SMT_NICE
++	if (!smt_should_schedule(p, best_cpu))
++		return false;
++#endif
++	resched_curr(cpu_rq(best_cpu));
++	return true;
++}
++
++static inline void resched_suitable_idle(struct task_struct *p)
++{
++	if (suitable_idle_cpus(p))
++		resched_best_idle(p);
++}
++/*
++ * Flags to tell us whether this CPU is running a CPU frequency governor that
++ * has slowed its speed or not. No locking required as the very rare wrongly
++ * read value would be harmless.
++ */
++void cpu_scaling(int cpu)
++{
++	cpu_rq(cpu)->scaling = true;
++}
++
++void cpu_nonscaling(int cpu)
++{
++	cpu_rq(cpu)->scaling = false;
++}
++
++static inline bool scaling_rq(struct rq *rq)
++{
++	return rq->scaling;
++}
++
++static inline int locality_diff(int cpu, struct rq *rq)
++{
++	return rq->cpu_locality[cpu];
++}
++#else /* CONFIG_SMP */
++static inline void set_cpuidle_map(int cpu)
++{
++}
++
++static inline void clear_cpuidle_map(int cpu)
++{
++}
++
++static inline bool suitable_idle_cpus(struct task_struct *p)
++{
++	return uprq->curr == uprq->idle;
++}
++
++static inline void resched_suitable_idle(struct task_struct *p)
++{
++}
++
++void cpu_scaling(int __unused)
++{
++}
++
++void cpu_nonscaling(int __unused)
++{
++}
++
++/*
++ * Although CPUs can scale in UP, there is nowhere else for tasks to go so this
++ * always returns 0.
++ */
++static inline bool scaling_rq(struct rq *rq)
++{
++	return false;
++}
++
++static inline int locality_diff(int cpu, struct rq *rq)
++{
++	return 0;
++}
++#endif /* CONFIG_SMP */
++EXPORT_SYMBOL_GPL(cpu_scaling);
++EXPORT_SYMBOL_GPL(cpu_nonscaling);
++
++static inline int normal_prio(struct task_struct *p)
++{
++	if (has_rt_policy(p))
++		return MAX_RT_PRIO - 1 - p->rt_priority;
++	if (idleprio_task(p))
++		return IDLE_PRIO;
++	if (iso_task(p))
++		return ISO_PRIO;
++	return NORMAL_PRIO;
++}
++
++/*
++ * Calculate the current priority, i.e. the priority
++ * taken into account by the scheduler. This value might
++ * be boosted by RT tasks as it will be RT if the task got
++ * RT-boosted. If not then it returns p->normal_prio.
++ */
++static int effective_prio(struct task_struct *p)
++{
++	p->normal_prio = normal_prio(p);
++	/*
++	 * If we are RT tasks or we were boosted to RT priority,
++	 * keep the priority unchanged. Otherwise, update priority
++	 * to the normal priority:
++	 */
++	if (!rt_prio(p->prio))
++		return p->normal_prio;
++	return p->prio;
++}
++
++/*
++ * activate_task - move a task to the runqueue. Enter with grq locked.
++ */
++static void activate_task(struct task_struct *p, struct rq *rq)
++{
++	update_clocks(rq);
++
++	/*
++	 * Sleep time is in units of nanosecs, so shift by 20 to get a
++	 * milliseconds-range estimation of the amount of time that the task
++	 * spent sleeping:
++	 */
++	if (unlikely(prof_on == SLEEP_PROFILING)) {
++		if (p->state == TASK_UNINTERRUPTIBLE)
++			profile_hits(SLEEP_PROFILING, (void *)get_wchan(p),
++				     (rq->clock_task - p->last_ran) >> 20);
++	}
++
++	p->prio = effective_prio(p);
++	if (task_contributes_to_load(p))
++		grq.nr_uninterruptible--;
++	enqueue_task(p, rq);
++	rq->soft_affined++;
++	p->on_rq = 1;
++	grq.nr_running++;
++	inc_qnr();
++}
++
++static inline void clear_sticky(struct task_struct *p);
++
++/*
++ * deactivate_task - If it's running, it's not on the grq and we can just
++ * decrement the nr_running. Enter with grq locked.
++ */
++static inline void deactivate_task(struct task_struct *p, struct rq *rq)
++{
++	if (task_contributes_to_load(p))
++		grq.nr_uninterruptible++;
++	rq->soft_affined--;
++	p->on_rq = 0;
++	grq.nr_running--;
++	clear_sticky(p);
++}
++
++#ifdef CONFIG_SMP
++void set_task_cpu(struct task_struct *p, unsigned int cpu)
++{
++#ifdef CONFIG_LOCKDEP
++	/*
++	 * The caller should hold grq lock.
++	 */
++	WARN_ON_ONCE(debug_locks && !lockdep_is_held(&grq.lock));
++#endif
++	if (task_cpu(p) == cpu)
++		return;
++	trace_sched_migrate_task(p, cpu);
++	perf_event_task_migrate(p);
++
++	/*
++	 * After ->cpu is set up to a new value, task_grq_lock(p, ...) can be
++	 * successfully executed on another CPU. We must ensure that updates of
++	 * per-task data have been completed by this moment.
++	 */
++	smp_wmb();
++	if (p->on_rq) {
++		task_rq(p)->soft_affined--;
++		cpu_rq(cpu)->soft_affined++;
++	}
++	task_thread_info(p)->cpu = cpu;
++}
++
++static inline void clear_sticky(struct task_struct *p)
++{
++	p->sticky = false;
++}
++
++static inline bool task_sticky(struct task_struct *p)
++{
++	return p->sticky;
++}
++
++/* Reschedule the best idle CPU that is not this one. */
++static void
++resched_closest_idle(struct rq *rq, int cpu, struct task_struct *p)
++{
++	cpumask_t tmpmask;
++
++	cpumask_and(&tmpmask, &p->cpus_allowed, &grq.cpu_idle_map);
++	cpumask_clear_cpu(cpu, &tmpmask);
++	if (cpumask_empty(&tmpmask))
++		return;
++	resched_best_mask(cpu, rq, &tmpmask);
++}
++
++/*
++ * We set the sticky flag on a task that is descheduled involuntarily meaning
++ * it is awaiting further CPU time. If the last sticky task is still sticky
++ * but unlucky enough to not be the next task scheduled, we unstick it and try
++ * to find it an idle CPU. Realtime tasks do not stick to minimise their
++ * latency at all times.
++ */
++static inline void
++swap_sticky(struct rq *rq, int cpu, struct task_struct *p)
++{
++	if (rq->sticky_task) {
++		if (rq->sticky_task == p) {
++			p->sticky = true;
++			return;
++		}
++		if (task_sticky(rq->sticky_task)) {
++			clear_sticky(rq->sticky_task);
++			resched_closest_idle(rq, cpu, rq->sticky_task);
++		}
++	}
++	if (!rt_task(p)) {
++		p->sticky = true;
++		rq->sticky_task = p;
++	} else {
++		resched_closest_idle(rq, cpu, p);
++		rq->sticky_task = NULL;
++	}
++}
++
++static inline void unstick_task(struct rq *rq, struct task_struct *p)
++{
++	rq->sticky_task = NULL;
++	clear_sticky(p);
++}
++#else
++static inline void clear_sticky(struct task_struct *p)
++{
++}
++
++static inline bool task_sticky(struct task_struct *p)
++{
++	return false;
++}
++
++static inline void
++swap_sticky(struct rq *rq, int cpu, struct task_struct *p)
++{
++}
++
++static inline void unstick_task(struct rq *rq, struct task_struct *p)
++{
++}
++#endif
++
++/*
++ * Move a task off the global queue and take it to a cpu for it will
++ * become the running task.
++ */
++static inline void take_task(int cpu, struct task_struct *p)
++{
++	set_task_cpu(p, cpu);
++	dequeue_task(p);
++	clear_sticky(p);
++	dec_qnr();
++}
++
++/*
++ * Returns a descheduling task to the grq runqueue unless it is being
++ * deactivated.
++ */
++static inline void return_task(struct task_struct *p, struct rq *rq, bool deactivate)
++{
++	if (deactivate)
++		deactivate_task(p, rq);
++	else {
++		inc_qnr();
++		enqueue_task(p, rq);
++	}
++}
++
++/* Enter with grq lock held. We know p is on the local cpu */
++static inline void __set_tsk_resched(struct task_struct *p)
++{
++	set_tsk_need_resched(p);
++	set_preempt_need_resched();
++}
++
++/*
++ * resched_task - mark a task 'to be rescheduled now'.
++ *
++ * On UP this means the setting of the need_resched flag, on SMP it
++ * might also involve a cross-CPU call to trigger the scheduler on
++ * the target CPU.
++ */
++void resched_task(struct task_struct *p)
++{
++	int cpu;
++
++	lockdep_assert_held(&grq.lock);
++
++	if (test_tsk_need_resched(p))
++		return;
++
++	set_tsk_need_resched(p);
++
++	cpu = task_cpu(p);
++	if (cpu == smp_processor_id()) {
++		set_preempt_need_resched();
++		return;
++	}
++
++	smp_send_reschedule(cpu);
++}
++
++/**
++ * task_curr - is this task currently executing on a CPU?
++ * @p: the task in question.
++ *
++ * Return: 1 if the task is currently executing. 0 otherwise.
++ */
++inline int task_curr(const struct task_struct *p)
++{
++	return cpu_curr(task_cpu(p)) == p;
++}
++
++#ifdef CONFIG_SMP
++struct migration_req {
++	struct task_struct *task;
++	int dest_cpu;
++};
++
++/*
++ * wait_task_inactive - wait for a thread to unschedule.
++ *
++ * If @match_state is nonzero, it's the @p->state value just checked and
++ * not expected to change.  If it changes, i.e. @p might have woken up,
++ * then return zero.  When we succeed in waiting for @p to be off its CPU,
++ * we return a positive number (its total switch count).  If a second call
++ * a short while later returns the same number, the caller can be sure that
++ * @p has remained unscheduled the whole time.
++ *
++ * The caller must ensure that the task *will* unschedule sometime soon,
++ * else this function might spin for a *long* time. This function can't
++ * be called with interrupts off, or it may introduce deadlock with
++ * smp_call_function() if an IPI is sent by the same process we are
++ * waiting to become inactive.
++ */
++unsigned long wait_task_inactive(struct task_struct *p, long match_state)
++{
++	unsigned long flags;
++	bool running, on_rq;
++	unsigned long ncsw;
++	struct rq *rq;
++
++	for (;;) {
++		rq = task_rq(p);
++
++		/*
++		 * If the task is actively running on another CPU
++		 * still, just relax and busy-wait without holding
++		 * any locks.
++		 *
++		 * NOTE! Since we don't hold any locks, it's not
++		 * even sure that "rq" stays as the right runqueue!
++		 * But we don't care, since this will return false
++		 * if the runqueue has changed and p is actually now
++		 * running somewhere else!
++		 */
++		while (task_running(p) && p == rq->curr) {
++			if (match_state && unlikely(p->state != match_state))
++				return 0;
++			cpu_relax();
++		}
++
++		/*
++		 * Ok, time to look more closely! We need the grq
++		 * lock now, to be *sure*. If we're wrong, we'll
++		 * just go back and repeat.
++		 */
++		rq = task_grq_lock(p, &flags);
++		trace_sched_wait_task(p);
++		running = task_running(p);
++		on_rq = p->on_rq;
++		ncsw = 0;
++		if (!match_state || p->state == match_state)
++			ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
++		task_grq_unlock(&flags);
++
++		/*
++		 * If it changed from the expected state, bail out now.
++		 */
++		if (unlikely(!ncsw))
++			break;
++
++		/*
++		 * Was it really running after all now that we
++		 * checked with the proper locks actually held?
++		 *
++		 * Oops. Go back and try again..
++		 */
++		if (unlikely(running)) {
++			cpu_relax();
++			continue;
++		}
++
++		/*
++		 * It's not enough that it's not actively running,
++		 * it must be off the runqueue _entirely_, and not
++		 * preempted!
++		 *
++		 * So if it was still runnable (but just not actively
++		 * running right now), it's preempted, and we should
++		 * yield - it could be a while.
++		 */
++		if (unlikely(on_rq)) {
++			ktime_t to = ktime_set(0, NSEC_PER_SEC / HZ);
++
++			set_current_state(TASK_UNINTERRUPTIBLE);
++			schedule_hrtimeout(&to, HRTIMER_MODE_REL);
++			continue;
++		}
++
++		/*
++		 * Ahh, all good. It wasn't running, and it wasn't
++		 * runnable, which means that it will never become
++		 * running in the future either. We're all done!
++		 */
++		break;
++	}
++
++	return ncsw;
++}
++
++/***
++ * kick_process - kick a running thread to enter/exit the kernel
++ * @p: the to-be-kicked thread
++ *
++ * Cause a process which is running on another CPU to enter
++ * kernel-mode, without any delay. (to get signals handled.)
++ *
++ * NOTE: this function doesn't have to take the runqueue lock,
++ * because all it wants to ensure is that the remote task enters
++ * the kernel. If the IPI races and the task has been migrated
++ * to another CPU then no harm is done and the purpose has been
++ * achieved as well.
++ */
++void kick_process(struct task_struct *p)
++{
++	int cpu;
++
++	preempt_disable();
++	cpu = task_cpu(p);
++	if ((cpu != smp_processor_id()) && task_curr(p))
++		smp_send_reschedule(cpu);
++	preempt_enable();
++}
++EXPORT_SYMBOL_GPL(kick_process);
++#endif
++
++/*
++ * RT tasks preempt purely on priority. SCHED_NORMAL tasks preempt on the
++ * basis of earlier deadlines. SCHED_IDLEPRIO don't preempt anything else or
++ * between themselves, they cooperatively multitask. An idle rq scores as
++ * prio PRIO_LIMIT so it is always preempted.
++ */
++static inline bool
++can_preempt(struct task_struct *p, int prio, u64 deadline)
++{
++	/* Better static priority RT task or better policy preemption */
++	if (p->prio < prio)
++		return true;
++	if (p->prio > prio)
++		return false;
++	/* SCHED_NORMAL, BATCH and ISO will preempt based on deadline */
++	if (!deadline_before(p->deadline, deadline))
++		return false;
++	return true;
++}
++
++#ifdef CONFIG_SMP
++#define cpu_online_map		(*(cpumask_t *)cpu_online_mask)
++#ifdef CONFIG_HOTPLUG_CPU
++/*
++ * Check to see if there is a task that is affined only to offline CPUs but
++ * still wants runtime. This happens to kernel threads during suspend/halt and
++ * disabling of CPUs.
++ */
++static inline bool online_cpus(struct task_struct *p)
++{
++	return (likely(cpumask_intersects(&cpu_online_map, &p->cpus_allowed)));
++}
++#else /* CONFIG_HOTPLUG_CPU */
++/* All available CPUs are always online without hotplug. */
++static inline bool online_cpus(struct task_struct *p)
++{
++	return true;
++}
++#endif
++
++/*
++ * Check to see if p can run on cpu, and if not, whether there are any online
++ * CPUs it can run on instead.
++ */
++static inline bool needs_other_cpu(struct task_struct *p, int cpu)
++{
++	if (unlikely(!cpumask_test_cpu(cpu, &p->cpus_allowed)))
++		return true;
++	return false;
++}
++
++/*
++ * When all else is equal, still prefer this_rq.
++ */
++static void try_preempt(struct task_struct *p, struct rq *this_rq)
++{
++	struct rq *highest_prio_rq = NULL;
++	int cpu, highest_prio;
++	u64 latest_deadline;
++	cpumask_t tmp;
++
++	/*
++	 * We clear the sticky flag here because for a task to have called
++	 * try_preempt with the sticky flag enabled means some complicated
++	 * re-scheduling has occurred and we should ignore the sticky flag.
++	 */
++	clear_sticky(p);
++
++	if (suitable_idle_cpus(p) && resched_best_idle(p))
++		return;
++
++	/* IDLEPRIO tasks never preempt anything but idle */
++	if (p->policy == SCHED_IDLEPRIO)
++		return;
++
++	if (likely(online_cpus(p)))
++		cpumask_and(&tmp, &cpu_online_map, &p->cpus_allowed);
++	else
++		return;
++
++	highest_prio = latest_deadline = 0;
++
++	for_each_cpu(cpu, &tmp) {
++		struct rq *rq;
++		int rq_prio;
++
++		rq = cpu_rq(cpu);
++		rq_prio = rq->rq_prio;
++		if (rq_prio < highest_prio)
++			continue;
++
++		if (rq_prio > highest_prio ||
++		    deadline_after(rq->rq_deadline, latest_deadline)) {
++			latest_deadline = rq->rq_deadline;
++			highest_prio = rq_prio;
++			highest_prio_rq = rq;
++		}
++	}
++
++	if (likely(highest_prio_rq)) {
++#ifdef CONFIG_SMT_NICE
++		cpu = cpu_of(highest_prio_rq);
++		if (!smt_should_schedule(p, cpu))
++			return;
++#endif
++		if (can_preempt(p, highest_prio, highest_prio_rq->rq_deadline))
++			resched_curr(highest_prio_rq);
++	}
++}
++static int __set_cpus_allowed_ptr(struct task_struct *p,
++				  const struct cpumask *new_mask, bool check);
++#else /* CONFIG_SMP */
++static inline bool needs_other_cpu(struct task_struct *p, int cpu)
++{
++	return false;
++}
++
++static void try_preempt(struct task_struct *p, struct rq *this_rq)
++{
++	if (p->policy == SCHED_IDLEPRIO)
++		return;
++	if (can_preempt(p, uprq->rq_prio, uprq->rq_deadline))
++		resched_curr(uprq);
++}
++
++static inline int __set_cpus_allowed_ptr(struct task_struct *p,
++					 const struct cpumask *new_mask, bool check)
++{
++	return set_cpus_allowed_ptr(p, new_mask);
++}
++#endif /* CONFIG_SMP */
++
++static void
++ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
++{
++#ifdef CONFIG_SCHEDSTATS
++	struct rq *rq = this_rq();
++
++#ifdef CONFIG_SMP
++	int this_cpu = smp_processor_id();
++
++	if (cpu == this_cpu)
++		schedstat_inc(rq, ttwu_local);
++	else {
++		struct sched_domain *sd;
++
++		rcu_read_lock();
++		for_each_domain(this_cpu, sd) {
++			if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
++				schedstat_inc(sd, ttwu_wake_remote);
++				break;
++			}
++		}
++		rcu_read_unlock();
++	}
++
++#endif /* CONFIG_SMP */
++
++	schedstat_inc(rq, ttwu_count);
++#endif /* CONFIG_SCHEDSTATS */
++}
++
++void wake_up_if_idle(int cpu)
++{
++	struct rq *rq = cpu_rq(cpu);
++	unsigned long flags;
++
++	rcu_read_lock();
++
++	if (!is_idle_task(rcu_dereference(rq->curr)))
++		goto out;
++
++	grq_lock_irqsave(&flags);
++	if (likely(is_idle_task(rq->curr)))
++		smp_send_reschedule(cpu);
++	/* Else cpu is not in idle, do nothing here */
++	grq_unlock_irqrestore(&flags);
++
++out:
++	rcu_read_unlock();
++}
++
++#ifdef CONFIG_SMP
++void scheduler_ipi(void)
++{
++	/*
++	 * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
++	 * TIF_NEED_RESCHED remotely (for the first time) will also send
++	 * this IPI.
++	 */
++	preempt_fold_need_resched();
++}
++#endif
++
++static inline void ttwu_activate(struct task_struct *p, struct rq *rq,
++				 bool is_sync)
++{
++	activate_task(p, rq);
++
++	/*
++	 * Sync wakeups (i.e. those types of wakeups where the waker
++	 * has indicated that it will leave the CPU in short order)
++	 * don't trigger a preemption if there are no idle cpus,
++	 * instead waiting for current to deschedule.
++	 */
++	if (!is_sync || suitable_idle_cpus(p))
++		try_preempt(p, rq);
++}
++
++static inline void ttwu_post_activation(struct task_struct *p, struct rq *rq,
++					bool success)
++{
++	trace_sched_wakeup(p);
++	p->state = TASK_RUNNING;
++
++	/*
++	 * if a worker is waking up, notify workqueue. Note that on BFS, we
++	 * don't really know what cpu it will be, so we fake it for
++	 * wq_worker_waking_up :/
++	 */
++	if ((p->flags & PF_WQ_WORKER) && success)
++		wq_worker_waking_up(p, cpu_of(rq));
++}
++
++/*
++ * wake flags
++ */
++#define WF_SYNC		0x01		/* waker goes to sleep after wakeup */
++#define WF_FORK		0x02		/* child wakeup after fork */
++#define WF_MIGRATED	0x4		/* internal use, task got migrated */
++
++/***
++ * try_to_wake_up - wake up a thread
++ * @p: the thread to be awakened
++ * @state: the mask of task states that can be woken
++ * @wake_flags: wake modifier flags (WF_*)
++ *
++ * Put it on the run-queue if it's not already there. The "current"
++ * thread is always on the run-queue (except when the actual
++ * re-schedule is in progress), and as such you're allowed to do
++ * the simpler "current->state = TASK_RUNNING" to mark yourself
++ * runnable without the overhead of this.
++ *
++ * Return: %true if @p was woken up, %false if it was already running.
++ * or @state didn't match @p's state.
++ */
++static bool try_to_wake_up(struct task_struct *p, unsigned int state,
++			  int wake_flags)
++{
++	bool success = false;
++	unsigned long flags;
++	struct rq *rq;
++	int cpu;
++
++	get_cpu();
++
++	/*
++	 * If we are going to wake up a thread waiting for CONDITION we
++	 * need to ensure that CONDITION=1 done by the caller can not be
++	 * reordered with p->state check below. This pairs with mb() in
++	 * set_current_state() the waiting thread does.
++	 */
++	smp_mb__before_spinlock();
++
++	/*
++	 * No need to do time_lock_grq as we only need to update the rq clock
++	 * if we activate the task
++	 */
++	rq = task_grq_lock(p, &flags);
++	cpu = task_cpu(p);
++
++	/* state is a volatile long, どうして、分からない */
++	if (!((unsigned int)p->state & state))
++		goto out_unlock;
++
++	trace_sched_waking(p);
++
++	if (task_queued(p) || task_running(p))
++		goto out_running;
++
++	ttwu_activate(p, rq, wake_flags & WF_SYNC);
++	success = true;
++
++out_running:
++	ttwu_post_activation(p, rq, success);
++out_unlock:
++	task_grq_unlock(&flags);
++
++	ttwu_stat(p, cpu, wake_flags);
++
++	put_cpu();
++
++	return success;
++}
++
++/**
++ * try_to_wake_up_local - try to wake up a local task with grq lock held
++ * @p: the thread to be awakened
++ *
++ * Put @p on the run-queue if it's not already there. The caller must
++ * ensure that grq is locked and, @p is not the current task.
++ * grq stays locked over invocation.
++ */
++static void try_to_wake_up_local(struct task_struct *p)
++{
++	struct rq *rq = task_rq(p);
++	bool success = false;
++
++	lockdep_assert_held(&grq.lock);
++
++	if (!(p->state & TASK_NORMAL))
++		return;
++
++	trace_sched_waking(p);
++
++	if (!task_queued(p)) {
++		if (likely(!task_running(p))) {
++			schedstat_inc(rq, ttwu_count);
++			schedstat_inc(rq, ttwu_local);
++		}
++		ttwu_activate(p, rq, false);
++		ttwu_stat(p, smp_processor_id(), 0);
++		success = true;
++	}
++	ttwu_post_activation(p, rq, success);
++}
++
++/**
++ * wake_up_process - Wake up a specific process
++ * @p: The process to be woken up.
++ *
++ * Attempt to wake up the nominated process and move it to the set of runnable
++ * processes.
++ *
++ * Return: 1 if the process was woken up, 0 if it was already running.
++ *
++ * It may be assumed that this function implies a write memory barrier before
++ * changing the task state if and only if any tasks are woken up.
++ */
++int wake_up_process(struct task_struct *p)
++{
++	return try_to_wake_up(p, TASK_NORMAL, 0);
++}
++EXPORT_SYMBOL(wake_up_process);
++
++int wake_up_state(struct task_struct *p, unsigned int state)
++{
++	return try_to_wake_up(p, state, 0);
++}
++
++static void time_slice_expired(struct task_struct *p);
++
++/*
++ * Perform scheduler related setup for a newly forked process p.
++ * p is forked by current.
++ */
++int sched_fork(unsigned long __maybe_unused clone_flags, struct task_struct *p)
++{
++#ifdef CONFIG_PREEMPT_NOTIFIERS
++	INIT_HLIST_HEAD(&p->preempt_notifiers);
++#endif
++	/*
++	 * The process state is set to the same value of the process executing
++	 * do_fork() code. That is running. This guarantees that nobody will
++	 * actually run it, and a signal or other external event cannot wake
++	 * it up and insert it on the runqueue either.
++	 */
++
++	/* Should be reset in fork.c but done here for ease of bfs patching */
++	p->on_rq =
++	p->utime =
++	p->stime =
++	p->utimescaled =
++	p->stimescaled =
++	p->sched_time =
++	p->stime_pc =
++	p->utime_pc = 0;
++
++	/*
++	 * Revert to default priority/policy on fork if requested.
++	 */
++	if (unlikely(p->sched_reset_on_fork)) {
++		if (p->policy == SCHED_FIFO || p->policy == SCHED_RR) {
++			p->policy = SCHED_NORMAL;
++			p->normal_prio = normal_prio(p);
++		}
++
++		if (PRIO_TO_NICE(p->static_prio) < 0) {
++			p->static_prio = NICE_TO_PRIO(0);
++			p->normal_prio = p->static_prio;
++		}
++
++		/*
++		 * We don't need the reset flag anymore after the fork. It has
++		 * fulfilled its duty:
++		 */
++		p->sched_reset_on_fork = 0;
++	}
++
++	INIT_LIST_HEAD(&p->run_list);
++#ifdef CONFIG_SCHED_INFO
++	if (unlikely(sched_info_on()))
++		memset(&p->sched_info, 0, sizeof(p->sched_info));
++#endif
++	p->on_cpu = false;
++	clear_sticky(p);
++	init_task_preempt_count(p);
++	return 0;
++}
++
++/*
++ * wake_up_new_task - wake up a newly created task for the first time.
++ *
++ * This function will do some initial scheduler statistics housekeeping
++ * that must be done for every newly created context, then puts the task
++ * on the runqueue and wakes it.
++ */
++void wake_up_new_task(struct task_struct *p)
++{
++	struct task_struct *parent;
++	unsigned long flags;
++	struct rq *rq;
++
++	parent = p->parent;
++	rq = task_grq_lock(p, &flags);
++
++	/*
++	 * Reinit new task deadline as its creator deadline could have changed
++	 * since call to dup_task_struct().
++	 */
++	p->deadline = rq->rq_deadline;
++
++	/*
++	 * If the task is a new process, current and parent are the same. If
++	 * the task is a new thread in the thread group, it will have much more
++	 * in common with current than with the parent.
++	 */
++	set_task_cpu(p, task_cpu(rq->curr));
++
++	/*
++	 * Make sure we do not leak PI boosting priority to the child.
++	 */
++	p->prio = rq->curr->normal_prio;
++
++	activate_task(p, rq);
++	trace_sched_wakeup_new(p);
++	if (unlikely(p->policy == SCHED_FIFO))
++		goto after_ts_init;
++
++	/*
++	 * Share the timeslice between parent and child, thus the
++	 * total amount of pending timeslices in the system doesn't change,
++	 * resulting in more scheduling fairness. If it's negative, it won't
++	 * matter since that's the same as being 0. current's time_slice is
++	 * actually in rq_time_slice when it's running, as is its last_ran
++	 * value. rq->rq_deadline is only modified within schedule() so it
++	 * is always equal to current->deadline.
++	 */
++	p->last_ran = rq->rq_last_ran;
++	if (likely(rq->rq_time_slice >= RESCHED_US * 2)) {
++		rq->rq_time_slice /= 2;
++		p->time_slice = rq->rq_time_slice;
++after_ts_init:
++		if (rq->curr == parent && !suitable_idle_cpus(p)) {
++			/*
++			 * The VM isn't cloned, so we're in a good position to
++			 * do child-runs-first in anticipation of an exec. This
++			 * usually avoids a lot of COW overhead.
++			 */
++			__set_tsk_resched(parent);
++		} else
++			try_preempt(p, rq);
++	} else {
++		if (rq->curr == parent) {
++			/*
++		 	* Forking task has run out of timeslice. Reschedule it and
++		 	* start its child with a new time slice and deadline. The
++		 	* child will end up running first because its deadline will
++		 	* be slightly earlier.
++		 	*/
++			rq->rq_time_slice = 0;
++			__set_tsk_resched(parent);
++		}
++		time_slice_expired(p);
++	}
++	task_grq_unlock(&flags);
++}
++
++#ifdef CONFIG_PREEMPT_NOTIFIERS
++
++static struct static_key preempt_notifier_key = STATIC_KEY_INIT_FALSE;
++
++void preempt_notifier_inc(void)
++{
++	static_key_slow_inc(&preempt_notifier_key);
++}
++EXPORT_SYMBOL_GPL(preempt_notifier_inc);
++
++void preempt_notifier_dec(void)
++{
++	static_key_slow_dec(&preempt_notifier_key);
++}
++EXPORT_SYMBOL_GPL(preempt_notifier_dec);
++
++/**
++ * preempt_notifier_register - tell me when current is being preempted & rescheduled
++ * @notifier: notifier struct to register
++ */
++void preempt_notifier_register(struct preempt_notifier *notifier)
++{
++	if (!static_key_false(&preempt_notifier_key))
++		WARN(1, "registering preempt_notifier while notifiers disabled\n");
++
++	hlist_add_head(&notifier->link, &current->preempt_notifiers);
++}
++EXPORT_SYMBOL_GPL(preempt_notifier_register);
++
++/**
++ * preempt_notifier_unregister - no longer interested in preemption notifications
++ * @notifier: notifier struct to unregister
++ *
++ * This is *not* safe to call from within a preemption notifier.
++ */
++void preempt_notifier_unregister(struct preempt_notifier *notifier)
++{
++	hlist_del(&notifier->link);
++}
++EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
++
++static void __fire_sched_in_preempt_notifiers(struct task_struct *curr)
++{
++	struct preempt_notifier *notifier;
++
++	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
++		notifier->ops->sched_in(notifier, raw_smp_processor_id());
++}
++
++static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
++{
++	if (static_key_false(&preempt_notifier_key))
++		__fire_sched_in_preempt_notifiers(curr);
++}
++
++static void
++__fire_sched_out_preempt_notifiers(struct task_struct *curr,
++				 struct task_struct *next)
++{
++	struct preempt_notifier *notifier;
++
++	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
++		notifier->ops->sched_out(notifier, next);
++}
++
++static __always_inline void
++fire_sched_out_preempt_notifiers(struct task_struct *curr,
++				 struct task_struct *next)
++{
++	if (static_key_false(&preempt_notifier_key))
++		__fire_sched_out_preempt_notifiers(curr, next);
++}
++
++#else /* !CONFIG_PREEMPT_NOTIFIERS */
++
++static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
++{
++}
++
++static inline void
++fire_sched_out_preempt_notifiers(struct task_struct *curr,
++				 struct task_struct *next)
++{
++}
++
++#endif /* CONFIG_PREEMPT_NOTIFIERS */
++
++/**
++ * prepare_task_switch - prepare to switch tasks
++ * @rq: the runqueue preparing to switch
++ * @next: the task we are going to switch to.
++ *
++ * This is called with the rq lock held and interrupts off. It must
++ * be paired with a subsequent finish_task_switch after the context
++ * switch.
++ *
++ * prepare_task_switch sets up locking and calls architecture specific
++ * hooks.
++ */
++static inline void
++prepare_task_switch(struct rq *rq, struct task_struct *prev,
++		    struct task_struct *next)
++{
++	sched_info_switch(rq, prev, next);
++	perf_event_task_sched_out(prev, next);
++	fire_sched_out_preempt_notifiers(prev, next);
++	prepare_lock_switch(rq, next);
++	prepare_arch_switch(next);
++}
++
++/**
++ * finish_task_switch - clean up after a task-switch
++ * @rq: runqueue associated with task-switch
++ * @prev: the thread we just switched away from.
++ *
++ * finish_task_switch must be called after the context switch, paired
++ * with a prepare_task_switch call before the context switch.
++ * finish_task_switch will reconcile locking set up by prepare_task_switch,
++ * and do any other architecture-specific cleanup actions.
++ *
++ * Note that we may have delayed dropping an mm in context_switch(). If
++ * so, we finish that here outside of the runqueue lock.  (Doing it
++ * with the lock held can cause deadlocks; see schedule() for
++ * details.)
++ *
++ * The context switch have flipped the stack from under us and restored the
++ * local variables which were saved when this task called schedule() in the
++ * past. prev == current is still correct but we need to recalculate this_rq
++ * because prev may have moved to another CPU.
++ */
++static struct rq *finish_task_switch(struct task_struct *prev)
++	__releases(grq.lock)
++{
++	struct rq *rq = this_rq();
++	struct mm_struct *mm = rq->prev_mm;
++	long prev_state;
++
++	/*
++	 * The previous task will have left us with a preempt_count of 2
++	 * because it left us after:
++	 *
++	 *	schedule()
++	 *	  preempt_disable();			// 1
++	 *	  __schedule()
++	 *	    raw_spin_lock_irq(&rq->lock)	// 2
++	 *
++	 * Also, see FORK_PREEMPT_COUNT.
++	 */
++	if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET,
++		      "corrupted preempt_count: %s/%d/0x%x\n",
++		      current->comm, current->pid, preempt_count()))
++		preempt_count_set(FORK_PREEMPT_COUNT);
++
++	rq->prev_mm = NULL;
++
++	/*
++	 * A task struct has one reference for the use as "current".
++	 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
++	 * schedule one last time. The schedule call will never return, and
++	 * the scheduled task must drop that reference.
++	 *
++	 * We must observe prev->state before clearing prev->on_cpu (in
++	 * finish_lock_switch), otherwise a concurrent wakeup can get prev
++	 * running on another CPU and we could rave with its RUNNING -> DEAD
++	 * transition, resulting in a double drop.
++	 */
++	prev_state = prev->state;
++	vtime_task_switch(prev);
++	perf_event_task_sched_in(prev, current);
++	finish_lock_switch(rq, prev);
++	finish_arch_post_lock_switch();
++
++	fire_sched_in_preempt_notifiers(current);
++	if (mm)
++		mmdrop(mm);
++	if (unlikely(prev_state == TASK_DEAD)) {
++		/*
++		 * Remove function-return probe instances associated with this
++		 * task and put them back on the free list.
++		 */
++		kprobe_flush_task(prev);
++		put_task_struct(prev);
++	}
++	return rq;
++}
++
++/**
++ * schedule_tail - first thing a freshly forked thread must call.
++ * @prev: the thread we just switched away from.
++ */
++asmlinkage __visible void schedule_tail(struct task_struct *prev)
++	__releases(grq.lock)
++{
++	struct rq *rq;
++
++	/*
++	 * New tasks start with FORK_PREEMPT_COUNT, see there and
++	 * finish_task_switch() for details.
++	 *
++	 * finish_task_switch() will drop rq->lock() and lower preempt_count
++	 * and the preempt_enable() will end up enabling preemption (on
++	 * PREEMPT_COUNT kernels).
++	 */
++
++	rq = finish_task_switch(prev);
++	preempt_enable();
++
++	if (current->set_child_tid)
++		put_user(task_pid_vnr(current), current->set_child_tid);
++}
++
++/*
++ * context_switch - switch to the new MM and the new thread's register state.
++ */
++static inline struct rq *
++context_switch(struct rq *rq, struct task_struct *prev,
++	       struct task_struct *next)
++{
++	struct mm_struct *mm, *oldmm;
++
++	prepare_task_switch(rq, prev, next);
++
++	mm = next->mm;
++	oldmm = prev->active_mm;
++	/*
++	 * For paravirt, this is coupled with an exit in switch_to to
++	 * combine the page table reload and the switch backend into
++	 * one hypercall.
++	 */
++	arch_start_context_switch(prev);
++
++	if (!mm) {
++		next->active_mm = oldmm;
++		atomic_inc(&oldmm->mm_count);
++		enter_lazy_tlb(oldmm, next);
++	} else
++		switch_mm(oldmm, mm, next);
++
++	if (!prev->mm) {
++		prev->active_mm = NULL;
++		rq->prev_mm = oldmm;
++	}
++	/*
++	 * Since the runqueue lock will be released by the next
++	 * task (which is an invalid locking op but in the case
++	 * of the scheduler it's an obvious special-case), so we
++	 * do an early lockdep release here:
++	 */
++	spin_release(&grq.lock.dep_map, 1, _THIS_IP_);
++
++	/* Here we just switch the register state and the stack. */
++	switch_to(prev, next, prev);
++	barrier();
++
++	return finish_task_switch(prev);
++}
++
++/*
++ * nr_running, nr_uninterruptible and nr_context_switches:
++ *
++ * externally visible scheduler statistics: current number of runnable
++ * threads, total number of context switches performed since bootup. All are
++ * measured without grabbing the grq lock but the occasional inaccurate result
++ * doesn't matter so long as it's positive.
++ */
++unsigned long nr_running(void)
++{
++	long nr = grq.nr_running;
++
++	if (unlikely(nr < 0))
++		nr = 0;
++	return (unsigned long)nr;
++}
++
++static unsigned long nr_uninterruptible(void)
++{
++	long nu = grq.nr_uninterruptible;
++
++	if (unlikely(nu < 0))
++		nu = 0;
++	return nu;
++}
++
++/*
++ * Check if only the current task is running on the cpu.
++ *
++ * Caution: this function does not check that the caller has disabled
++ * preemption, thus the result might have a time-of-check-to-time-of-use
++ * race.  The caller is responsible to use it correctly, for example:
++ *
++ * - from a non-preemptable section (of course)
++ *
++ * - from a thread that is bound to a single CPU
++ *
++ * - in a loop with very short iterations (e.g. a polling loop)
++ */
++bool single_task_running(void)
++{
++	if (cpu_rq(smp_processor_id())->soft_affined == 1)
++		return true;
++	else
++		return false;
++}
++EXPORT_SYMBOL(single_task_running);
++
++unsigned long long nr_context_switches(void)
++{
++	long long ns = grq.nr_switches;
++
++	/* This is of course impossible */
++	if (unlikely(ns < 0))
++		ns = 1;
++	return (unsigned long long)ns;
++}
++
++unsigned long nr_iowait(void)
++{
++	unsigned long i, sum = 0;
++
++	for_each_possible_cpu(i)
++		sum += atomic_read(&cpu_rq(i)->nr_iowait);
++
++	return sum;
++}
++
++unsigned long nr_iowait_cpu(int cpu)
++{
++	struct rq *this = cpu_rq(cpu);
++	return atomic_read(&this->nr_iowait);
++}
++
++unsigned long nr_active(void)
++{
++	return nr_running() + nr_uninterruptible();
++}
++
++/* Beyond a task running on this CPU, load is equal everywhere on BFS, so we
++ * base it on the number of running or queued tasks with their ->rq pointer
++ * set to this cpu as being the CPU they're more likely to run on. */
++void get_iowait_load(unsigned long *nr_waiters, unsigned long *load)
++{
++	struct rq *rq = this_rq();
++
++	*nr_waiters = atomic_read(&rq->nr_iowait);
++	*load = rq->soft_affined;
++}
++
++/* Variables and functions for calc_load */
++static unsigned long calc_load_update;
++unsigned long avenrun[3];
++EXPORT_SYMBOL(avenrun);
++
++/**
++ * get_avenrun - get the load average array
++ * @loads:	pointer to dest load array
++ * @offset:	offset to add
++ * @shift:	shift count to shift the result left
++ *
++ * These values are estimates at best, so no need for locking.
++ */
++void get_avenrun(unsigned long *loads, unsigned long offset, int shift)
++{
++	loads[0] = (avenrun[0] + offset) << shift;
++	loads[1] = (avenrun[1] + offset) << shift;
++	loads[2] = (avenrun[2] + offset) << shift;
++}
++
++static unsigned long
++calc_load(unsigned long load, unsigned long exp, unsigned long active)
++{
++	load *= exp;
++	load += active * (FIXED_1 - exp);
++	return load >> FSHIFT;
++}
++
++/*
++ * calc_load - update the avenrun load estimates every LOAD_FREQ seconds.
++ */
++void calc_global_load(unsigned long ticks)
++{
++	long active;
++
++	if (time_before(jiffies, calc_load_update))
++		return;
++	active = nr_active() * FIXED_1;
++
++	avenrun[0] = calc_load(avenrun[0], EXP_1, active);
++	avenrun[1] = calc_load(avenrun[1], EXP_5, active);
++	avenrun[2] = calc_load(avenrun[2], EXP_15, active);
++
++	calc_load_update = jiffies + LOAD_FREQ;
++}
++
++DEFINE_PER_CPU(struct kernel_stat, kstat);
++DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat);
++
++EXPORT_PER_CPU_SYMBOL(kstat);
++EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
++
++#ifdef CONFIG_IRQ_TIME_ACCOUNTING
++
++/*
++ * There are no locks covering percpu hardirq/softirq time.
++ * They are only modified in account_system_vtime, on corresponding CPU
++ * with interrupts disabled. So, writes are safe.
++ * They are read and saved off onto struct rq in update_rq_clock().
++ * This may result in other CPU reading this CPU's irq time and can
++ * race with irq/account_system_vtime on this CPU. We would either get old
++ * or new value with a side effect of accounting a slice of irq time to wrong
++ * task when irq is in progress while we read rq->clock. That is a worthy
++ * compromise in place of having locks on each irq in account_system_time.
++ */
++static DEFINE_PER_CPU(u64, cpu_hardirq_time);
++static DEFINE_PER_CPU(u64, cpu_softirq_time);
++
++static DEFINE_PER_CPU(u64, irq_start_time);
++static int sched_clock_irqtime;
++
++void enable_sched_clock_irqtime(void)
++{
++	sched_clock_irqtime = 1;
++}
++
++void disable_sched_clock_irqtime(void)
++{
++	sched_clock_irqtime = 0;
++}
++
++#ifndef CONFIG_64BIT
++static DEFINE_PER_CPU(seqcount_t, irq_time_seq);
++
++static inline void irq_time_write_begin(void)
++{
++	__this_cpu_inc(irq_time_seq.sequence);
++	smp_wmb();
++}
++
++static inline void irq_time_write_end(void)
++{
++	smp_wmb();
++	__this_cpu_inc(irq_time_seq.sequence);
++}
++
++static inline u64 irq_time_read(int cpu)
++{
++	u64 irq_time;
++	unsigned seq;
++
++	do {
++		seq = read_seqcount_begin(&per_cpu(irq_time_seq, cpu));
++		irq_time = per_cpu(cpu_softirq_time, cpu) +
++			   per_cpu(cpu_hardirq_time, cpu);
++	} while (read_seqcount_retry(&per_cpu(irq_time_seq, cpu), seq));
++
++	return irq_time;
++}
++#else /* CONFIG_64BIT */
++static inline void irq_time_write_begin(void)
++{
++}
++
++static inline void irq_time_write_end(void)
++{
++}
++
++static inline u64 irq_time_read(int cpu)
++{
++	return per_cpu(cpu_softirq_time, cpu) + per_cpu(cpu_hardirq_time, cpu);
++}
++#endif /* CONFIG_64BIT */
++
++/*
++ * Called before incrementing preempt_count on {soft,}irq_enter
++ * and before decrementing preempt_count on {soft,}irq_exit.
++ */
++void irqtime_account_irq(struct task_struct *curr)
++{
++	unsigned long flags;
++	s64 delta;
++	int cpu;
++
++	if (!sched_clock_irqtime)
++		return;
++
++	local_irq_save(flags);
++
++	cpu = smp_processor_id();
++	delta = sched_clock_cpu(cpu) - __this_cpu_read(irq_start_time);
++	__this_cpu_add(irq_start_time, delta);
++
++	irq_time_write_begin();
++	/*
++	 * We do not account for softirq time from ksoftirqd here.
++	 * We want to continue accounting softirq time to ksoftirqd thread
++	 * in that case, so as not to confuse scheduler with a special task
++	 * that do not consume any time, but still wants to run.
++	 */
++	if (hardirq_count())
++		__this_cpu_add(cpu_hardirq_time, delta);
++	else if (in_serving_softirq() && curr != this_cpu_ksoftirqd())
++		__this_cpu_add(cpu_softirq_time, delta);
++
++	irq_time_write_end();
++	local_irq_restore(flags);
++}
++EXPORT_SYMBOL_GPL(irqtime_account_irq);
++
++#endif /* CONFIG_IRQ_TIME_ACCOUNTING */
++
++#ifdef CONFIG_PARAVIRT
++static inline u64 steal_ticks(u64 steal)
++{
++	if (unlikely(steal > NSEC_PER_SEC))
++		return div_u64(steal, TICK_NSEC);
++
++	return __iter_div_u64_rem(steal, TICK_NSEC, &steal);
++}
++#endif
++
++static void update_rq_clock_task(struct rq *rq, s64 delta)
++{
++/*
++ * In theory, the compile should just see 0 here, and optimize out the call
++ * to sched_rt_avg_update. But I don't trust it...
++ */
++#ifdef CONFIG_IRQ_TIME_ACCOUNTING
++	s64 irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
++
++	/*
++	 * Since irq_time is only updated on {soft,}irq_exit, we might run into
++	 * this case when a previous update_rq_clock() happened inside a
++	 * {soft,}irq region.
++	 *
++	 * When this happens, we stop ->clock_task and only update the
++	 * prev_irq_time stamp to account for the part that fit, so that a next
++	 * update will consume the rest. This ensures ->clock_task is
++	 * monotonic.
++	 *
++	 * It does however cause some slight miss-attribution of {soft,}irq
++	 * time, a more accurate solution would be to update the irq_time using
++	 * the current rq->clock timestamp, except that would require using
++	 * atomic ops.
++	 */
++	if (irq_delta > delta)
++		irq_delta = delta;
++
++	rq->prev_irq_time += irq_delta;
++	delta -= irq_delta;
++#endif
++#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
++	if (static_key_false((&paravirt_steal_rq_enabled))) {
++		s64 steal = paravirt_steal_clock(cpu_of(rq));
++
++		steal -= rq->prev_steal_time_rq;
++
++		if (unlikely(steal > delta))
++			steal = delta;
++
++		rq->prev_steal_time_rq += steal;
++
++		delta -= steal;
++	}
++#endif
++
++	rq->clock_task += delta;
++}
++
++#ifndef nsecs_to_cputime
++# define nsecs_to_cputime(__nsecs)	nsecs_to_jiffies(__nsecs)
++#endif
++
++#ifdef CONFIG_IRQ_TIME_ACCOUNTING
++static void irqtime_account_hi_si(void)
++{
++	u64 *cpustat = kcpustat_this_cpu->cpustat;
++	u64 latest_ns;
++
++	latest_ns = nsecs_to_cputime64(this_cpu_read(cpu_hardirq_time));
++	if (latest_ns > cpustat[CPUTIME_IRQ])
++		cpustat[CPUTIME_IRQ] += (__force u64)cputime_one_jiffy;
++
++	latest_ns = nsecs_to_cputime64(this_cpu_read(cpu_softirq_time));
++	if (latest_ns > cpustat[CPUTIME_SOFTIRQ])
++		cpustat[CPUTIME_SOFTIRQ] += (__force u64)cputime_one_jiffy;
++}
++#else /* CONFIG_IRQ_TIME_ACCOUNTING */
++
++#define sched_clock_irqtime	(0)
++
++static inline void irqtime_account_hi_si(void)
++{
++}
++#endif /* CONFIG_IRQ_TIME_ACCOUNTING */
++
++static __always_inline bool steal_account_process_tick(void)
++{
++#ifdef CONFIG_PARAVIRT
++	if (static_key_false(&paravirt_steal_enabled)) {
++		u64 steal;
++		cputime_t steal_ct;
++
++		steal = paravirt_steal_clock(smp_processor_id());
++		steal -= this_rq()->prev_steal_time;
++
++		/*
++		 * cputime_t may be less precise than nsecs (eg: if it's
++		 * based on jiffies). Lets cast the result to cputime
++		 * granularity and account the rest on the next rounds.
++		 */
++		steal_ct = nsecs_to_cputime(steal);
++		this_rq()->prev_steal_time += cputime_to_nsecs(steal_ct);
++
++		account_steal_time(steal_ct);
++		return steal_ct;
++	}
++#endif
++	return false;
++}
++
++/*
++ * Accumulate raw cputime values of dead tasks (sig->[us]time) and live
++ * tasks (sum on group iteration) belonging to @tsk's group.
++ */
++void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times)
++{
++	struct signal_struct *sig = tsk->signal;
++	cputime_t utime, stime;
++	struct task_struct *t;
++	unsigned int seq, nextseq;
++	unsigned long flags;
++
++	rcu_read_lock();
++	/* Attempt a lockless read on the first round. */
++	nextseq = 0;
++	do {
++		seq = nextseq;
++		flags = read_seqbegin_or_lock_irqsave(&sig->stats_lock, &seq);
++		times->utime = sig->utime;
++		times->stime = sig->stime;
++		times->sum_exec_runtime = sig->sum_sched_runtime;
++
++		for_each_thread(tsk, t) {
++			task_cputime(t, &utime, &stime);
++			times->utime += utime;
++			times->stime += stime;
++			times->sum_exec_runtime += task_sched_runtime(t);
++		}
++		/* If lockless access failed, take the lock. */
++		nextseq = 1;
++	} while (need_seqretry(&sig->stats_lock, seq));
++	done_seqretry_irqrestore(&sig->stats_lock, seq, flags);
++	rcu_read_unlock();
++}
++
++/*
++ * On each tick, see what percentage of that tick was attributed to each
++ * component and add the percentage to the _pc values. Once a _pc value has
++ * accumulated one tick's worth, account for that. This means the total
++ * percentage of load components will always be 128 (pseudo 100) per tick.
++ */
++static void pc_idle_time(struct rq *rq, struct task_struct *idle, unsigned long pc)
++{
++	u64 *cpustat = kcpustat_this_cpu->cpustat;
++
++	if (atomic_read(&rq->nr_iowait) > 0) {
++		rq->iowait_pc += pc;
++		if (rq->iowait_pc >= 128) {
++			cpustat[CPUTIME_IOWAIT] += (__force u64)cputime_one_jiffy * rq->iowait_pc / 128;
++			rq->iowait_pc %= 128;
++		}
++	} else {
++		rq->idle_pc += pc;
++		if (rq->idle_pc >= 128) {
++			cpustat[CPUTIME_IDLE] += (__force u64)cputime_one_jiffy * rq->idle_pc / 128;
++			rq->idle_pc %= 128;
++		}
++	}
++	acct_update_integrals(idle);
++}
++
++static void
++pc_system_time(struct rq *rq, struct task_struct *p, int hardirq_offset,
++	       unsigned long pc, unsigned long ns)
++{
++	u64 *cpustat = kcpustat_this_cpu->cpustat;
++	cputime_t one_jiffy_scaled = cputime_to_scaled(cputime_one_jiffy);
++
++	p->stime_pc += pc;
++	if (p->stime_pc >= 128) {
++		int jiffs = p->stime_pc / 128;
++
++		p->stime_pc %= 128;
++		p->stime += (__force u64)cputime_one_jiffy * jiffs;
++		p->stimescaled += one_jiffy_scaled * jiffs;
++		account_group_system_time(p, cputime_one_jiffy * jiffs);
++	}
++	p->sched_time += ns;
++	account_group_exec_runtime(p, ns);
++
++	if (hardirq_count() - hardirq_offset) {
++		rq->irq_pc += pc;
++		if (rq->irq_pc >= 128) {
++			cpustat[CPUTIME_IRQ] += (__force u64)cputime_one_jiffy * rq->irq_pc / 128;
++			rq->irq_pc %= 128;
++		}
++	} else if (in_serving_softirq()) {
++		rq->softirq_pc += pc;
++		if (rq->softirq_pc >= 128) {
++			cpustat[CPUTIME_SOFTIRQ] += (__force u64)cputime_one_jiffy * rq->softirq_pc / 128;
++			rq->softirq_pc %= 128;
++		}
++	} else {
++		rq->system_pc += pc;
++		if (rq->system_pc >= 128) {
++			cpustat[CPUTIME_SYSTEM] += (__force u64)cputime_one_jiffy * rq->system_pc / 128;
++			rq->system_pc %= 128;
++		}
++	}
++	acct_update_integrals(p);
++}
++
++static void pc_user_time(struct rq *rq, struct task_struct *p,
++			 unsigned long pc, unsigned long ns)
++{
++	u64 *cpustat = kcpustat_this_cpu->cpustat;
++	cputime_t one_jiffy_scaled = cputime_to_scaled(cputime_one_jiffy);
++
++	p->utime_pc += pc;
++	if (p->utime_pc >= 128) {
++		int jiffs = p->utime_pc / 128;
++
++		p->utime_pc %= 128;
++		p->utime += (__force u64)cputime_one_jiffy * jiffs;
++		p->utimescaled += one_jiffy_scaled * jiffs;
++		account_group_user_time(p, cputime_one_jiffy * jiffs);
++	}
++	p->sched_time += ns;
++	account_group_exec_runtime(p, ns);
++
++	if (this_cpu_ksoftirqd() == p) {
++		/*
++		 * ksoftirqd time do not get accounted in cpu_softirq_time.
++		 * So, we have to handle it separately here.
++		 */
++		rq->softirq_pc += pc;
++		if (rq->softirq_pc >= 128) {
++			cpustat[CPUTIME_SOFTIRQ] += (__force u64)cputime_one_jiffy * rq->softirq_pc / 128;
++			rq->softirq_pc %= 128;
++		}
++	}
++
++	if (task_nice(p) > 0 || idleprio_task(p)) {
++		rq->nice_pc += pc;
++		if (rq->nice_pc >= 128) {
++			cpustat[CPUTIME_NICE] += (__force u64)cputime_one_jiffy * rq->nice_pc / 128;
++			rq->nice_pc %= 128;
++		}
++	} else {
++		rq->user_pc += pc;
++		if (rq->user_pc >= 128) {
++			cpustat[CPUTIME_USER] += (__force u64)cputime_one_jiffy * rq->user_pc / 128;
++			rq->user_pc %= 128;
++		}
++	}
++	acct_update_integrals(p);
++}
++
++/*
++ * Convert nanoseconds to pseudo percentage of one tick. Use 128 for fast
++ * shifts instead of 100
++ */
++#define NS_TO_PC(NS)	(NS * 128 / JIFFY_NS)
++
++/*
++ * This is called on clock ticks.
++ * Bank in p->sched_time the ns elapsed since the last tick or switch.
++ * CPU scheduler quota accounting is also performed here in microseconds.
++ */
++static void
++update_cpu_clock_tick(struct rq *rq, struct task_struct *p)
++{
++	long account_ns = rq->clock_task - rq->rq_last_ran;
++	struct task_struct *idle = rq->idle;
++	unsigned long account_pc;
++
++	if (unlikely(account_ns < 0) || steal_account_process_tick())
++		goto ts_account;
++
++	account_pc = NS_TO_PC(account_ns);
++
++	/* Accurate tick timekeeping */
++	if (user_mode(get_irq_regs()))
++		pc_user_time(rq, p, account_pc, account_ns);
++	else if (p != idle || (irq_count() != HARDIRQ_OFFSET))
++		pc_system_time(rq, p, HARDIRQ_OFFSET,
++			       account_pc, account_ns);
++	else
++		pc_idle_time(rq, idle, account_pc);
++
++	if (sched_clock_irqtime)
++		irqtime_account_hi_si();
++
++ts_account:
++	/* time_slice accounting is done in usecs to avoid overflow on 32bit */
++	if (rq->rq_policy != SCHED_FIFO && p != idle) {
++		s64 time_diff = rq->clock - rq->timekeep_clock;
++
++		niffy_diff(&time_diff, 1);
++		rq->rq_time_slice -= NS_TO_US(time_diff);
++	}
++
++	rq->rq_last_ran = rq->clock_task;
++	rq->timekeep_clock = rq->clock;
++}
++
++/*
++ * This is called on context switches.
++ * Bank in p->sched_time the ns elapsed since the last tick or switch.
++ * CPU scheduler quota accounting is also performed here in microseconds.
++ */
++static void
++update_cpu_clock_switch(struct rq *rq, struct task_struct *p)
++{
++	long account_ns = rq->clock_task - rq->rq_last_ran;
++	struct task_struct *idle = rq->idle;
++	unsigned long account_pc;
++
++	if (unlikely(account_ns < 0))
++		goto ts_account;
++
++	account_pc = NS_TO_PC(account_ns);
++
++	/* Accurate subtick timekeeping */
++	if (p != idle) {
++		pc_user_time(rq, p, account_pc, account_ns);
++	}
++	else
++		pc_idle_time(rq, idle, account_pc);
++
++ts_account:
++	/* time_slice accounting is done in usecs to avoid overflow on 32bit */
++	if (rq->rq_policy != SCHED_FIFO && p != idle) {
++		s64 time_diff = rq->clock - rq->timekeep_clock;
++
++		niffy_diff(&time_diff, 1);
++		rq->rq_time_slice -= NS_TO_US(time_diff);
++	}
++
++	rq->rq_last_ran = rq->clock_task;
++	rq->timekeep_clock = rq->clock;
++}
++
++/*
++ * Return any ns on the sched_clock that have not yet been accounted in
++ * @p in case that task is currently running.
++ *
++ * Called with task_grq_lock() held.
++ */
++static inline u64 do_task_delta_exec(struct task_struct *p, struct rq *rq)
++{
++	u64 ns = 0;
++
++	/*
++	 * Must be ->curr _and_ ->on_rq.  If dequeued, we would
++	 * project cycles that may never be accounted to this
++	 * thread, breaking clock_gettime().
++	 */
++	if (p == rq->curr && p->on_rq) {
++		update_clocks(rq);
++		ns = rq->clock_task - rq->rq_last_ran;
++		if (unlikely((s64)ns < 0))
++			ns = 0;
++	}
++
++	return ns;
++}
++
++/*
++ * Return accounted runtime for the task.
++ * Return separately the current's pending runtime that have not been
++ * accounted yet.
++ *
++ */
++unsigned long long task_sched_runtime(struct task_struct *p)
++{
++	unsigned long flags;
++	struct rq *rq;
++	u64 ns;
++
++#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
++	/*
++	 * 64-bit doesn't need locks to atomically read a 64bit value.
++	 * So we have a optimization chance when the task's delta_exec is 0.
++	 * Reading ->on_cpu is racy, but this is ok.
++	 *
++	 * If we race with it leaving cpu, we'll take a lock. So we're correct.
++	 * If we race with it entering cpu, unaccounted time is 0. This is
++	 * indistinguishable from the read occurring a few cycles earlier.
++	 * If we see ->on_cpu without ->on_rq, the task is leaving, and has
++	 * been accounted, so we're correct here as well.
++	 */
++	if (!p->on_cpu || !p->on_rq)
++		return tsk_seruntime(p);
++#endif
++
++	rq = task_grq_lock(p, &flags);
++	ns = p->sched_time + do_task_delta_exec(p, rq);
++	task_grq_unlock(&flags);
++
++	return ns;
++}
++
++/* Compatibility crap */
++void account_user_time(struct task_struct *p, cputime_t cputime,
++		       cputime_t cputime_scaled)
++{
++}
++
++void account_idle_time(cputime_t cputime)
++{
++}
++
++/*
++ * Account guest cpu time to a process.
++ * @p: the process that the cpu time gets accounted to
++ * @cputime: the cpu time spent in virtual machine since the last update
++ * @cputime_scaled: cputime scaled by cpu frequency
++ */
++static void account_guest_time(struct task_struct *p, cputime_t cputime,
++			       cputime_t cputime_scaled)
++{
++	u64 *cpustat = kcpustat_this_cpu->cpustat;
++
++	/* Add guest time to process. */
++	p->utime += (__force u64)cputime;
++	p->utimescaled += (__force u64)cputime_scaled;
++	account_group_user_time(p, cputime);
++	p->gtime += (__force u64)cputime;
++
++	/* Add guest time to cpustat. */
++	if (task_nice(p) > 0) {
++		cpustat[CPUTIME_NICE] += (__force u64)cputime;
++		cpustat[CPUTIME_GUEST_NICE] += (__force u64)cputime;
++	} else {
++		cpustat[CPUTIME_USER] += (__force u64)cputime;
++		cpustat[CPUTIME_GUEST] += (__force u64)cputime;
++	}
++}
++
++/*
++ * Account system cpu time to a process and desired cpustat field
++ * @p: the process that the cpu time gets accounted to
++ * @cputime: the cpu time spent in kernel space since the last update
++ * @cputime_scaled: cputime scaled by cpu frequency
++ * @target_cputime64: pointer to cpustat field that has to be updated
++ */
++static inline
++void __account_system_time(struct task_struct *p, cputime_t cputime,
++			cputime_t cputime_scaled, cputime64_t *target_cputime64)
++{
++	/* Add system time to process. */
++	p->stime += (__force u64)cputime;
++	p->stimescaled += (__force u64)cputime_scaled;
++	account_group_system_time(p, cputime);
++
++	/* Add system time to cpustat. */
++	*target_cputime64 += (__force u64)cputime;
++
++	/* Account for system time used */
++	acct_update_integrals(p);
++}
++
++/*
++ * Account system cpu time to a process.
++ * @p: the process that the cpu time gets accounted to
++ * @hardirq_offset: the offset to subtract from hardirq_count()
++ * @cputime: the cpu time spent in kernel space since the last update
++ * @cputime_scaled: cputime scaled by cpu frequency
++ * This is for guest only now.
++ */
++void account_system_time(struct task_struct *p, int hardirq_offset,
++			 cputime_t cputime, cputime_t cputime_scaled)
++{
++
++	if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0))
++		account_guest_time(p, cputime, cputime_scaled);
++}
++
++/*
++ * Account for involuntary wait time.
++ * @steal: the cpu time spent in involuntary wait
++ */
++void account_steal_time(cputime_t cputime)
++{
++	u64 *cpustat = kcpustat_this_cpu->cpustat;
++
++	cpustat[CPUTIME_STEAL] += (__force u64)cputime;
++}
++
++/*
++ * Account for idle time.
++ * @cputime: the cpu time spent in idle wait
++ */
++static void account_idle_times(cputime_t cputime)
++{
++	u64 *cpustat = kcpustat_this_cpu->cpustat;
++	struct rq *rq = this_rq();
++
++	if (atomic_read(&rq->nr_iowait) > 0)
++		cpustat[CPUTIME_IOWAIT] += (__force u64)cputime;
++	else
++		cpustat[CPUTIME_IDLE] += (__force u64)cputime;
++}
++
++#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
++
++void account_process_tick(struct task_struct *p, int user_tick)
++{
++}
++
++/*
++ * Account multiple ticks of steal time.
++ * @p: the process from which the cpu time has been stolen
++ * @ticks: number of stolen ticks
++ */
++void account_steal_ticks(unsigned long ticks)
++{
++	account_steal_time(jiffies_to_cputime(ticks));
++}
++
++/*
++ * Account multiple ticks of idle time.
++ * @ticks: number of stolen ticks
++ */
++void account_idle_ticks(unsigned long ticks)
++{
++	account_idle_times(jiffies_to_cputime(ticks));
++}
++#endif
++
++static inline void grq_iso_lock(void)
++	__acquires(grq.iso_lock)
++{
++	raw_spin_lock(&grq.iso_lock);
++}
++
++static inline void grq_iso_unlock(void)
++	__releases(grq.iso_lock)
++{
++	raw_spin_unlock(&grq.iso_lock);
++}
++
++/*
++ * Functions to test for when SCHED_ISO tasks have used their allocated
++ * quota as real time scheduling and convert them back to SCHED_NORMAL.
++ * Where possible, the data is tested lockless, to avoid grabbing iso_lock
++ * because the occasional inaccurate result won't matter. However the
++ * tick data is only ever modified under lock. iso_refractory is only simply
++ * set to 0 or 1 so it's not worth grabbing the lock yet again for that.
++ */
++static bool set_iso_refractory(void)
++{
++	grq.iso_refractory = true;
++	return grq.iso_refractory;
++}
++
++static bool clear_iso_refractory(void)
++{
++	grq.iso_refractory = false;
++	return grq.iso_refractory;
++}
++
++/*
++ * Test if SCHED_ISO tasks have run longer than their alloted period as RT
++ * tasks and set the refractory flag if necessary. There is 10% hysteresis
++ * for unsetting the flag. 115/128 is ~90/100 as a fast shift instead of a
++ * slow division.
++ */
++static bool test_ret_isorefractory(struct rq *rq)
++{
++	if (likely(!grq.iso_refractory)) {
++		if (grq.iso_ticks > ISO_PERIOD * sched_iso_cpu)
++			return set_iso_refractory();
++	} else {
++		if (grq.iso_ticks < ISO_PERIOD * (sched_iso_cpu * 115 / 128))
++			return clear_iso_refractory();
++	}
++	return grq.iso_refractory;
++}
++
++static void iso_tick(void)
++{
++	grq_iso_lock();
++	grq.iso_ticks += 100;
++	grq_iso_unlock();
++}
++
++/* No SCHED_ISO task was running so decrease rq->iso_ticks */
++static inline void no_iso_tick(void)
++{
++	if (grq.iso_ticks) {
++		grq_iso_lock();
++		grq.iso_ticks -= grq.iso_ticks / ISO_PERIOD + 1;
++		if (unlikely(grq.iso_refractory && grq.iso_ticks <
++		    ISO_PERIOD * (sched_iso_cpu * 115 / 128)))
++			clear_iso_refractory();
++		grq_iso_unlock();
++	}
++}
++
++/* This manages tasks that have run out of timeslice during a scheduler_tick */
++static void task_running_tick(struct rq *rq)
++{
++	struct task_struct *p;
++
++	/*
++	 * If a SCHED_ISO task is running we increment the iso_ticks. In
++	 * order to prevent SCHED_ISO tasks from causing starvation in the
++	 * presence of true RT tasks we account those as iso_ticks as well.
++	 */
++	if ((rt_queue(rq) || (iso_queue(rq) && !grq.iso_refractory))) {
++		if (grq.iso_ticks <= (ISO_PERIOD * 128) - 128)
++			iso_tick();
++	} else
++		no_iso_tick();
++
++	if (iso_queue(rq)) {
++		if (unlikely(test_ret_isorefractory(rq))) {
++			if (rq_running_iso(rq)) {
++				/*
++				 * SCHED_ISO task is running as RT and limit
++				 * has been hit. Force it to reschedule as
++				 * SCHED_NORMAL by zeroing its time_slice
++				 */
++				rq->rq_time_slice = 0;
++			}
++		}
++	}
++
++	/* SCHED_FIFO tasks never run out of timeslice. */
++	if (rq->rq_policy == SCHED_FIFO)
++		return;
++	/*
++	 * Tasks that were scheduled in the first half of a tick are not
++	 * allowed to run into the 2nd half of the next tick if they will
++	 * run out of time slice in the interim. Otherwise, if they have
++	 * less than RESCHED_US μs of time slice left they will be rescheduled.
++	 */
++	if (rq->dither) {
++		if (rq->rq_time_slice > HALF_JIFFY_US)
++			return;
++		else
++			rq->rq_time_slice = 0;
++	} else if (rq->rq_time_slice >= RESCHED_US)
++			return;
++
++	/* p->time_slice < RESCHED_US. We only modify task_struct under grq lock */
++	p = rq->curr;
++
++	grq_lock();
++	requeue_task(p);
++	__set_tsk_resched(p);
++	grq_unlock();
++}
++
++/*
++ * This function gets called by the timer code, with HZ frequency.
++ * We call it with interrupts disabled. The data modified is all
++ * local to struct rq so we don't need to grab grq lock.
++ */
++void scheduler_tick(void)
++{
++	int cpu __maybe_unused = smp_processor_id();
++	struct rq *rq = cpu_rq(cpu);
++
++	sched_clock_tick();
++	/* grq lock not grabbed, so only update rq clock */
++	update_rq_clock(rq);
++	update_cpu_clock_tick(rq, rq->curr);
++	if (!rq_idle(rq))
++		task_running_tick(rq);
++	else
++		no_iso_tick();
++	rq->last_tick = rq->clock;
++	perf_event_task_tick();
++}
++
++notrace unsigned long get_parent_ip(unsigned long addr)
++{
++	if (in_lock_functions(addr)) {
++		addr = CALLER_ADDR2;
++		if (in_lock_functions(addr))
++			addr = CALLER_ADDR3;
++	}
++	return addr;
++}
++
++#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
++				defined(CONFIG_PREEMPT_TRACER))
++void preempt_count_add(int val)
++{
++#ifdef CONFIG_DEBUG_PREEMPT
++	/*
++	 * Underflow?
++	 */
++	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
++		return;
++#endif
++	__preempt_count_add(val);
++#ifdef CONFIG_DEBUG_PREEMPT
++	/*
++	 * Spinlock count overflowing soon?
++	 */
++	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
++				PREEMPT_MASK - 10);
++#endif
++	if (preempt_count() == val) {
++		unsigned long ip = get_parent_ip(CALLER_ADDR1);
++#ifdef CONFIG_DEBUG_PREEMPT
++		current->preempt_disable_ip = ip;
++#endif
++		trace_preempt_off(CALLER_ADDR0, ip);
++	}
++}
++EXPORT_SYMBOL(preempt_count_add);
++NOKPROBE_SYMBOL(preempt_count_add);
++
++void preempt_count_sub(int val)
++{
++#ifdef CONFIG_DEBUG_PREEMPT
++	/*
++	 * Underflow?
++	 */
++	if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
++		return;
++	/*
++	 * Is the spinlock portion underflowing?
++	 */
++	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
++			!(preempt_count() & PREEMPT_MASK)))
++		return;
++#endif
++
++	if (preempt_count() == val)
++		trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
++	__preempt_count_sub(val);
++}
++EXPORT_SYMBOL(preempt_count_sub);
++NOKPROBE_SYMBOL(preempt_count_sub);
++#endif
++
++/*
++ * Deadline is "now" in niffies + (offset by priority). Setting the deadline
++ * is the key to everything. It distributes cpu fairly amongst tasks of the
++ * same nice value, it proportions cpu according to nice level, it means the
++ * task that last woke up the longest ago has the earliest deadline, thus
++ * ensuring that interactive tasks get low latency on wake up. The CPU
++ * proportion works out to the square of the virtual deadline difference, so
++ * this equation will give nice 19 3% CPU compared to nice 0.
++ */
++static inline u64 prio_deadline_diff(int user_prio)
++{
++	return (prio_ratios[user_prio] * rr_interval * (MS_TO_NS(1) / 128));
++}
++
++static inline u64 task_deadline_diff(struct task_struct *p)
++{
++	return prio_deadline_diff(TASK_USER_PRIO(p));
++}
++
++static inline u64 static_deadline_diff(int static_prio)
++{
++	return prio_deadline_diff(USER_PRIO(static_prio));
++}
++
++static inline int longest_deadline_diff(void)
++{
++	return prio_deadline_diff(39);
++}
++
++static inline int ms_longest_deadline_diff(void)
++{
++	return NS_TO_MS(longest_deadline_diff());
++}
++
++/*
++ * The time_slice is only refilled when it is empty and that is when we set a
++ * new deadline.
++ */
++static void time_slice_expired(struct task_struct *p)
++{
++	p->time_slice = timeslice();
++	p->deadline = grq.niffies + task_deadline_diff(p);
++#ifdef CONFIG_SMT_NICE
++	if (!p->mm)
++		p->smt_bias = 0;
++	else if (rt_task(p))
++		p->smt_bias = 1 << 30;
++	else if (task_running_iso(p))
++		p->smt_bias = 1 << 29;
++	else if (idleprio_task(p)) {
++		if (task_running_idle(p))
++			p->smt_bias = 0;
++		else
++			p->smt_bias = 1;
++	} else if (--p->smt_bias < 1)
++		p->smt_bias = MAX_PRIO - p->static_prio;
++#endif
++}
++
++/*
++ * Timeslices below RESCHED_US are considered as good as expired as there's no
++ * point rescheduling when there's so little time left. SCHED_BATCH tasks
++ * have been flagged be not latency sensitive and likely to be fully CPU
++ * bound so every time they're rescheduled they have their time_slice
++ * refilled, but get a new later deadline to have little effect on
++ * SCHED_NORMAL tasks.
++
++ */
++static inline void check_deadline(struct task_struct *p)
++{
++	if (p->time_slice < RESCHED_US || batch_task(p))
++		time_slice_expired(p);
++}
++
++#define BITOP_WORD(nr)		((nr) / BITS_PER_LONG)
++
++/*
++ * Scheduler queue bitmap specific find next bit.
++ */
++static inline unsigned long
++next_sched_bit(const unsigned long *addr, unsigned long offset)
++{
++	const unsigned long *p;
++	unsigned long result;
++	unsigned long size;
++	unsigned long tmp;
++
++	size = PRIO_LIMIT;
++	if (offset >= size)
++		return size;
++
++	p = addr + BITOP_WORD(offset);
++	result = offset & ~(BITS_PER_LONG-1);
++	size -= result;
++	offset %= BITS_PER_LONG;
++	if (offset) {
++		tmp = *(p++);
++		tmp &= (~0UL << offset);
++		if (size < BITS_PER_LONG)
++			goto found_first;
++		if (tmp)
++			goto found_middle;
++		size -= BITS_PER_LONG;
++		result += BITS_PER_LONG;
++	}
++	while (size & ~(BITS_PER_LONG-1)) {
++		if ((tmp = *(p++)))
++			goto found_middle;
++		result += BITS_PER_LONG;
++		size -= BITS_PER_LONG;
++	}
++	if (!size)
++		return result;
++	tmp = *p;
++
++found_first:
++	tmp &= (~0UL >> (BITS_PER_LONG - size));
++	if (tmp == 0UL)		/* Are any bits set? */
++		return result + size;	/* Nope. */
++found_middle:
++	return result + __ffs(tmp);
++}
++
++/*
++ * O(n) lookup of all tasks in the global runqueue. The real brainfuck
++ * of lock contention and O(n). It's not really O(n) as only the queued,
++ * but not running tasks are scanned, and is O(n) queued in the worst case
++ * scenario only because the right task can be found before scanning all of
++ * them.
++ * Tasks are selected in this order:
++ * Real time tasks are selected purely by their static priority and in the
++ * order they were queued, so the lowest value idx, and the first queued task
++ * of that priority value is chosen.
++ * If no real time tasks are found, the SCHED_ISO priority is checked, and
++ * all SCHED_ISO tasks have the same priority value, so they're selected by
++ * the earliest deadline value.
++ * If no SCHED_ISO tasks are found, SCHED_NORMAL tasks are selected by the
++ * earliest deadline.
++ * Finally if no SCHED_NORMAL tasks are found, SCHED_IDLEPRIO tasks are
++ * selected by the earliest deadline.
++ */
++static inline struct
++task_struct *earliest_deadline_task(struct rq *rq, int cpu, struct task_struct *idle)
++{
++	struct task_struct *edt = NULL;
++	unsigned long idx = -1;
++
++	do {
++		struct list_head *queue;
++		struct task_struct *p;
++		u64 earliest_deadline;
++
++		idx = next_sched_bit(grq.prio_bitmap, ++idx);
++		if (idx >= PRIO_LIMIT)
++			return idle;
++		queue = grq.queue + idx;
++
++		if (idx < MAX_RT_PRIO) {
++			/* We found an rt task */
++			list_for_each_entry(p, queue, run_list) {
++				/* Make sure cpu affinity is ok */
++				if (needs_other_cpu(p, cpu))
++					continue;
++				edt = p;
++				goto out_take;
++			}
++			/*
++			 * None of the RT tasks at this priority can run on
++			 * this cpu
++			 */
++			continue;
++		}
++
++		/*
++		 * No rt tasks. Find the earliest deadline task. Now we're in
++		 * O(n) territory.
++		 */
++		earliest_deadline = ~0ULL;
++		list_for_each_entry(p, queue, run_list) {
++			u64 dl;
++
++			/* Make sure cpu affinity is ok */
++			if (needs_other_cpu(p, cpu))
++				continue;
++
++#ifdef CONFIG_SMT_NICE
++			if (!smt_should_schedule(p, cpu))
++				continue;
++#endif
++			/*
++			 * Soft affinity happens here by not scheduling a task
++			 * with its sticky flag set that ran on a different CPU
++			 * last when the CPU is scaling, or by greatly biasing
++			 * against its deadline when not, based on cpu cache
++			 * locality.
++			 */
++			if (sched_interactive)
++				dl = p->deadline;
++			else {
++				int tcpu = task_cpu(p);
++
++				if (tcpu != cpu && task_sticky(p) && scaling_rq(rq))
++					continue;
++				dl = p->deadline << locality_diff(tcpu, rq);
++			}
++
++			if (deadline_before(dl, earliest_deadline)) {
++				earliest_deadline = dl;
++				edt = p;
++			}
++		}
++	} while (!edt);
++
++out_take:
++	take_task(cpu, edt);
++	return edt;
++}
++
++
++/*
++ * Print scheduling while atomic bug:
++ */
++static noinline void __schedule_bug(struct task_struct *prev)
++{
++	if (oops_in_progress)
++		return;
++
++	printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
++		prev->comm, prev->pid, preempt_count());
++
++	debug_show_held_locks(prev);
++	print_modules();
++	if (irqs_disabled())
++		print_irqtrace_events(prev);
++#ifdef CONFIG_DEBUG_PREEMPT
++	if (in_atomic_preempt_off()) {
++		pr_err("Preemption disabled at:");
++		print_ip_sym(current->preempt_disable_ip);
++		pr_cont("\n");
++	}
++#endif
++	dump_stack();
++	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
++}
++
++/*
++ * Various schedule()-time debugging checks and statistics:
++ */
++static inline void schedule_debug(struct task_struct *prev)
++{
++#ifdef CONFIG_SCHED_STACK_END_CHECK
++	BUG_ON(task_stack_end_corrupted(prev));
++#endif
++
++	if (unlikely(in_atomic_preempt_off())) {
++		__schedule_bug(prev);
++		preempt_count_set(PREEMPT_DISABLED);
++	}
++	rcu_sleep_check();
++
++	profile_hit(SCHED_PROFILING, __builtin_return_address(0));
++
++	schedstat_inc(this_rq(), sched_count);
++}
++
++/*
++ * The currently running task's information is all stored in rq local data
++ * which is only modified by the local CPU, thereby allowing the data to be
++ * changed without grabbing the grq lock.
++ */
++static inline void set_rq_task(struct rq *rq, struct task_struct *p)
++{
++	rq->rq_time_slice = p->time_slice;
++	rq->rq_deadline = p->deadline;
++	rq->rq_last_ran = p->last_ran = rq->clock_task;
++	rq->rq_policy = p->policy;
++	rq->rq_prio = p->prio;
++#ifdef CONFIG_SMT_NICE
++	rq->rq_mm = p->mm;
++	rq->rq_smt_bias = p->smt_bias;
++#endif
++	if (p != rq->idle)
++		rq->rq_running = true;
++	else
++		rq->rq_running = false;
++}
++
++static void reset_rq_task(struct rq *rq, struct task_struct *p)
++{
++	rq->rq_policy = p->policy;
++	rq->rq_prio = p->prio;
++#ifdef CONFIG_SMT_NICE
++	rq->rq_smt_bias = p->smt_bias;
++#endif
++}
++
++#ifdef CONFIG_SMT_NICE
++/* Iterate over smt siblings when we've scheduled a process on cpu and decide
++ * whether they should continue running or be descheduled. */
++static void check_smt_siblings(int cpu)
++{
++	int other_cpu;
++
++	for_each_cpu(other_cpu, thread_cpumask(cpu)) {
++		struct task_struct *p;
++		struct rq *rq;
++
++		if (other_cpu == cpu)
++			continue;
++		rq = cpu_rq(other_cpu);
++		if (rq_idle(rq))
++			continue;
++		if (!rq->online)
++			continue;
++		p = rq->curr;
++		if (!smt_should_schedule(p, cpu)) {
++			set_tsk_need_resched(p);
++			smp_send_reschedule(other_cpu);
++		}
++	}
++}
++
++static void wake_smt_siblings(int cpu)
++{
++	int other_cpu;
++
++	if (!queued_notrunning())
++		return;
++
++	for_each_cpu(other_cpu, thread_cpumask(cpu)) {
++		struct rq *rq;
++
++		if (other_cpu == cpu)
++			continue;
++		rq = cpu_rq(other_cpu);
++		if (rq_idle(rq)) {
++			struct task_struct *p = rq->curr;
++
++			set_tsk_need_resched(p);
++			smp_send_reschedule(other_cpu);
++		}
++	}
++}
++#else
++static void check_smt_siblings(int __maybe_unused cpu) {}
++static void wake_smt_siblings(int __maybe_unused cpu) {}
++#endif
++
++/*
++ * schedule() is the main scheduler function.
++ *
++ * The main means of driving the scheduler and thus entering this function are:
++ *
++ *   1. Explicit blocking: mutex, semaphore, waitqueue, etc.
++ *
++ *   2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return
++ *      paths. For example, see arch/x86/entry_64.S.
++ *
++ *      To drive preemption between tasks, the scheduler sets the flag in timer
++ *      interrupt handler scheduler_tick().
++ *
++ *   3. Wakeups don't really cause entry into schedule(). They add a
++ *      task to the run-queue and that's it.
++ *
++ *      Now, if the new task added to the run-queue preempts the current
++ *      task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets
++ *      called on the nearest possible occasion:
++ *
++ *       - If the kernel is preemptible (CONFIG_PREEMPT=y):
++ *
++ *         - in syscall or exception context, at the next outmost
++ *           preempt_enable(). (this might be as soon as the wake_up()'s
++ *           spin_unlock()!)
++ *
++ *         - in IRQ context, return from interrupt-handler to
++ *           preemptible context
++ *
++ *       - If the kernel is not preemptible (CONFIG_PREEMPT is not set)
++ *         then at the next:
++ *
++ *          - cond_resched() call
++ *          - explicit schedule() call
++ *          - return from syscall or exception to user-space
++ *          - return from interrupt-handler to user-space
++ *
++ * WARNING: must be called with preemption disabled!
++ */
++static void __sched notrace __schedule(bool preempt)
++{
++	struct task_struct *prev, *next, *idle;
++	unsigned long *switch_count;
++	bool deactivate = false;
++	struct rq *rq;
++	int cpu;
++
++	cpu = smp_processor_id();
++	rq = cpu_rq(cpu);
++	rcu_note_context_switch();
++	prev = rq->curr;
++
++	/*
++	 * do_exit() calls schedule() with preemption disabled as an exception;
++	 * however we must fix that up, otherwise the next task will see an
++	 * inconsistent (higher) preempt count.
++	 *
++	 * It also avoids the below schedule_debug() test from complaining
++	 * about this.
++	 */
++	if (unlikely(prev->state == TASK_DEAD))
++		preempt_enable_no_resched_notrace();
++
++	schedule_debug(prev);
++
++	/*
++	 * Make sure that signal_pending_state()->signal_pending() below
++	 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
++	 * done by the caller to avoid the race with signal_wake_up().
++	 */
++	smp_mb__before_spinlock();
++	grq_lock_irq();
++
++	switch_count = &prev->nivcsw;
++	if (!preempt && prev->state) {
++		if (unlikely(signal_pending_state(prev->state, prev))) {
++			prev->state = TASK_RUNNING;
++		} else {
++			deactivate = true;
++			prev->on_rq = 0;
++
++			/*
++			 * If a worker is going to sleep, notify and
++			 * ask workqueue whether it wants to wake up a
++			 * task to maintain concurrency.  If so, wake
++			 * up the task.
++			 */
++			if (prev->flags & PF_WQ_WORKER) {
++				struct task_struct *to_wakeup;
++
++				to_wakeup = wq_worker_sleeping(prev, cpu);
++				if (to_wakeup) {
++					/* This shouldn't happen, but does */
++					if (unlikely(to_wakeup == prev))
++						deactivate = false;
++					else
++						try_to_wake_up_local(to_wakeup);
++				}
++			}
++		}
++		switch_count = &prev->nvcsw;
++	}
++
++	update_clocks(rq);
++	update_cpu_clock_switch(rq, prev);
++	if (rq->clock - rq->last_tick > HALF_JIFFY_NS)
++		rq->dither = false;
++	else
++		rq->dither = true;
++
++	clear_tsk_need_resched(prev);
++	clear_preempt_need_resched();
++
++	idle = rq->idle;
++	if (idle != prev) {
++		/* Update all the information stored on struct rq */
++		prev->time_slice = rq->rq_time_slice;
++		prev->deadline = rq->rq_deadline;
++		check_deadline(prev);
++		prev->last_ran = rq->clock_task;
++
++		/* Task changed affinity off this CPU */
++		if (likely(!needs_other_cpu(prev, cpu))) {
++			if (!deactivate) {
++				if (!queued_notrunning()) {
++					/*
++					 * We now know prev is the only thing that is
++					 * awaiting CPU so we can bypass rechecking for
++					 * the earliest deadline task and just run it
++					 * again.
++					 */
++					set_rq_task(rq, prev);
++					check_smt_siblings(cpu);
++					grq_unlock_irq();
++					goto rerun_prev_unlocked;
++				} else
++					swap_sticky(rq, cpu, prev);
++			}
++		}
++		return_task(prev, rq, deactivate);
++	}
++
++	if (unlikely(!queued_notrunning())) {
++		/*
++		 * This CPU is now truly idle as opposed to when idle is
++		 * scheduled as a high priority task in its own right.
++		 */
++		next = idle;
++		schedstat_inc(rq, sched_goidle);
++		set_cpuidle_map(cpu);
++	} else {
++		next = earliest_deadline_task(rq, cpu, idle);
++		if (likely(next->prio != PRIO_LIMIT))
++			clear_cpuidle_map(cpu);
++		else
++			set_cpuidle_map(cpu);
++	}
++
++	if (likely(prev != next)) {
++		/*
++		 * Don't reschedule an idle task or deactivated tasks
++		 */
++		if (prev != idle && !deactivate)
++			resched_suitable_idle(prev);
++		/*
++		 * Don't stick tasks when a real time task is going to run as
++		 * they may literally get stuck.
++		 */
++		if (rt_task(next))
++			unstick_task(rq, prev);
++		set_rq_task(rq, next);
++		if (next != idle)
++			check_smt_siblings(cpu);
++		else
++			wake_smt_siblings(cpu);
++		grq.nr_switches++;
++		prev->on_cpu = false;
++		next->on_cpu = true;
++		rq->curr = next;
++		++*switch_count;
++
++		trace_sched_switch(preempt, prev, next);
++		rq = context_switch(rq, prev, next); /* unlocks the grq */
++		cpu = cpu_of(rq);
++		idle = rq->idle;
++	} else {
++		check_smt_siblings(cpu);
++		grq_unlock_irq();
++	}
++
++rerun_prev_unlocked:
++	return;
++}
++
++static inline void sched_submit_work(struct task_struct *tsk)
++{
++	if (!tsk->state || tsk_is_pi_blocked(tsk) ||
++	    preempt_count() ||
++	    signal_pending_state(tsk->state, tsk))
++		return;
++
++	/*
++	 * If we are going to sleep and we have plugged IO queued,
++	 * make sure to submit it to avoid deadlocks.
++	 */
++	if (blk_needs_flush_plug(tsk))
++		blk_schedule_flush_plug(tsk);
++}
++
++asmlinkage __visible void __sched schedule(void)
++{
++	struct task_struct *tsk = current;
++
++	sched_submit_work(tsk);
++	do {
++		preempt_disable();
++		__schedule(false);
++		sched_preempt_enable_no_resched();
++	} while (need_resched());
++}
++
++EXPORT_SYMBOL(schedule);
++
++#ifdef CONFIG_CONTEXT_TRACKING
++asmlinkage __visible void __sched schedule_user(void)
++{
++	/*
++	 * If we come here after a random call to set_need_resched(),
++	 * or we have been woken up remotely but the IPI has not yet arrived,
++	 * we haven't yet exited the RCU idle mode. Do it here manually until
++	 * we find a better solution.
++	 *
++	 * NB: There are buggy callers of this function.  Ideally we
++	 * should warn if prev_state != IN_USER, but that will trigger
++	 * too frequently to make sense yet.
++	 */
++	enum ctx_state prev_state = exception_enter();
++	schedule();
++	exception_exit(prev_state);
++}
++#endif
++
++/**
++ * schedule_preempt_disabled - called with preemption disabled
++ *
++ * Returns with preemption disabled. Note: preempt_count must be 1
++ */
++void __sched schedule_preempt_disabled(void)
++{
++	sched_preempt_enable_no_resched();
++	schedule();
++	preempt_disable();
++}
++
++static void __sched notrace preempt_schedule_common(void)
++{
++	do {
++		preempt_disable_notrace();
++		__schedule(true);
++		preempt_enable_no_resched_notrace();
++
++		/*
++		 * Check again in case we missed a preemption opportunity
++		 * between schedule and now.
++		 */
++	} while (need_resched());
++}
++
++#ifdef CONFIG_PREEMPT
++/*
++ * this is the entry point to schedule() from in-kernel preemption
++ * off of preempt_enable. Kernel preemptions off return from interrupt
++ * occur there and call schedule directly.
++ */
++asmlinkage __visible void __sched notrace preempt_schedule(void)
++{
++	/*
++	 * If there is a non-zero preempt_count or interrupts are disabled,
++	 * we do not want to preempt the current task. Just return..
++	 */
++	if (likely(!preemptible()))
++		return;
++
++	preempt_schedule_common();
++}
++NOKPROBE_SYMBOL(preempt_schedule);
++EXPORT_SYMBOL(preempt_schedule);
++
++/**
++ * preempt_schedule_notrace - preempt_schedule called by tracing
++ *
++ * The tracing infrastructure uses preempt_enable_notrace to prevent
++ * recursion and tracing preempt enabling caused by the tracing
++ * infrastructure itself. But as tracing can happen in areas coming
++ * from userspace or just about to enter userspace, a preempt enable
++ * can occur before user_exit() is called. This will cause the scheduler
++ * to be called when the system is still in usermode.
++ *
++ * To prevent this, the preempt_enable_notrace will use this function
++ * instead of preempt_schedule() to exit user context if needed before
++ * calling the scheduler.
++ */
++asmlinkage __visible void __sched notrace preempt_schedule_notrace(void)
++{
++	enum ctx_state prev_ctx;
++
++	if (likely(!preemptible()))
++		return;
++
++	do {
++		preempt_disable_notrace();
++		/*
++		 * Needs preempt disabled in case user_exit() is traced
++		 * and the tracer calls preempt_enable_notrace() causing
++		 * an infinite recursion.
++		 */
++		prev_ctx = exception_enter();
++		__schedule(true);
++		exception_exit(prev_ctx);
++
++		preempt_enable_no_resched_notrace();
++	} while (need_resched());
++}
++EXPORT_SYMBOL_GPL(preempt_schedule_notrace);
++
++#endif /* CONFIG_PREEMPT */
++
++/*
++ * this is the entry point to schedule() from kernel preemption
++ * off of irq context.
++ * Note, that this is called and return with irqs disabled. This will
++ * protect us against recursive calling from irq.
++ */
++asmlinkage __visible void __sched preempt_schedule_irq(void)
++{
++	enum ctx_state prev_state;
++
++	/* Catch callers which need to be fixed */
++	BUG_ON(preempt_count() || !irqs_disabled());
++
++	prev_state = exception_enter();
++
++	do {
++		preempt_disable();
++		local_irq_enable();
++		__schedule(true);
++		local_irq_disable();
++		sched_preempt_enable_no_resched();
++	} while (need_resched());
++
++	exception_exit(prev_state);
++}
++
++int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
++			  void *key)
++{
++	return try_to_wake_up(curr->private, mode, wake_flags);
++}
++EXPORT_SYMBOL(default_wake_function);
++
++#ifdef CONFIG_RT_MUTEXES
++
++/*
++ * rt_mutex_setprio - set the current priority of a task
++ * @p: task
++ * @prio: prio value (kernel-internal form)
++ *
++ * This function changes the 'effective' priority of a task. It does
++ * not touch ->normal_prio like __setscheduler().
++ *
++ * Used by the rt_mutex code to implement priority inheritance
++ * logic. Call site only calls if the priority of the task changed.
++ */
++void rt_mutex_setprio(struct task_struct *p, int prio)
++{
++	unsigned long flags;
++	int queued, oldprio;
++	struct rq *rq;
++
++	BUG_ON(prio < 0 || prio > MAX_PRIO);
++
++	rq = task_grq_lock(p, &flags);
++
++	/*
++	 * Idle task boosting is a nono in general. There is one
++	 * exception, when PREEMPT_RT and NOHZ is active:
++	 *
++	 * The idle task calls get_next_timer_interrupt() and holds
++	 * the timer wheel base->lock on the CPU and another CPU wants
++	 * to access the timer (probably to cancel it). We can safely
++	 * ignore the boosting request, as the idle CPU runs this code
++	 * with interrupts disabled and will complete the lock
++	 * protected section without being interrupted. So there is no
++	 * real need to boost.
++	 */
++	if (unlikely(p == rq->idle)) {
++		WARN_ON(p != rq->curr);
++		WARN_ON(p->pi_blocked_on);
++		goto out_unlock;
++	}
++
++	trace_sched_pi_setprio(p, prio);
++	oldprio = p->prio;
++	queued = task_queued(p);
++	if (queued)
++		dequeue_task(p);
++	p->prio = prio;
++	if (task_running(p) && prio > oldprio)
++		resched_task(p);
++	if (queued) {
++		enqueue_task(p, rq);
++		try_preempt(p, rq);
++	}
++
++out_unlock:
++	task_grq_unlock(&flags);
++}
++
++#endif
++
++/*
++ * Adjust the deadline for when the priority is to change, before it's
++ * changed.
++ */
++static inline void adjust_deadline(struct task_struct *p, int new_prio)
++{
++	p->deadline += static_deadline_diff(new_prio) - task_deadline_diff(p);
++}
++
++void set_user_nice(struct task_struct *p, long nice)
++{
++	int queued, new_static, old_static;
++	unsigned long flags;
++	struct rq *rq;
++
++	if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)
++		return;
++	new_static = NICE_TO_PRIO(nice);
++	/*
++	 * We have to be careful, if called from sys_setpriority(),
++	 * the task might be in the middle of scheduling on another CPU.
++	 */
++	rq = time_task_grq_lock(p, &flags);
++	/*
++	 * The RT priorities are set via sched_setscheduler(), but we still
++	 * allow the 'normal' nice value to be set - but as expected
++	 * it wont have any effect on scheduling until the task is
++	 * not SCHED_NORMAL/SCHED_BATCH:
++	 */
++	if (has_rt_policy(p)) {
++		p->static_prio = new_static;
++		goto out_unlock;
++	}
++	queued = task_queued(p);
++	if (queued)
++		dequeue_task(p);
++
++	adjust_deadline(p, new_static);
++	old_static = p->static_prio;
++	p->static_prio = new_static;
++	p->prio = effective_prio(p);
++
++	if (queued) {
++		enqueue_task(p, rq);
++		if (new_static < old_static)
++			try_preempt(p, rq);
++	} else if (task_running(p)) {
++		reset_rq_task(rq, p);
++		if (old_static < new_static)
++			resched_task(p);
++	}
++out_unlock:
++	task_grq_unlock(&flags);
++}
++EXPORT_SYMBOL(set_user_nice);
++
++/*
++ * can_nice - check if a task can reduce its nice value
++ * @p: task
++ * @nice: nice value
++ */
++int can_nice(const struct task_struct *p, const int nice)
++{
++	/* convert nice value [19,-20] to rlimit style value [1,40] */
++	int nice_rlim = nice_to_rlimit(nice);
++
++	return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
++		capable(CAP_SYS_NICE));
++}
++
++#ifdef __ARCH_WANT_SYS_NICE
++
++/*
++ * sys_nice - change the priority of the current process.
++ * @increment: priority increment
++ *
++ * sys_setpriority is a more generic, but much slower function that
++ * does similar things.
++ */
++SYSCALL_DEFINE1(nice, int, increment)
++{
++	long nice, retval;
++
++	/*
++	 * Setpriority might change our priority at the same moment.
++	 * We don't have to worry. Conceptually one call occurs first
++	 * and we have a single winner.
++	 */
++
++	increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH);
++	nice = task_nice(current) + increment;
++
++	nice = clamp_val(nice, MIN_NICE, MAX_NICE);
++	if (increment < 0 && !can_nice(current, nice))
++		return -EPERM;
++
++	retval = security_task_setnice(current, nice);
++	if (retval)
++		return retval;
++
++	set_user_nice(current, nice);
++	return 0;
++}
++
++#endif
++
++/**
++ * task_prio - return the priority value of a given task.
++ * @p: the task in question.
++ *
++ * Return: The priority value as seen by users in /proc.
++ * RT tasks are offset by -100. Normal tasks are centered around 1, value goes
++ * from 0 (SCHED_ISO) up to 82 (nice +19 SCHED_IDLEPRIO).
++ */
++int task_prio(const struct task_struct *p)
++{
++	int delta, prio = p->prio - MAX_RT_PRIO;
++
++	/* rt tasks and iso tasks */
++	if (prio <= 0)
++		goto out;
++
++	/* Convert to ms to avoid overflows */
++	delta = NS_TO_MS(p->deadline - grq.niffies);
++	delta = delta * 40 / ms_longest_deadline_diff();
++	if (delta > 0 && delta <= 80)
++		prio += delta;
++	if (idleprio_task(p))
++		prio += 40;
++out:
++	return prio;
++}
++
++/**
++ * idle_cpu - is a given cpu idle currently?
++ * @cpu: the processor in question.
++ *
++ * Return: 1 if the CPU is currently idle. 0 otherwise.
++ */
++int idle_cpu(int cpu)
++{
++	return cpu_curr(cpu) == cpu_rq(cpu)->idle;
++}
++
++/**
++ * idle_task - return the idle task for a given cpu.
++ * @cpu: the processor in question.
++ *
++ * Return: The idle task for the cpu @cpu.
++ */
++struct task_struct *idle_task(int cpu)
++{
++	return cpu_rq(cpu)->idle;
++}
++
++/**
++ * find_process_by_pid - find a process with a matching PID value.
++ * @pid: the pid in question.
++ *
++ * The task of @pid, if found. %NULL otherwise.
++ */
++static inline struct task_struct *find_process_by_pid(pid_t pid)
++{
++	return pid ? find_task_by_vpid(pid) : current;
++}
++
++/* Actually do priority change: must hold grq lock. */
++static void __setscheduler(struct task_struct *p, struct rq *rq, int policy,
++			   int prio, bool keep_boost)
++{
++	int oldrtprio, oldprio;
++
++	p->policy = policy;
++	oldrtprio = p->rt_priority;
++	p->rt_priority = prio;
++	p->normal_prio = normal_prio(p);
++	oldprio = p->prio;
++	/*
++	 * Keep a potential priority boosting if called from
++	 * sched_setscheduler().
++	 */
++	if (keep_boost) {
++		/*
++		 * Take priority boosted tasks into account. If the new
++		 * effective priority is unchanged, we just store the new
++		 * normal parameters and do not touch the scheduler class and
++		 * the runqueue. This will be done when the task deboost
++		 * itself.
++		 */
++		p->prio = rt_mutex_get_effective_prio(p, p->normal_prio);
++	} else
++		p->prio = p->normal_prio;
++	if (task_running(p)) {
++		reset_rq_task(rq, p);
++		/* Resched only if we might now be preempted */
++		if (p->prio > oldprio || p->rt_priority > oldrtprio)
++			resched_task(p);
++	}
++}
++
++/*
++ * check the target process has a UID that matches the current process's
++ */
++static bool check_same_owner(struct task_struct *p)
++{
++	const struct cred *cred = current_cred(), *pcred;
++	bool match;
++
++	rcu_read_lock();
++	pcred = __task_cred(p);
++	match = (uid_eq(cred->euid, pcred->euid) ||
++		 uid_eq(cred->euid, pcred->uid));
++	rcu_read_unlock();
++	return match;
++}
++
++static int
++__sched_setscheduler(struct task_struct *p, int policy,
++		     const struct sched_param *param, bool user, bool pi)
++{
++	struct sched_param zero_param = { .sched_priority = 0 };
++	int queued, retval, oldpolicy = -1;
++	unsigned long flags, rlim_rtprio = 0;
++	int reset_on_fork;
++	struct rq *rq;
++
++	/* may grab non-irq protected spin_locks */
++	BUG_ON(in_interrupt());
++
++	if (is_rt_policy(policy) && !capable(CAP_SYS_NICE)) {
++		unsigned long lflags;
++
++		if (!lock_task_sighand(p, &lflags))
++			return -ESRCH;
++		rlim_rtprio = task_rlimit(p, RLIMIT_RTPRIO);
++		unlock_task_sighand(p, &lflags);
++		if (rlim_rtprio)
++			goto recheck;
++		/*
++		 * If the caller requested an RT policy without having the
++		 * necessary rights, we downgrade the policy to SCHED_ISO.
++		 * We also set the parameter to zero to pass the checks.
++		 */
++		policy = SCHED_ISO;
++		param = &zero_param;
++	}
++recheck:
++	/* double check policy once rq lock held */
++	if (policy < 0) {
++		reset_on_fork = p->sched_reset_on_fork;
++		policy = oldpolicy = p->policy;
++	} else {
++		reset_on_fork = !!(policy & SCHED_RESET_ON_FORK);
++		policy &= ~SCHED_RESET_ON_FORK;
++
++		if (!SCHED_RANGE(policy))
++			return -EINVAL;
++	}
++
++	/*
++	 * Valid priorities for SCHED_FIFO and SCHED_RR are
++	 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL and
++	 * SCHED_BATCH is 0.
++	 */
++	if (param->sched_priority < 0 ||
++	    (p->mm && param->sched_priority > MAX_USER_RT_PRIO - 1) ||
++	    (!p->mm && param->sched_priority > MAX_RT_PRIO - 1))
++		return -EINVAL;
++	if (is_rt_policy(policy) != (param->sched_priority != 0))
++		return -EINVAL;
++
++	/*
++	 * Allow unprivileged RT tasks to decrease priority:
++	 */
++	if (user && !capable(CAP_SYS_NICE)) {
++		if (is_rt_policy(policy)) {
++			unsigned long rlim_rtprio =
++					task_rlimit(p, RLIMIT_RTPRIO);
++
++			/* can't set/change the rt policy */
++			if (policy != p->policy && !rlim_rtprio)
++				return -EPERM;
++
++			/* can't increase priority */
++			if (param->sched_priority > p->rt_priority &&
++			    param->sched_priority > rlim_rtprio)
++				return -EPERM;
++		} else {
++			switch (p->policy) {
++				/*
++				 * Can only downgrade policies but not back to
++				 * SCHED_NORMAL
++				 */
++				case SCHED_ISO:
++					if (policy == SCHED_ISO)
++						goto out;
++					if (policy == SCHED_NORMAL)
++						return -EPERM;
++					break;
++				case SCHED_BATCH:
++					if (policy == SCHED_BATCH)
++						goto out;
++					if (policy != SCHED_IDLEPRIO)
++						return -EPERM;
++					break;
++				case SCHED_IDLEPRIO:
++					if (policy == SCHED_IDLEPRIO)
++						goto out;
++					return -EPERM;
++				default:
++					break;
++			}
++		}
++
++		/* can't change other user's priorities */
++		if (!check_same_owner(p))
++			return -EPERM;
++
++		/* Normal users shall not reset the sched_reset_on_fork flag */
++		if (p->sched_reset_on_fork && !reset_on_fork)
++			return -EPERM;
++	}
++
++	if (user) {
++		retval = security_task_setscheduler(p);
++		if (retval)
++			return retval;
++	}
++
++	/*
++	 * make sure no PI-waiters arrive (or leave) while we are
++	 * changing the priority of the task:
++	 */
++	raw_spin_lock_irqsave(&p->pi_lock, flags);
++	/*
++	 * To be able to change p->policy safely, the grunqueue lock must be
++	 * held.
++	 */
++	rq = __task_grq_lock(p);
++
++	/*
++	 * Changing the policy of the stop threads its a very bad idea
++	 */
++	if (p == rq->stop) {
++		__task_grq_unlock();
++		raw_spin_unlock_irqrestore(&p->pi_lock, flags);
++		return -EINVAL;
++	}
++
++	/*
++	 * If not changing anything there's no need to proceed further:
++	 */
++	if (unlikely(policy == p->policy && (!is_rt_policy(policy) ||
++			param->sched_priority == p->rt_priority))) {
++
++		__task_grq_unlock();
++		raw_spin_unlock_irqrestore(&p->pi_lock, flags);
++		return 0;
++	}
++
++	/* recheck policy now with rq lock held */
++	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
++		policy = oldpolicy = -1;
++		__task_grq_unlock();
++		raw_spin_unlock_irqrestore(&p->pi_lock, flags);
++		goto recheck;
++	}
++	update_clocks(rq);
++	p->sched_reset_on_fork = reset_on_fork;
++
++	queued = task_queued(p);
++	if (queued)
++		dequeue_task(p);
++	__setscheduler(p, rq, policy, param->sched_priority, pi);
++	if (queued) {
++		enqueue_task(p, rq);
++		try_preempt(p, rq);
++	}
++	__task_grq_unlock();
++	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
++
++	if (pi)
++		rt_mutex_adjust_pi(p);
++out:
++	return 0;
++}
++
++/**
++ * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
++ * @p: the task in question.
++ * @policy: new policy.
++ * @param: structure containing the new RT priority.
++ *
++ * Return: 0 on success. An error code otherwise.
++ *
++ * NOTE that the task may be already dead.
++ */
++int sched_setscheduler(struct task_struct *p, int policy,
++		       const struct sched_param *param)
++{
++	return __sched_setscheduler(p, policy, param, true, true);
++}
++
++EXPORT_SYMBOL_GPL(sched_setscheduler);
++
++int sched_setattr(struct task_struct *p, const struct sched_attr *attr)
++{
++	const struct sched_param param = { .sched_priority = attr->sched_priority };
++	int policy = attr->sched_policy;
++
++	return __sched_setscheduler(p, policy, &param, true, true);
++}
++EXPORT_SYMBOL_GPL(sched_setattr);
++
++/**
++ * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
++ * @p: the task in question.
++ * @policy: new policy.
++ * @param: structure containing the new RT priority.
++ *
++ * Just like sched_setscheduler, only don't bother checking if the
++ * current context has permission.  For example, this is needed in
++ * stop_machine(): we create temporary high priority worker threads,
++ * but our caller might not have that capability.
++ *
++ * Return: 0 on success. An error code otherwise.
++ */
++int sched_setscheduler_nocheck(struct task_struct *p, int policy,
++			       const struct sched_param *param)
++{
++	return __sched_setscheduler(p, policy, param, false, true);
++}
++EXPORT_SYMBOL_GPL(sched_setscheduler_nocheck);
++
++static int
++do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
++{
++	struct sched_param lparam;
++	struct task_struct *p;
++	int retval;
++
++	if (!param || pid < 0)
++		return -EINVAL;
++	if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
++		return -EFAULT;
++
++	rcu_read_lock();
++	retval = -ESRCH;
++	p = find_process_by_pid(pid);
++	if (p != NULL)
++		retval = sched_setscheduler(p, policy, &lparam);
++	rcu_read_unlock();
++
++	return retval;
++}
++
++/*
++ * Mimics kernel/events/core.c perf_copy_attr().
++ */
++static int sched_copy_attr(struct sched_attr __user *uattr,
++			   struct sched_attr *attr)
++{
++	u32 size;
++	int ret;
++
++	if (!access_ok(VERIFY_WRITE, uattr, SCHED_ATTR_SIZE_VER0))
++		return -EFAULT;
++
++	/*
++	 * zero the full structure, so that a short copy will be nice.
++	 */
++	memset(attr, 0, sizeof(*attr));
++
++	ret = get_user(size, &uattr->size);
++	if (ret)
++		return ret;
++
++	if (size > PAGE_SIZE)	/* silly large */
++		goto err_size;
++
++	if (!size)		/* abi compat */
++		size = SCHED_ATTR_SIZE_VER0;
++
++	if (size < SCHED_ATTR_SIZE_VER0)
++		goto err_size;
++
++	/*
++	 * If we're handed a bigger struct than we know of,
++	 * ensure all the unknown bits are 0 - i.e. new
++	 * user-space does not rely on any kernel feature
++	 * extensions we dont know about yet.
++	 */
++	if (size > sizeof(*attr)) {
++		unsigned char __user *addr;
++		unsigned char __user *end;
++		unsigned char val;
++
++		addr = (void __user *)uattr + sizeof(*attr);
++		end  = (void __user *)uattr + size;
++
++		for (; addr < end; addr++) {
++			ret = get_user(val, addr);
++			if (ret)
++				return ret;
++			if (val)
++				goto err_size;
++		}
++		size = sizeof(*attr);
++	}
++
++	ret = copy_from_user(attr, uattr, size);
++	if (ret)
++		return -EFAULT;
++
++	/*
++	 * XXX: do we want to be lenient like existing syscalls; or do we want
++	 * to be strict and return an error on out-of-bounds values?
++	 */
++	attr->sched_nice = clamp(attr->sched_nice, -20, 19);
++
++	/* sched/core.c uses zero here but we already know ret is zero */
++	return 0;
++
++err_size:
++	put_user(sizeof(*attr), &uattr->size);
++	return -E2BIG;
++}
++
++/**
++ * sys_sched_setscheduler - set/change the scheduler policy and RT priority
++ * @pid: the pid in question.
++ * @policy: new policy.
++ *
++ * Return: 0 on success. An error code otherwise.
++ * @param: structure containing the new RT priority.
++ */
++asmlinkage long sys_sched_setscheduler(pid_t pid, int policy,
++				       struct sched_param __user *param)
++{
++	/* negative values for policy are not valid */
++	if (policy < 0)
++		return -EINVAL;
++
++	return do_sched_setscheduler(pid, policy, param);
++}
++
++/*
++ * sched_setparam() passes in -1 for its policy, to let the functions
++ * it calls know not to change it.
++ */
++#define SETPARAM_POLICY	-1
++
++/**
++ * sys_sched_setparam - set/change the RT priority of a thread
++ * @pid: the pid in question.
++ * @param: structure containing the new RT priority.
++ *
++ * Return: 0 on success. An error code otherwise.
++ */
++SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
++{
++	return do_sched_setscheduler(pid, SETPARAM_POLICY, param);
++}
++
++/**
++ * sys_sched_setattr - same as above, but with extended sched_attr
++ * @pid: the pid in question.
++ * @uattr: structure containing the extended parameters.
++ */
++SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr,
++			       unsigned int, flags)
++{
++	struct sched_attr attr;
++	struct task_struct *p;
++	int retval;
++
++	if (!uattr || pid < 0 || flags)
++		return -EINVAL;
++
++	retval = sched_copy_attr(uattr, &attr);
++	if (retval)
++		return retval;
++
++	if ((int)attr.sched_policy < 0)
++		return -EINVAL;
++
++	rcu_read_lock();
++	retval = -ESRCH;
++	p = find_process_by_pid(pid);
++	if (p != NULL)
++		retval = sched_setattr(p, &attr);
++	rcu_read_unlock();
++
++	return retval;
++}
++
++/**
++ * sys_sched_getscheduler - get the policy (scheduling class) of a thread
++ * @pid: the pid in question.
++ *
++ * Return: On success, the policy of the thread. Otherwise, a negative error
++ * code.
++ */
++SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
++{
++	struct task_struct *p;
++	int retval = -EINVAL;
++
++	if (pid < 0)
++		goto out_nounlock;
++
++	retval = -ESRCH;
++	rcu_read_lock();
++	p = find_process_by_pid(pid);
++	if (p) {
++		retval = security_task_getscheduler(p);
++		if (!retval)
++			retval = p->policy;
++	}
++	rcu_read_unlock();
++
++out_nounlock:
++	return retval;
++}
++
++/**
++ * sys_sched_getscheduler - get the RT priority of a thread
++ * @pid: the pid in question.
++ * @param: structure containing the RT priority.
++ *
++ * Return: On success, 0 and the RT priority is in @param. Otherwise, an error
++ * code.
++ */
++SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
++{
++	struct sched_param lp = { .sched_priority = 0 };
++	struct task_struct *p;
++	int retval = -EINVAL;
++
++	if (!param || pid < 0)
++		goto out_nounlock;
++
++	rcu_read_lock();
++	p = find_process_by_pid(pid);
++	retval = -ESRCH;
++	if (!p)
++		goto out_unlock;
++
++	retval = security_task_getscheduler(p);
++	if (retval)
++		goto out_unlock;
++
++	if (has_rt_policy(p))
++		lp.sched_priority = p->rt_priority;
++	rcu_read_unlock();
++
++	/*
++	 * This one might sleep, we cannot do it with a spinlock held ...
++	 */
++	retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
++
++out_nounlock:
++	return retval;
++
++out_unlock:
++	rcu_read_unlock();
++	return retval;
++}
++
++static int sched_read_attr(struct sched_attr __user *uattr,
++			   struct sched_attr *attr,
++			   unsigned int usize)
++{
++	int ret;
++
++	if (!access_ok(VERIFY_WRITE, uattr, usize))
++		return -EFAULT;
++
++	/*
++	 * If we're handed a smaller struct than we know of,
++	 * ensure all the unknown bits are 0 - i.e. old
++	 * user-space does not get uncomplete information.
++	 */
++	if (usize < sizeof(*attr)) {
++		unsigned char *addr;
++		unsigned char *end;
++
++		addr = (void *)attr + usize;
++		end  = (void *)attr + sizeof(*attr);
++
++		for (; addr < end; addr++) {
++			if (*addr)
++				return -EFBIG;
++		}
++
++		attr->size = usize;
++	}
++
++	ret = copy_to_user(uattr, attr, attr->size);
++	if (ret)
++		return -EFAULT;
++
++	/* sched/core.c uses zero here but we already know ret is zero */
++	return ret;
++}
++
++/**
++ * sys_sched_getattr - similar to sched_getparam, but with sched_attr
++ * @pid: the pid in question.
++ * @uattr: structure containing the extended parameters.
++ * @size: sizeof(attr) for fwd/bwd comp.
++ * @flags: for future extension.
++ */
++SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
++		unsigned int, size, unsigned int, flags)
++{
++	struct sched_attr attr = {
++		.size = sizeof(struct sched_attr),
++	};
++	struct task_struct *p;
++	int retval;
++
++	if (!uattr || pid < 0 || size > PAGE_SIZE ||
++	    size < SCHED_ATTR_SIZE_VER0 || flags)
++		return -EINVAL;
++
++	rcu_read_lock();
++	p = find_process_by_pid(pid);
++	retval = -ESRCH;
++	if (!p)
++		goto out_unlock;
++
++	retval = security_task_getscheduler(p);
++	if (retval)
++		goto out_unlock;
++
++	attr.sched_policy = p->policy;
++	if (rt_task(p))
++		attr.sched_priority = p->rt_priority;
++	else
++		attr.sched_nice = task_nice(p);
++
++	rcu_read_unlock();
++
++	retval = sched_read_attr(uattr, &attr, size);
++	return retval;
++
++out_unlock:
++	rcu_read_unlock();
++	return retval;
++}
++
++long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
++{
++	cpumask_var_t cpus_allowed, new_mask;
++	struct task_struct *p;
++	int retval;
++
++	get_online_cpus();
++	rcu_read_lock();
++
++	p = find_process_by_pid(pid);
++	if (!p) {
++		rcu_read_unlock();
++		put_online_cpus();
++		return -ESRCH;
++	}
++
++	/* Prevent p going away */
++	get_task_struct(p);
++	rcu_read_unlock();
++
++	if (p->flags & PF_NO_SETAFFINITY) {
++		retval = -EINVAL;
++		goto out_put_task;
++	}
++	if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
++		retval = -ENOMEM;
++		goto out_put_task;
++	}
++	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
++		retval = -ENOMEM;
++		goto out_free_cpus_allowed;
++	}
++	retval = -EPERM;
++	if (!check_same_owner(p)) {
++		rcu_read_lock();
++		if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) {
++			rcu_read_unlock();
++			goto out_unlock;
++		}
++		rcu_read_unlock();
++	}
++
++	retval = security_task_setscheduler(p);
++	if (retval)
++		goto out_unlock;
++
++	cpuset_cpus_allowed(p, cpus_allowed);
++	cpumask_and(new_mask, in_mask, cpus_allowed);
++again:
++	retval = __set_cpus_allowed_ptr(p, new_mask, true);
++
++	if (!retval) {
++		cpuset_cpus_allowed(p, cpus_allowed);
++		if (!cpumask_subset(new_mask, cpus_allowed)) {
++			/*
++			 * We must have raced with a concurrent cpuset
++			 * update. Just reset the cpus_allowed to the
++			 * cpuset's cpus_allowed
++			 */
++			cpumask_copy(new_mask, cpus_allowed);
++			goto again;
++		}
++	}
++out_unlock:
++	free_cpumask_var(new_mask);
++out_free_cpus_allowed:
++	free_cpumask_var(cpus_allowed);
++out_put_task:
++	put_task_struct(p);
++	put_online_cpus();
++	return retval;
++}
++
++static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
++			     cpumask_t *new_mask)
++{
++	if (len < sizeof(cpumask_t)) {
++		memset(new_mask, 0, sizeof(cpumask_t));
++	} else if (len > sizeof(cpumask_t)) {
++		len = sizeof(cpumask_t);
++	}
++	return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
++}
++
++
++/**
++ * sys_sched_setaffinity - set the cpu affinity of a process
++ * @pid: pid of the process
++ * @len: length in bytes of the bitmask pointed to by user_mask_ptr
++ * @user_mask_ptr: user-space pointer to the new cpu mask
++ *
++ * Return: 0 on success. An error code otherwise.
++ */
++SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
++		unsigned long __user *, user_mask_ptr)
++{
++	cpumask_var_t new_mask;
++	int retval;
++
++	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
++		return -ENOMEM;
++
++	retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
++	if (retval == 0)
++		retval = sched_setaffinity(pid, new_mask);
++	free_cpumask_var(new_mask);
++	return retval;
++}
++
++long sched_getaffinity(pid_t pid, cpumask_t *mask)
++{
++	struct task_struct *p;
++	unsigned long flags;
++	int retval;
++
++	get_online_cpus();
++	rcu_read_lock();
++
++	retval = -ESRCH;
++	p = find_process_by_pid(pid);
++	if (!p)
++		goto out_unlock;
++
++	retval = security_task_getscheduler(p);
++	if (retval)
++		goto out_unlock;
++
++	grq_lock_irqsave(&flags);
++	cpumask_and(mask, tsk_cpus_allowed(p), cpu_active_mask);
++	grq_unlock_irqrestore(&flags);
++
++out_unlock:
++	rcu_read_unlock();
++	put_online_cpus();
++
++	return retval;
++}
++
++/**
++ * sys_sched_getaffinity - get the cpu affinity of a process
++ * @pid: pid of the process
++ * @len: length in bytes of the bitmask pointed to by user_mask_ptr
++ * @user_mask_ptr: user-space pointer to hold the current cpu mask
++ *
++ * Return: 0 on success. An error code otherwise.
++ */
++SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
++		unsigned long __user *, user_mask_ptr)
++{
++	int ret;
++	cpumask_var_t mask;
++
++	if ((len * BITS_PER_BYTE) < nr_cpu_ids)
++		return -EINVAL;
++	if (len & (sizeof(unsigned long)-1))
++		return -EINVAL;
++
++	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
++		return -ENOMEM;
++
++	ret = sched_getaffinity(pid, mask);
++	if (ret == 0) {
++		size_t retlen = min_t(size_t, len, cpumask_size());
++
++		if (copy_to_user(user_mask_ptr, mask, retlen))
++			ret = -EFAULT;
++		else
++			ret = retlen;
++	}
++	free_cpumask_var(mask);
++
++	return ret;
++}
++
++/**
++ * sys_sched_yield - yield the current processor to other threads.
++ *
++ * This function yields the current CPU to other tasks. It does this by
++ * scheduling away the current task. If it still has the earliest deadline
++ * it will be scheduled again as the next task.
++ *
++ * Return: 0.
++ */
++SYSCALL_DEFINE0(sched_yield)
++{
++	struct task_struct *p;
++
++	p = current;
++	grq_lock_irq();
++	schedstat_inc(task_rq(p), yld_count);
++	requeue_task(p);
++
++	/*
++	 * Since we are going to call schedule() anyway, there's
++	 * no need to preempt or enable interrupts:
++	 */
++	__release(grq.lock);
++	spin_release(&grq.lock.dep_map, 1, _THIS_IP_);
++	do_raw_spin_unlock(&grq.lock);
++	sched_preempt_enable_no_resched();
++
++	schedule();
++
++	return 0;
++}
++
++int __sched _cond_resched(void)
++{
++	if (should_resched(0)) {
++		preempt_schedule_common();
++		return 1;
++	}
++	return 0;
++}
++EXPORT_SYMBOL(_cond_resched);
++
++/*
++ * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
++ * call schedule, and on return reacquire the lock.
++ *
++ * This works OK both with and without CONFIG_PREEMPT.  We do strange low-level
++ * operations here to prevent schedule() from being called twice (once via
++ * spin_unlock(), once by hand).
++ */
++int __cond_resched_lock(spinlock_t *lock)
++{
++	int resched = should_resched(PREEMPT_LOCK_OFFSET);
++	int ret = 0;
++
++	lockdep_assert_held(lock);
++
++	if (spin_needbreak(lock) || resched) {
++		spin_unlock(lock);
++		if (resched)
++			preempt_schedule_common();
++		else
++			cpu_relax();
++		ret = 1;
++		spin_lock(lock);
++	}
++	return ret;
++}
++EXPORT_SYMBOL(__cond_resched_lock);
++
++int __sched __cond_resched_softirq(void)
++{
++	BUG_ON(!in_softirq());
++
++	if (should_resched(SOFTIRQ_DISABLE_OFFSET)) {
++		local_bh_enable();
++		preempt_schedule_common();
++		local_bh_disable();
++		return 1;
++	}
++	return 0;
++}
++EXPORT_SYMBOL(__cond_resched_softirq);
++
++/**
++ * yield - yield the current processor to other threads.
++ *
++ * Do not ever use this function, there's a 99% chance you're doing it wrong.
++ *
++ * The scheduler is at all times free to pick the calling task as the most
++ * eligible task to run, if removing the yield() call from your code breaks
++ * it, its already broken.
++ *
++ * Typical broken usage is:
++ *
++ * while (!event)
++ * 	yield();
++ *
++ * where one assumes that yield() will let 'the other' process run that will
++ * make event true. If the current task is a SCHED_FIFO task that will never
++ * happen. Never use yield() as a progress guarantee!!
++ *
++ * If you want to use yield() to wait for something, use wait_event().
++ * If you want to use yield() to be 'nice' for others, use cond_resched().
++ * If you still want to use yield(), do not!
++ */
++void __sched yield(void)
++{
++	set_current_state(TASK_RUNNING);
++	sys_sched_yield();
++}
++EXPORT_SYMBOL(yield);
++
++/**
++ * yield_to - yield the current processor to another thread in
++ * your thread group, or accelerate that thread toward the
++ * processor it's on.
++ * @p: target task
++ * @preempt: whether task preemption is allowed or not
++ *
++ * It's the caller's job to ensure that the target task struct
++ * can't go away on us before we can do any checks.
++ *
++ * Return:
++ *	true (>0) if we indeed boosted the target task.
++ *	false (0) if we failed to boost the target.
++ *	-ESRCH if there's no task to yield to.
++ */
++int __sched yield_to(struct task_struct *p, bool preempt)
++{
++	struct rq *rq, *p_rq;
++	unsigned long flags;
++	int yielded = 0;
++
++	rq = this_rq();
++	grq_lock_irqsave(&flags);
++	if (task_running(p) || p->state) {
++		yielded = -ESRCH;
++		goto out_unlock;
++	}
++
++	p_rq = task_rq(p);
++	yielded = 1;
++	if (p->deadline > rq->rq_deadline)
++		p->deadline = rq->rq_deadline;
++	p->time_slice += rq->rq_time_slice;
++	rq->rq_time_slice = 0;
++	if (p->time_slice > timeslice())
++		p->time_slice = timeslice();
++	if (preempt && rq != p_rq)
++		resched_curr(p_rq);
++out_unlock:
++	grq_unlock_irqrestore(&flags);
++
++	if (yielded > 0)
++		schedule();
++	return yielded;
++}
++EXPORT_SYMBOL_GPL(yield_to);
++
++/*
++ * This task is about to go to sleep on IO.  Increment rq->nr_iowait so
++ * that process accounting knows that this is a task in IO wait state.
++ *
++ * But don't do that if it is a deliberate, throttling IO wait (this task
++ * has set its backing_dev_info: the queue against which it should throttle)
++ */
++
++long __sched io_schedule_timeout(long timeout)
++{
++	int old_iowait = current->in_iowait;
++	struct rq *rq;
++	long ret;
++
++	current->in_iowait = 1;
++	blk_schedule_flush_plug(current);
++
++	delayacct_blkio_start();
++	rq = raw_rq();
++	atomic_inc(&rq->nr_iowait);
++	ret = schedule_timeout(timeout);
++	current->in_iowait = old_iowait;
++	atomic_dec(&rq->nr_iowait);
++	delayacct_blkio_end();
++
++	return ret;
++}
++EXPORT_SYMBOL(io_schedule_timeout);
++
++/**
++ * sys_sched_get_priority_max - return maximum RT priority.
++ * @policy: scheduling class.
++ *
++ * Return: On success, this syscall returns the maximum
++ * rt_priority that can be used by a given scheduling class.
++ * On failure, a negative error code is returned.
++ */
++SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
++{
++	int ret = -EINVAL;
++
++	switch (policy) {
++	case SCHED_FIFO:
++	case SCHED_RR:
++		ret = MAX_USER_RT_PRIO-1;
++		break;
++	case SCHED_NORMAL:
++	case SCHED_BATCH:
++	case SCHED_ISO:
++	case SCHED_IDLEPRIO:
++		ret = 0;
++		break;
++	}
++	return ret;
++}
++
++/**
++ * sys_sched_get_priority_min - return minimum RT priority.
++ * @policy: scheduling class.
++ *
++ * Return: On success, this syscall returns the minimum
++ * rt_priority that can be used by a given scheduling class.
++ * On failure, a negative error code is returned.
++ */
++SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
++{
++	int ret = -EINVAL;
++
++	switch (policy) {
++	case SCHED_FIFO:
++	case SCHED_RR:
++		ret = 1;
++		break;
++	case SCHED_NORMAL:
++	case SCHED_BATCH:
++	case SCHED_ISO:
++	case SCHED_IDLEPRIO:
++		ret = 0;
++		break;
++	}
++	return ret;
++}
++
++/**
++ * sys_sched_rr_get_interval - return the default timeslice of a process.
++ * @pid: pid of the process.
++ * @interval: userspace pointer to the timeslice value.
++ *
++ *
++ * Return: On success, 0 and the timeslice is in @interval. Otherwise,
++ * an error code.
++ */
++SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
++		struct timespec __user *, interval)
++{
++	struct task_struct *p;
++	unsigned int time_slice;
++	unsigned long flags;
++	int retval;
++	struct timespec t;
++
++	if (pid < 0)
++		return -EINVAL;
++
++	retval = -ESRCH;
++	rcu_read_lock();
++	p = find_process_by_pid(pid);
++	if (!p)
++		goto out_unlock;
++
++	retval = security_task_getscheduler(p);
++	if (retval)
++		goto out_unlock;
++
++	grq_lock_irqsave(&flags);
++	time_slice = p->policy == SCHED_FIFO ? 0 : MS_TO_NS(task_timeslice(p));
++	grq_unlock_irqrestore(&flags);
++
++	rcu_read_unlock();
++	t = ns_to_timespec(time_slice);
++	retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
++	return retval;
++
++out_unlock:
++	rcu_read_unlock();
++	return retval;
++}
++
++static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
++
++void sched_show_task(struct task_struct *p)
++{
++	unsigned long free = 0;
++	int ppid;
++	unsigned long state = p->state;
++
++	if (state)
++		state = __ffs(state) + 1;
++	printk(KERN_INFO "%-15.15s %c", p->comm,
++		state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
++#if BITS_PER_LONG == 32
++	if (state == TASK_RUNNING)
++		printk(KERN_CONT " running  ");
++	else
++		printk(KERN_CONT " %08lx ", thread_saved_pc(p));
++#else
++	if (state == TASK_RUNNING)
++		printk(KERN_CONT "  running task    ");
++	else
++		printk(KERN_CONT " %016lx ", thread_saved_pc(p));
++#endif
++#ifdef CONFIG_DEBUG_STACK_USAGE
++	free = stack_not_used(p);
++#endif
++	ppid = 0;
++	rcu_read_lock();
++	if (pid_alive(p))
++		ppid = task_pid_nr(rcu_dereference(p->real_parent));
++	rcu_read_unlock();
++	printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
++		task_pid_nr(p), ppid,
++		(unsigned long)task_thread_info(p)->flags);
++
++	print_worker_info(KERN_INFO, p);
++	show_stack(p, NULL);
++}
++
++void show_state_filter(unsigned long state_filter)
++{
++	struct task_struct *g, *p;
++
++#if BITS_PER_LONG == 32
++	printk(KERN_INFO
++		"  task                PC stack   pid father\n");
++#else
++	printk(KERN_INFO
++		"  task                        PC stack   pid father\n");
++#endif
++	rcu_read_lock();
++	for_each_process_thread(g, p) {
++		/*
++		 * reset the NMI-timeout, listing all files on a slow
++		 * console might take a lot of time:
++		 */
++		touch_nmi_watchdog();
++		if (!state_filter || (p->state & state_filter))
++			sched_show_task(p);
++	}
++
++	touch_all_softlockup_watchdogs();
++
++	rcu_read_unlock();
++	/*
++	 * Only show locks if all tasks are dumped:
++	 */
++	if (!state_filter)
++		debug_show_all_locks();
++}
++
++void dump_cpu_task(int cpu)
++{
++	pr_info("Task dump for CPU %d:\n", cpu);
++	sched_show_task(cpu_curr(cpu));
++}
++
++#ifdef CONFIG_SMP
++void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask)
++{
++	cpumask_copy(&p->cpus_allowed, new_mask);
++	p->nr_cpus_allowed = cpumask_weight(new_mask);
++}
++
++void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
++{
++	cpumask_copy(tsk_cpus_allowed(p), new_mask);
++}
++#endif
++
++/**
++ * init_idle - set up an idle thread for a given CPU
++ * @idle: task in question
++ * @cpu: cpu the idle task belongs to
++ *
++ * NOTE: this function does not set the idle thread's NEED_RESCHED
++ * flag, to make booting more robust.
++ */
++void init_idle(struct task_struct *idle, int cpu)
++{
++	struct rq *rq = cpu_rq(cpu);
++	unsigned long flags;
++
++	raw_spin_lock_irqsave(&idle->pi_lock, flags);
++	time_lock_grq(rq);
++	idle->last_ran = rq->clock_task;
++	idle->state = TASK_RUNNING;
++	/* Setting prio to illegal value shouldn't matter when never queued */
++	idle->prio = PRIO_LIMIT;
++#ifdef CONFIG_SMT_NICE
++	idle->smt_bias = 0;
++#endif
++	set_rq_task(rq, idle);
++	do_set_cpus_allowed(idle, get_cpu_mask(cpu));
++	/* Silence PROVE_RCU */
++	rcu_read_lock();
++	set_task_cpu(idle, cpu);
++	rcu_read_unlock();
++	rq->curr = rq->idle = idle;
++	idle->on_cpu = 1;
++	grq_unlock();
++	raw_spin_unlock_irqrestore(&idle->pi_lock, flags);
++
++	/* Set the preempt count _outside_ the spinlocks! */
++	init_idle_preempt_count(idle, cpu);
++
++	ftrace_graph_init_idle_task(idle, cpu);
++#ifdef CONFIG_SMP
++	sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
++#endif
++}
++
++int cpuset_cpumask_can_shrink(const struct cpumask __maybe_unused *cur,
++			      const struct cpumask __maybe_unused *trial)
++{
++	return 1;
++}
++
++int task_can_attach(struct task_struct *p,
++		    const struct cpumask *cs_cpus_allowed)
++{
++	int ret = 0;
++
++	/*
++	 * Kthreads which disallow setaffinity shouldn't be moved
++	 * to a new cpuset; we don't want to change their cpu
++	 * affinity and isolating such threads by their set of
++	 * allowed nodes is unnecessary.  Thus, cpusets are not
++	 * applicable for such threads.  This prevents checking for
++	 * success of set_cpus_allowed_ptr() on all attached tasks
++	 * before cpus_allowed may be changed.
++	 */
++	if (p->flags & PF_NO_SETAFFINITY)
++		ret = -EINVAL;
++
++	return ret;
++}
++
++void wake_q_add(struct wake_q_head *head, struct task_struct *task)
++{
++	struct wake_q_node *node = &task->wake_q;
++
++	/*
++	 * Atomically grab the task, if ->wake_q is !nil already it means
++	 * its already queued (either by us or someone else) and will get the
++	 * wakeup due to that.
++	 *
++	 * This cmpxchg() implies a full barrier, which pairs with the write
++	 * barrier implied by the wakeup in wake_up_list().
++	 */
++	if (cmpxchg(&node->next, NULL, WAKE_Q_TAIL))
++		return;
++
++	get_task_struct(task);
++
++	/*
++	 * The head is context local, there can be no concurrency.
++	 */
++	*head->lastp = node;
++	head->lastp = &node->next;
++}
++
++void wake_up_q(struct wake_q_head *head)
++{
++	struct wake_q_node *node = head->first;
++
++	while (node != WAKE_Q_TAIL) {
++		struct task_struct *task;
++
++		task = container_of(node, struct task_struct, wake_q);
++		BUG_ON(!task);
++		/* task can safely be re-inserted now */
++		node = node->next;
++		task->wake_q.next = NULL;
++
++		/*
++		 * wake_up_process() implies a wmb() to pair with the queueing
++		 * in wake_q_add() so as not to miss wakeups.
++		 */
++		wake_up_process(task);
++		put_task_struct(task);
++	}
++}
++
++void resched_cpu(int cpu)
++{
++	unsigned long flags;
++
++	grq_lock_irqsave(&flags);
++	resched_task(cpu_curr(cpu));
++	grq_unlock_irqrestore(&flags);
++}
++
++#ifdef CONFIG_SMP
++#ifdef CONFIG_NO_HZ_COMMON
++void nohz_balance_enter_idle(int cpu)
++{
++}
++
++void select_nohz_load_balancer(int stop_tick)
++{
++}
++
++void set_cpu_sd_state_idle(void) {}
++#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
++/**
++ * lowest_flag_domain - Return lowest sched_domain containing flag.
++ * @cpu:	The cpu whose lowest level of sched domain is to
++ *		be returned.
++ * @flag:	The flag to check for the lowest sched_domain
++ *		for the given cpu.
++ *
++ * Returns the lowest sched_domain of a cpu which contains the given flag.
++ */
++static inline struct sched_domain *lowest_flag_domain(int cpu, int flag)
++{
++	struct sched_domain *sd;
++
++	for_each_domain(cpu, sd)
++		if (sd && (sd->flags & flag))
++			break;
++
++	return sd;
++}
++
++/**
++ * for_each_flag_domain - Iterates over sched_domains containing the flag.
++ * @cpu:	The cpu whose domains we're iterating over.
++ * @sd:		variable holding the value of the power_savings_sd
++ *		for cpu.
++ * @flag:	The flag to filter the sched_domains to be iterated.
++ *
++ * Iterates over all the scheduler domains for a given cpu that has the 'flag'
++ * set, starting from the lowest sched_domain to the highest.
++ */
++#define for_each_flag_domain(cpu, sd, flag) \
++	for (sd = lowest_flag_domain(cpu, flag); \
++		(sd && (sd->flags & flag)); sd = sd->parent)
++
++#endif /*  (CONFIG_SCHED_MC || CONFIG_SCHED_SMT) */
++
++/*
++ * In the semi idle case, use the nearest busy cpu for migrating timers
++ * from an idle cpu.  This is good for power-savings.
++ *
++ * We don't do similar optimization for completely idle system, as
++ * selecting an idle cpu will add more delays to the timers than intended
++ * (as that cpu's timer base may not be uptodate wrt jiffies etc).
++ */
++int get_nohz_timer_target(void)
++{
++	int i, cpu = smp_processor_id();
++	struct sched_domain *sd;
++
++	if (!idle_cpu(cpu) && is_housekeeping_cpu(cpu))
++		return cpu;
++
++	rcu_read_lock();
++	for_each_domain(cpu, sd) {
++		for_each_cpu(i, sched_domain_span(sd)) {
++			if (!idle_cpu(i) && is_housekeeping_cpu(cpu)) {
++				cpu = i;
++				goto unlock;
++			}
++		}
++	}
++
++	if (!is_housekeeping_cpu(cpu))
++		cpu = housekeeping_any_cpu();
++unlock:
++	rcu_read_unlock();
++	return cpu;
++}
++
++/*
++ * When add_timer_on() enqueues a timer into the timer wheel of an
++ * idle CPU then this timer might expire before the next timer event
++ * which is scheduled to wake up that CPU. In case of a completely
++ * idle system the next event might even be infinite time into the
++ * future. wake_up_idle_cpu() ensures that the CPU is woken up and
++ * leaves the inner idle loop so the newly added timer is taken into
++ * account when the CPU goes back to idle and evaluates the timer
++ * wheel for the next timer event.
++ */
++void wake_up_idle_cpu(int cpu)
++{
++	if (cpu == smp_processor_id())
++		return;
++
++	set_tsk_need_resched(cpu_rq(cpu)->idle);
++	smp_send_reschedule(cpu);
++}
++
++void wake_up_nohz_cpu(int cpu)
++{
++	wake_up_idle_cpu(cpu);
++}
++#endif /* CONFIG_NO_HZ_COMMON */
++
++/*
++ * Change a given task's CPU affinity. Migrate the thread to a
++ * proper CPU and schedule it away if the CPU it's executing on
++ * is removed from the allowed bitmask.
++ *
++ * NOTE: the caller must have a valid reference to the task, the
++ * task must not exit() & deallocate itself prematurely. The
++ * call is not atomic; no spinlocks may be held.
++ */
++static int __set_cpus_allowed_ptr(struct task_struct *p,
++				  const struct cpumask *new_mask, bool check)
++{
++	bool running_wrong = false;
++	bool queued = false;
++	unsigned long flags;
++	struct rq *rq;
++	int ret = 0;
++
++	rq = task_grq_lock(p, &flags);
++
++	/*
++	 * Must re-check here, to close a race against __kthread_bind(),
++	 * sched_setaffinity() is not guaranteed to observe the flag.
++	 */
++	if (check && (p->flags & PF_NO_SETAFFINITY)) {
++		ret = -EINVAL;
++		goto out;
++	}
++
++	if (cpumask_equal(tsk_cpus_allowed(p), new_mask))
++		goto out;
++
++	if (!cpumask_intersects(new_mask, cpu_active_mask)) {
++		ret = -EINVAL;
++		goto out;
++	}
++
++	queued = task_queued(p);
++
++	do_set_cpus_allowed(p, new_mask);
++
++	/* Can the task run on the task's current CPU? If so, we're done */
++	if (cpumask_test_cpu(task_cpu(p), new_mask))
++		goto out;
++
++	if (task_running(p)) {
++		/* Task is running on the wrong cpu now, reschedule it. */
++		if (rq == this_rq()) {
++			set_tsk_need_resched(p);
++			running_wrong = true;
++		} else
++			resched_task(p);
++	} else
++		set_task_cpu(p, cpumask_any_and(cpu_active_mask, new_mask));
++
++out:
++	if (queued)
++		try_preempt(p, rq);
++	task_grq_unlock(&flags);
++
++	if (running_wrong)
++		preempt_schedule_common();
++
++	return ret;
++}
++
++int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
++{
++	return __set_cpus_allowed_ptr(p, new_mask, false);
++}
++EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
++
++#ifdef CONFIG_HOTPLUG_CPU
++/* Run through task list and find tasks affined to the dead cpu, then remove
++ * that cpu from the list, enable cpu0 and set the zerobound flag. */
++static void bind_zero(int src_cpu)
++{
++	struct task_struct *p, *t;
++	int bound = 0;
++
++	if (src_cpu == 0)
++		return;
++
++	do_each_thread(t, p) {
++		if (cpumask_test_cpu(src_cpu, tsk_cpus_allowed(p))) {
++			cpumask_clear_cpu(src_cpu, tsk_cpus_allowed(p));
++			cpumask_set_cpu(0, tsk_cpus_allowed(p));
++			p->zerobound = true;
++			bound++;
++		}
++		clear_sticky(p);
++	} while_each_thread(t, p);
++
++	if (bound) {
++		printk(KERN_INFO "Removed affinity for %d processes to cpu %d\n",
++		       bound, src_cpu);
++	}
++}
++
++/* Find processes with the zerobound flag and reenable their affinity for the
++ * CPU coming alive. */
++static void unbind_zero(int src_cpu)
++{
++	int unbound = 0, zerobound = 0;
++	struct task_struct *p, *t;
++
++	if (src_cpu == 0)
++		return;
++
++	do_each_thread(t, p) {
++		if (!p->mm)
++			p->zerobound = false;
++		if (p->zerobound) {
++			unbound++;
++			cpumask_set_cpu(src_cpu, tsk_cpus_allowed(p));
++			/* Once every CPU affinity has been re-enabled, remove
++			 * the zerobound flag */
++			if (cpumask_subset(cpu_possible_mask, tsk_cpus_allowed(p))) {
++				p->zerobound = false;
++				zerobound++;
++			}
++		}
++	} while_each_thread(t, p);
++
++	if (unbound) {
++		printk(KERN_INFO "Added affinity for %d processes to cpu %d\n",
++		       unbound, src_cpu);
++	}
++	if (zerobound) {
++		printk(KERN_INFO "Released forced binding to cpu0 for %d processes\n",
++		       zerobound);
++	}
++}
++
++/*
++ * Ensures that the idle task is using init_mm right before its cpu goes
++ * offline.
++ */
++void idle_task_exit(void)
++{
++	struct mm_struct *mm = current->active_mm;
++
++	BUG_ON(cpu_online(smp_processor_id()));
++
++	if (mm != &init_mm) {
++		switch_mm(mm, &init_mm, current);
++		finish_arch_post_lock_switch();
++	}
++	mmdrop(mm);
++}
++#else /* CONFIG_HOTPLUG_CPU */
++static void unbind_zero(int src_cpu) {}
++#endif /* CONFIG_HOTPLUG_CPU */
++
++void sched_set_stop_task(int cpu, struct task_struct *stop)
++{
++	struct sched_param stop_param = { .sched_priority = STOP_PRIO };
++	struct sched_param start_param = { .sched_priority = 0 };
++	struct task_struct *old_stop = cpu_rq(cpu)->stop;
++
++	if (stop) {
++		/*
++		 * Make it appear like a SCHED_FIFO task, its something
++		 * userspace knows about and won't get confused about.
++		 *
++		 * Also, it will make PI more or less work without too
++		 * much confusion -- but then, stop work should not
++		 * rely on PI working anyway.
++		 */
++		sched_setscheduler_nocheck(stop, SCHED_FIFO, &stop_param);
++	}
++
++	cpu_rq(cpu)->stop = stop;
++
++	if (old_stop) {
++		/*
++		 * Reset it back to a normal scheduling policy so that
++		 * it can die in pieces.
++		 */
++		sched_setscheduler_nocheck(old_stop, SCHED_NORMAL, &start_param);
++	}
++}
++
++
++#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)
++
++static struct ctl_table sd_ctl_dir[] = {
++	{
++		.procname	= "sched_domain",
++		.mode		= 0555,
++	},
++	{}
++};
++
++static struct ctl_table sd_ctl_root[] = {
++	{
++		.procname	= "kernel",
++		.mode		= 0555,
++		.child		= sd_ctl_dir,
++	},
++	{}
++};
++
++static struct ctl_table *sd_alloc_ctl_entry(int n)
++{
++	struct ctl_table *entry =
++		kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
++
++	return entry;
++}
++
++static void sd_free_ctl_entry(struct ctl_table **tablep)
++{
++	struct ctl_table *entry;
++
++	/*
++	 * In the intermediate directories, both the child directory and
++	 * procname are dynamically allocated and could fail but the mode
++	 * will always be set. In the lowest directory the names are
++	 * static strings and all have proc handlers.
++	 */
++	for (entry = *tablep; entry->mode; entry++) {
++		if (entry->child)
++			sd_free_ctl_entry(&entry->child);
++		if (entry->proc_handler == NULL)
++			kfree(entry->procname);
++	}
++
++	kfree(*tablep);
++	*tablep = NULL;
++}
++
++static void
++set_table_entry(struct ctl_table *entry,
++		const char *procname, void *data, int maxlen,
++		mode_t mode, proc_handler *proc_handler)
++{
++	entry->procname = procname;
++	entry->data = data;
++	entry->maxlen = maxlen;
++	entry->mode = mode;
++	entry->proc_handler = proc_handler;
++}
++
++static struct ctl_table *
++sd_alloc_ctl_domain_table(struct sched_domain *sd)
++{
++	struct ctl_table *table = sd_alloc_ctl_entry(14);
++
++	if (table == NULL)
++		return NULL;
++
++	set_table_entry(&table[0], "min_interval", &sd->min_interval,
++		sizeof(long), 0644, proc_doulongvec_minmax);
++	set_table_entry(&table[1], "max_interval", &sd->max_interval,
++		sizeof(long), 0644, proc_doulongvec_minmax);
++	set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
++		sizeof(int), 0644, proc_dointvec_minmax);
++	set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
++		sizeof(int), 0644, proc_dointvec_minmax);
++	set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
++		sizeof(int), 0644, proc_dointvec_minmax);
++	set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
++		sizeof(int), 0644, proc_dointvec_minmax);
++	set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
++		sizeof(int), 0644, proc_dointvec_minmax);
++	set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
++		sizeof(int), 0644, proc_dointvec_minmax);
++	set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
++		sizeof(int), 0644, proc_dointvec_minmax);
++	set_table_entry(&table[9], "cache_nice_tries",
++		&sd->cache_nice_tries,
++		sizeof(int), 0644, proc_dointvec_minmax);
++	set_table_entry(&table[10], "flags", &sd->flags,
++		sizeof(int), 0644, proc_dointvec_minmax);
++	set_table_entry(&table[11], "max_newidle_lb_cost",
++		&sd->max_newidle_lb_cost,
++		sizeof(long), 0644, proc_doulongvec_minmax);
++	set_table_entry(&table[12], "name", sd->name,
++		CORENAME_MAX_SIZE, 0444, proc_dostring);
++	/* &table[13] is terminator */
++
++	return table;
++}
++
++static struct ctl_table *sd_alloc_ctl_cpu_table(int cpu)
++{
++	struct ctl_table *entry, *table;
++	struct sched_domain *sd;
++	int domain_num = 0, i;
++	char buf[32];
++
++	for_each_domain(cpu, sd)
++		domain_num++;
++	entry = table = sd_alloc_ctl_entry(domain_num + 1);
++	if (table == NULL)
++		return NULL;
++
++	i = 0;
++	for_each_domain(cpu, sd) {
++		snprintf(buf, 32, "domain%d", i);
++		entry->procname = kstrdup(buf, GFP_KERNEL);
++		entry->mode = 0555;
++		entry->child = sd_alloc_ctl_domain_table(sd);
++		entry++;
++		i++;
++	}
++	return table;
++}
++
++static struct ctl_table_header *sd_sysctl_header;
++static void register_sched_domain_sysctl(void)
++{
++	int i, cpu_num = num_possible_cpus();
++	struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
++	char buf[32];
++
++	WARN_ON(sd_ctl_dir[0].child);
++	sd_ctl_dir[0].child = entry;
++
++	if (entry == NULL)
++		return;
++
++	for_each_possible_cpu(i) {
++		snprintf(buf, 32, "cpu%d", i);
++		entry->procname = kstrdup(buf, GFP_KERNEL);
++		entry->mode = 0555;
++		entry->child = sd_alloc_ctl_cpu_table(i);
++		entry++;
++	}
++
++	WARN_ON(sd_sysctl_header);
++	sd_sysctl_header = register_sysctl_table(sd_ctl_root);
++}
++
++/* may be called multiple times per register */
++static void unregister_sched_domain_sysctl(void)
++{
++	unregister_sysctl_table(sd_sysctl_header);
++	sd_sysctl_header = NULL;
++	if (sd_ctl_dir[0].child)
++		sd_free_ctl_entry(&sd_ctl_dir[0].child);
++}
++#else /* CONFIG_SCHED_DEBUG && CONFIG_SYSCTL */
++static void register_sched_domain_sysctl(void)
++{
++}
++static void unregister_sched_domain_sysctl(void)
++{
++}
++#endif /* CONFIG_SCHED_DEBUG && CONFIG_SYSCTL */
++
++static void set_rq_online(struct rq *rq)
++{
++	if (!rq->online) {
++		cpumask_set_cpu(cpu_of(rq), rq->rd->online);
++		rq->online = true;
++	}
++}
++
++static void set_rq_offline(struct rq *rq)
++{
++	if (rq->online) {
++		cpumask_clear_cpu(cpu_of(rq), rq->rd->online);
++		rq->online = false;
++	}
++}
++
++/*
++ * migration_call - callback that gets triggered when a CPU is added.
++ */
++static int
++migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
++{
++	int cpu = (long)hcpu;
++	unsigned long flags;
++	struct rq *rq = cpu_rq(cpu);
++#ifdef CONFIG_HOTPLUG_CPU
++	struct task_struct *idle = rq->idle;
++#endif
++
++	switch (action & ~CPU_TASKS_FROZEN) {
++	case CPU_STARTING:
++		return NOTIFY_OK;
++	case CPU_UP_PREPARE:
++		break;
++
++	case CPU_ONLINE:
++		/* Update our root-domain */
++		grq_lock_irqsave(&flags);
++		if (rq->rd) {
++			BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
++
++			set_rq_online(rq);
++		}
++		unbind_zero(cpu);
++		grq.noc = num_online_cpus();
++		grq_unlock_irqrestore(&flags);
++		break;
++
++#ifdef CONFIG_HOTPLUG_CPU
++	case CPU_DEAD:
++		grq_lock_irq();
++		set_rq_task(rq, idle);
++		update_clocks(rq);
++		grq_unlock_irq();
++		break;
++
++	case CPU_DYING:
++		/* Update our root-domain */
++		grq_lock_irqsave(&flags);
++		if (rq->rd) {
++			BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
++			set_rq_offline(rq);
++		}
++		bind_zero(cpu);
++		grq.noc = num_online_cpus();
++		grq_unlock_irqrestore(&flags);
++		break;
++#endif
++	}
++	return NOTIFY_OK;
++}
++
++/*
++ * Register at high priority so that task migration (migrate_all_tasks)
++ * happens before everything else.  This has to be lower priority than
++ * the notifier in the perf_counter subsystem, though.
++ */
++static struct notifier_block  migration_notifier = {
++	.notifier_call = migration_call,
++	.priority = CPU_PRI_MIGRATION,
++};
++
++static int sched_cpu_active(struct notifier_block *nfb,
++				      unsigned long action, void *hcpu)
++{
++	int cpu = (long)hcpu;
++
++	switch (action & ~CPU_TASKS_FROZEN) {
++	case CPU_STARTING:
++		return NOTIFY_OK;
++	case CPU_ONLINE:
++		/*
++		 * At this point a starting CPU has marked itself as online via
++		 * set_cpu_online(). But it might not yet have marked itself
++		 * as active, which is essential from here on.
++		 */
++		set_cpu_active(cpu, true);
++		stop_machine_unpark(cpu);
++		return NOTIFY_OK;
++
++	case CPU_DOWN_FAILED:
++		set_cpu_active(cpu, true);
++		return NOTIFY_OK;
++	default:
++		return NOTIFY_DONE;
++	}
++}
++
++static int sched_cpu_inactive(struct notifier_block *nfb,
++					unsigned long action, void *hcpu)
++{
++	switch (action & ~CPU_TASKS_FROZEN) {
++	case CPU_DOWN_PREPARE:
++		set_cpu_active((long)hcpu, false);
++		return NOTIFY_OK;
++	default:
++		return NOTIFY_DONE;
++	}
++}
++
++int __init migration_init(void)
++{
++	void *cpu = (void *)(long)smp_processor_id();
++	int err;
++
++	/* Initialise migration for the boot CPU */
++	err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
++	BUG_ON(err == NOTIFY_BAD);
++	migration_call(&migration_notifier, CPU_ONLINE, cpu);
++	register_cpu_notifier(&migration_notifier);
++
++	/* Register cpu active notifiers */
++	cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE);
++	cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE);
++
++	return 0;
++}
++early_initcall(migration_init);
++
++static cpumask_var_t sched_domains_tmpmask; /* sched_domains_mutex */
++
++#ifdef CONFIG_SCHED_DEBUG
++
++static __read_mostly int sched_debug_enabled;
++
++static int __init sched_debug_setup(char *str)
++{
++	sched_debug_enabled = 1;
++
++	return 0;
++}
++early_param("sched_debug", sched_debug_setup);
++
++static inline bool sched_debug(void)
++{
++	return sched_debug_enabled;
++}
++
++static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
++				  struct cpumask *groupmask)
++{
++	cpumask_clear(groupmask);
++
++	printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
++
++	if (!(sd->flags & SD_LOAD_BALANCE)) {
++		printk("does not load-balance\n");
++		if (sd->parent)
++			printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
++					" has parent");
++		return -1;
++	}
++
++	printk(KERN_CONT "span %*pbl level %s\n",
++	       cpumask_pr_args(sched_domain_span(sd)), sd->name);
++
++	if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
++		printk(KERN_ERR "ERROR: domain->span does not contain "
++				"CPU%d\n", cpu);
++	}
++
++	printk(KERN_CONT "\n");
++
++	if (!cpumask_equal(sched_domain_span(sd), groupmask))
++		printk(KERN_ERR "ERROR: groups don't span domain->span\n");
++
++	if (sd->parent &&
++	    !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
++		printk(KERN_ERR "ERROR: parent span is not a superset "
++			"of domain->span\n");
++	return 0;
++}
++
++static void sched_domain_debug(struct sched_domain *sd, int cpu)
++{
++	int level = 0;
++
++	if (!sched_debug_enabled)
++		return;
++
++	if (!sd) {
++		printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
++		return;
++	}
++
++	printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
++
++	for (;;) {
++		if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask))
++			break;
++		level++;
++		sd = sd->parent;
++		if (!sd)
++			break;
++	}
++}
++#else /* !CONFIG_SCHED_DEBUG */
++# define sched_domain_debug(sd, cpu) do { } while (0)
++static inline bool sched_debug(void)
++{
++	return false;
++}
++#endif /* CONFIG_SCHED_DEBUG */
++
++static int sd_degenerate(struct sched_domain *sd)
++{
++	if (cpumask_weight(sched_domain_span(sd)) == 1)
++		return 1;
++
++	/* Following flags don't use groups */
++	if (sd->flags & (SD_WAKE_AFFINE))
++		return 0;
++
++	return 1;
++}
++
++static int
++sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
++{
++	unsigned long cflags = sd->flags, pflags = parent->flags;
++
++	if (sd_degenerate(parent))
++		return 1;
++
++	if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
++		return 0;
++
++	if (~cflags & pflags)
++		return 0;
++
++	return 1;
++}
++
++static void free_rootdomain(struct rcu_head *rcu)
++{
++	struct root_domain *rd = container_of(rcu, struct root_domain, rcu);
++
++	cpupri_cleanup(&rd->cpupri);
++	free_cpumask_var(rd->rto_mask);
++	free_cpumask_var(rd->online);
++	free_cpumask_var(rd->span);
++	kfree(rd);
++}
++
++static void rq_attach_root(struct rq *rq, struct root_domain *rd)
++{
++	struct root_domain *old_rd = NULL;
++	unsigned long flags;
++
++	grq_lock_irqsave(&flags);
++
++	if (rq->rd) {
++		old_rd = rq->rd;
++
++		if (cpumask_test_cpu(rq->cpu, old_rd->online))
++			set_rq_offline(rq);
++
++		cpumask_clear_cpu(rq->cpu, old_rd->span);
++
++		/*
++		 * If we dont want to free the old_rd yet then
++		 * set old_rd to NULL to skip the freeing later
++		 * in this function:
++		 */
++		if (!atomic_dec_and_test(&old_rd->refcount))
++			old_rd = NULL;
++	}
++
++	atomic_inc(&rd->refcount);
++	rq->rd = rd;
++
++	cpumask_set_cpu(rq->cpu, rd->span);
++	if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
++		set_rq_online(rq);
++
++	grq_unlock_irqrestore(&flags);
++
++	if (old_rd)
++		call_rcu_sched(&old_rd->rcu, free_rootdomain);
++}
++
++static int init_rootdomain(struct root_domain *rd)
++{
++	memset(rd, 0, sizeof(*rd));
++
++	if (!zalloc_cpumask_var(&rd->span, GFP_KERNEL))
++		goto out;
++	if (!zalloc_cpumask_var(&rd->online, GFP_KERNEL))
++		goto free_span;
++	if (!zalloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
++		goto free_online;
++
++	if (cpupri_init(&rd->cpupri) != 0)
++		goto free_rto_mask;
++	return 0;
++
++free_rto_mask:
++	free_cpumask_var(rd->rto_mask);
++free_online:
++	free_cpumask_var(rd->online);
++free_span:
++	free_cpumask_var(rd->span);
++out:
++	return -ENOMEM;
++}
++
++static void init_defrootdomain(void)
++{
++	init_rootdomain(&def_root_domain);
++
++	atomic_set(&def_root_domain.refcount, 1);
++}
++
++static struct root_domain *alloc_rootdomain(void)
++{
++	struct root_domain *rd;
++
++	rd = kmalloc(sizeof(*rd), GFP_KERNEL);
++	if (!rd)
++		return NULL;
++
++	if (init_rootdomain(rd) != 0) {
++		kfree(rd);
++		return NULL;
++	}
++
++	return rd;
++}
++
++static void free_sched_domain(struct rcu_head *rcu)
++{
++	struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu);
++
++	kfree(sd);
++}
++
++static void destroy_sched_domain(struct sched_domain *sd, int cpu)
++{
++	call_rcu(&sd->rcu, free_sched_domain);
++}
++
++static void destroy_sched_domains(struct sched_domain *sd, int cpu)
++{
++	for (; sd; sd = sd->parent)
++		destroy_sched_domain(sd, cpu);
++}
++
++/*
++ * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
++ * hold the hotplug lock.
++ */
++static void
++cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
++{
++	struct rq *rq = cpu_rq(cpu);
++	struct sched_domain *tmp;
++
++	/* Remove the sched domains which do not contribute to scheduling. */
++	for (tmp = sd; tmp; ) {
++		struct sched_domain *parent = tmp->parent;
++		if (!parent)
++			break;
++
++		if (sd_parent_degenerate(tmp, parent)) {
++			tmp->parent = parent->parent;
++			if (parent->parent)
++				parent->parent->child = tmp;
++			/*
++			 * Transfer SD_PREFER_SIBLING down in case of a
++			 * degenerate parent; the spans match for this
++			 * so the property transfers.
++			 */
++			if (parent->flags & SD_PREFER_SIBLING)
++				tmp->flags |= SD_PREFER_SIBLING;
++			destroy_sched_domain(parent, cpu);
++		} else
++			tmp = tmp->parent;
++	}
++
++	if (sd && sd_degenerate(sd)) {
++		tmp = sd;
++		sd = sd->parent;
++		destroy_sched_domain(tmp, cpu);
++		if (sd)
++			sd->child = NULL;
++	}
++
++	sched_domain_debug(sd, cpu);
++
++	rq_attach_root(rq, rd);
++	tmp = rq->sd;
++	rcu_assign_pointer(rq->sd, sd);
++	destroy_sched_domains(tmp, cpu);
++}
++
++/* Setup the mask of cpus configured for isolated domains */
++static int __init isolated_cpu_setup(char *str)
++{
++	alloc_bootmem_cpumask_var(&cpu_isolated_map);
++	cpulist_parse(str, cpu_isolated_map);
++	return 1;
++}
++
++__setup("isolcpus=", isolated_cpu_setup);
++
++struct s_data {
++	struct sched_domain ** __percpu sd;
++	struct root_domain	*rd;
++};
++
++enum s_alloc {
++	sa_rootdomain,
++	sa_sd,
++	sa_sd_storage,
++	sa_none,
++};
++
++/*
++ * Initializers for schedule domains
++ * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
++ */
++
++static int default_relax_domain_level = -1;
++int sched_domain_level_max;
++
++static int __init setup_relax_domain_level(char *str)
++{
++	if (kstrtoint(str, 0, &default_relax_domain_level))
++		pr_warn("Unable to set relax_domain_level\n");
++
++	return 1;
++}
++__setup("relax_domain_level=", setup_relax_domain_level);
++
++static void set_domain_attribute(struct sched_domain *sd,
++				 struct sched_domain_attr *attr)
++{
++	int request;
++
++	if (!attr || attr->relax_domain_level < 0) {
++		if (default_relax_domain_level < 0)
++			return;
++		else
++			request = default_relax_domain_level;
++	} else
++		request = attr->relax_domain_level;
++	if (request < sd->level) {
++		/* turn off idle balance on this domain */
++		sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
++	} else {
++		/* turn on idle balance on this domain */
++		sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
++	}
++}
++
++static void __sdt_free(const struct cpumask *cpu_map);
++static int __sdt_alloc(const struct cpumask *cpu_map);
++
++static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
++				 const struct cpumask *cpu_map)
++{
++	switch (what) {
++	case sa_rootdomain:
++		if (!atomic_read(&d->rd->refcount))
++			free_rootdomain(&d->rd->rcu); /* fall through */
++	case sa_sd:
++		free_percpu(d->sd); /* fall through */
++	case sa_sd_storage:
++		__sdt_free(cpu_map); /* fall through */
++	case sa_none:
++		break;
++	}
++}
++
++static enum s_alloc __visit_domain_allocation_hell(struct s_data *d,
++						   const struct cpumask *cpu_map)
++{
++	memset(d, 0, sizeof(*d));
++
++	if (__sdt_alloc(cpu_map))
++		return sa_sd_storage;
++	d->sd = alloc_percpu(struct sched_domain *);
++	if (!d->sd)
++		return sa_sd_storage;
++	d->rd = alloc_rootdomain();
++	if (!d->rd)
++		return sa_sd;
++	return sa_rootdomain;
++}
++
++/*
++ * NULL the sd_data elements we've used to build the sched_domain
++ * structure so that the subsequent __free_domain_allocs()
++ * will not free the data we're using.
++ */
++static void claim_allocations(int cpu, struct sched_domain *sd)
++{
++	struct sd_data *sdd = sd->private;
++
++	WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd);
++	*per_cpu_ptr(sdd->sd, cpu) = NULL;
++}
++
++#ifdef CONFIG_NUMA
++static int sched_domains_numa_levels;
++static int *sched_domains_numa_distance;
++static struct cpumask ***sched_domains_numa_masks;
++static int sched_domains_curr_level;
++#endif
++
++/*
++ * SD_flags allowed in topology descriptions.
++ *
++ * SD_SHARE_CPUCAPACITY      - describes SMT topologies
++ * SD_SHARE_PKG_RESOURCES - describes shared caches
++ * SD_NUMA                - describes NUMA topologies
++ * SD_SHARE_POWERDOMAIN   - describes shared power domain
++ *
++ * Odd one out:
++ * SD_ASYM_PACKING        - describes SMT quirks
++ */
++#define TOPOLOGY_SD_FLAGS		\
++	(SD_SHARE_CPUCAPACITY |		\
++	 SD_SHARE_PKG_RESOURCES |	\
++	 SD_NUMA |			\
++	 SD_ASYM_PACKING |		\
++	 SD_SHARE_POWERDOMAIN)
++
++static struct sched_domain *
++sd_init(struct sched_domain_topology_level *tl, int cpu)
++{
++	struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu);
++	int sd_weight, sd_flags = 0;
++
++#ifdef CONFIG_NUMA
++	/*
++	 * Ugly hack to pass state to sd_numa_mask()...
++	 */
++	sched_domains_curr_level = tl->numa_level;
++#endif
++
++	sd_weight = cpumask_weight(tl->mask(cpu));
++
++	if (tl->sd_flags)
++		sd_flags = (*tl->sd_flags)();
++	if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS,
++			"wrong sd_flags in topology description\n"))
++		sd_flags &= ~TOPOLOGY_SD_FLAGS;
++
++	*sd = (struct sched_domain){
++		.min_interval		= sd_weight,
++		.max_interval		= 2*sd_weight,
++		.busy_factor		= 32,
++		.imbalance_pct		= 125,
++
++		.cache_nice_tries	= 0,
++		.busy_idx		= 0,
++		.idle_idx		= 0,
++		.newidle_idx		= 0,
++		.wake_idx		= 0,
++		.forkexec_idx		= 0,
++
++		.flags			= 1*SD_LOAD_BALANCE
++					| 1*SD_BALANCE_NEWIDLE
++					| 1*SD_BALANCE_EXEC
++					| 1*SD_BALANCE_FORK
++					| 0*SD_BALANCE_WAKE
++					| 1*SD_WAKE_AFFINE
++					| 0*SD_SHARE_CPUCAPACITY
++					| 0*SD_SHARE_PKG_RESOURCES
++					| 0*SD_SERIALIZE
++					| 0*SD_PREFER_SIBLING
++					| 0*SD_NUMA
++					| sd_flags
++					,
++
++		.last_balance		= jiffies,
++		.balance_interval	= sd_weight,
++		.smt_gain		= 0,
++		.max_newidle_lb_cost	= 0,
++		.next_decay_max_lb_cost	= jiffies,
++#ifdef CONFIG_SCHED_DEBUG
++		.name			= tl->name,
++#endif
++	};
++
++	/*
++	 * Convert topological properties into behaviour.
++	 */
++
++	if (sd->flags & SD_SHARE_CPUCAPACITY) {
++		sd->flags |= SD_PREFER_SIBLING;
++		sd->imbalance_pct = 110;
++		sd->smt_gain = 1178; /* ~15% */
++
++	} else if (sd->flags & SD_SHARE_PKG_RESOURCES) {
++		sd->imbalance_pct = 117;
++		sd->cache_nice_tries = 1;
++		sd->busy_idx = 2;
++
++#ifdef CONFIG_NUMA
++	} else if (sd->flags & SD_NUMA) {
++		sd->cache_nice_tries = 2;
++		sd->busy_idx = 3;
++		sd->idle_idx = 2;
++
++		sd->flags |= SD_SERIALIZE;
++		if (sched_domains_numa_distance[tl->numa_level] > RECLAIM_DISTANCE) {
++			sd->flags &= ~(SD_BALANCE_EXEC |
++				       SD_BALANCE_FORK |
++				       SD_WAKE_AFFINE);
++		}
++
++#endif
++	} else {
++		sd->flags |= SD_PREFER_SIBLING;
++		sd->cache_nice_tries = 1;
++		sd->busy_idx = 2;
++		sd->idle_idx = 1;
++	}
++
++	sd->private = &tl->data;
++
++	return sd;
++}
++
++/*
++ * Topology list, bottom-up.
++ */
++static struct sched_domain_topology_level default_topology[] = {
++#ifdef CONFIG_SCHED_SMT
++	{ cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
++#endif
++#ifdef CONFIG_SCHED_MC
++	{ cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
++#endif
++	{ cpu_cpu_mask, SD_INIT_NAME(DIE) },
++	{ NULL, },
++};
++
++static struct sched_domain_topology_level *sched_domain_topology =
++	default_topology;
++
++#define for_each_sd_topology(tl)			\
++	for (tl = sched_domain_topology; tl->mask; tl++)
++
++void set_sched_topology(struct sched_domain_topology_level *tl)
++{
++	sched_domain_topology = tl;
++}
++
++#ifdef CONFIG_NUMA
++
++static const struct cpumask *sd_numa_mask(int cpu)
++{
++	return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)];
++}
++
++static void sched_numa_warn(const char *str)
++{
++	static int done = false;
++	int i,j;
++
++	if (done)
++		return;
++
++	done = true;
++
++	printk(KERN_WARNING "ERROR: %s\n\n", str);
++
++	for (i = 0; i < nr_node_ids; i++) {
++		printk(KERN_WARNING "  ");
++		for (j = 0; j < nr_node_ids; j++)
++			printk(KERN_CONT "%02d ", node_distance(i,j));
++		printk(KERN_CONT "\n");
++	}
++	printk(KERN_WARNING "\n");
++}
++
++static bool find_numa_distance(int distance)
++{
++	int i;
++
++	if (distance == node_distance(0, 0))
++		return true;
++
++	for (i = 0; i < sched_domains_numa_levels; i++) {
++		if (sched_domains_numa_distance[i] == distance)
++			return true;
++	}
++
++	return false;
++}
++
++static void sched_init_numa(void)
++{
++	int next_distance, curr_distance = node_distance(0, 0);
++	struct sched_domain_topology_level *tl;
++	int level = 0;
++	int i, j, k;
++
++	sched_domains_numa_distance = kzalloc(sizeof(int) * nr_node_ids, GFP_KERNEL);
++	if (!sched_domains_numa_distance)
++		return;
++
++	/*
++	 * O(nr_nodes^2) deduplicating selection sort -- in order to find the
++	 * unique distances in the node_distance() table.
++	 *
++	 * Assumes node_distance(0,j) includes all distances in
++	 * node_distance(i,j) in order to avoid cubic time.
++	 */
++	next_distance = curr_distance;
++	for (i = 0; i < nr_node_ids; i++) {
++		for (j = 0; j < nr_node_ids; j++) {
++			for (k = 0; k < nr_node_ids; k++) {
++				int distance = node_distance(i, k);
++
++				if (distance > curr_distance &&
++				    (distance < next_distance ||
++				     next_distance == curr_distance))
++					next_distance = distance;
++
++				/*
++				 * While not a strong assumption it would be nice to know
++				 * about cases where if node A is connected to B, B is not
++				 * equally connected to A.
++				 */
++				if (sched_debug() && node_distance(k, i) != distance)
++					sched_numa_warn("Node-distance not symmetric");
++
++				if (sched_debug() && i && !find_numa_distance(distance))
++					sched_numa_warn("Node-0 not representative");
++			}
++			if (next_distance != curr_distance) {
++				sched_domains_numa_distance[level++] = next_distance;
++				sched_domains_numa_levels = level;
++				curr_distance = next_distance;
++			} else break;
++		}
++
++		/*
++		 * In case of sched_debug() we verify the above assumption.
++		 */
++		if (!sched_debug())
++			break;
++	}
++	/*
++	 * 'level' contains the number of unique distances, excluding the
++	 * identity distance node_distance(i,i).
++	 *
++	 * The sched_domains_numa_distance[] array includes the actual distance
++	 * numbers.
++	 */
++
++	/*
++	 * Here, we should temporarily reset sched_domains_numa_levels to 0.
++	 * If it fails to allocate memory for array sched_domains_numa_masks[][],
++	 * the array will contain less then 'level' members. This could be
++	 * dangerous when we use it to iterate array sched_domains_numa_masks[][]
++	 * in other functions.
++	 *
++	 * We reset it to 'level' at the end of this function.
++	 */
++	sched_domains_numa_levels = 0;
++
++	sched_domains_numa_masks = kzalloc(sizeof(void *) * level, GFP_KERNEL);
++	if (!sched_domains_numa_masks)
++		return;
++
++	/*
++	 * Now for each level, construct a mask per node which contains all
++	 * cpus of nodes that are that many hops away from us.
++	 */
++	for (i = 0; i < level; i++) {
++		sched_domains_numa_masks[i] =
++			kzalloc(nr_node_ids * sizeof(void *), GFP_KERNEL);
++		if (!sched_domains_numa_masks[i])
++			return;
++
++		for (j = 0; j < nr_node_ids; j++) {
++			struct cpumask *mask = kzalloc(cpumask_size(), GFP_KERNEL);
++			if (!mask)
++				return;
++
++			sched_domains_numa_masks[i][j] = mask;
++
++			for (k = 0; k < nr_node_ids; k++) {
++				if (node_distance(j, k) > sched_domains_numa_distance[i])
++					continue;
++
++				cpumask_or(mask, mask, cpumask_of_node(k));
++			}
++		}
++	}
++
++	/* Compute default topology size */
++	for (i = 0; sched_domain_topology[i].mask; i++);
++
++	tl = kzalloc((i + level + 1) *
++			sizeof(struct sched_domain_topology_level), GFP_KERNEL);
++	if (!tl)
++		return;
++
++	/*
++	 * Copy the default topology bits..
++	 */
++	for (i = 0; sched_domain_topology[i].mask; i++)
++		tl[i] = sched_domain_topology[i];
++
++	/*
++	 * .. and append 'j' levels of NUMA goodness.
++	 */
++	for (j = 0; j < level; i++, j++) {
++		tl[i] = (struct sched_domain_topology_level){
++			.mask = sd_numa_mask,
++			.sd_flags = cpu_numa_flags,
++			.flags = SDTL_OVERLAP,
++			.numa_level = j,
++			SD_INIT_NAME(NUMA)
++		};
++	}
++
++	sched_domain_topology = tl;
++
++	sched_domains_numa_levels = level;
++}
++
++static void sched_domains_numa_masks_set(int cpu)
++{
++	int i, j;
++	int node = cpu_to_node(cpu);
++
++	for (i = 0; i < sched_domains_numa_levels; i++) {
++		for (j = 0; j < nr_node_ids; j++) {
++			if (node_distance(j, node) <= sched_domains_numa_distance[i])
++				cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]);
++		}
++	}
++}
++
++static void sched_domains_numa_masks_clear(int cpu)
++{
++	int i, j;
++	for (i = 0; i < sched_domains_numa_levels; i++) {
++		for (j = 0; j < nr_node_ids; j++)
++			cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]);
++	}
++}
++
++/*
++ * Update sched_domains_numa_masks[level][node] array when new cpus
++ * are onlined.
++ */
++static int sched_domains_numa_masks_update(struct notifier_block *nfb,
++					   unsigned long action,
++					   void *hcpu)
++{
++	int cpu = (long)hcpu;
++
++	switch (action & ~CPU_TASKS_FROZEN) {
++	case CPU_ONLINE:
++		sched_domains_numa_masks_set(cpu);
++		break;
++
++	case CPU_DEAD:
++		sched_domains_numa_masks_clear(cpu);
++		break;
++
++	default:
++		return NOTIFY_DONE;
++	}
++
++	return NOTIFY_OK;
++}
++#else
++static inline void sched_init_numa(void)
++{
++}
++
++static int sched_domains_numa_masks_update(struct notifier_block *nfb,
++					   unsigned long action,
++					   void *hcpu)
++{
++	return 0;
++}
++#endif /* CONFIG_NUMA */
++
++static int __sdt_alloc(const struct cpumask *cpu_map)
++{
++	struct sched_domain_topology_level *tl;
++	int j;
++
++	for_each_sd_topology(tl) {
++		struct sd_data *sdd = &tl->data;
++
++		sdd->sd = alloc_percpu(struct sched_domain *);
++		if (!sdd->sd)
++			return -ENOMEM;
++
++		for_each_cpu(j, cpu_map) {
++			struct sched_domain *sd;
++
++			sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(),
++					GFP_KERNEL, cpu_to_node(j));
++			if (!sd)
++				return -ENOMEM;
++
++			*per_cpu_ptr(sdd->sd, j) = sd;
++		}
++	}
++
++	return 0;
++}
++
++static void __sdt_free(const struct cpumask *cpu_map)
++{
++	struct sched_domain_topology_level *tl;
++	int j;
++
++	for_each_sd_topology(tl) {
++		struct sd_data *sdd = &tl->data;
++
++		for_each_cpu(j, cpu_map) {
++			struct sched_domain *sd;
++
++			if (sdd->sd) {
++				sd = *per_cpu_ptr(sdd->sd, j);
++				kfree(*per_cpu_ptr(sdd->sd, j));
++			}
++		}
++		free_percpu(sdd->sd);
++		sdd->sd = NULL;
++	}
++}
++
++struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,
++		const struct cpumask *cpu_map, struct sched_domain_attr *attr,
++		struct sched_domain *child, int cpu)
++{
++	struct sched_domain *sd = sd_init(tl, cpu);
++	if (!sd)
++		return child;
++
++	cpumask_and(sched_domain_span(sd), cpu_map, tl->mask(cpu));
++	if (child) {
++		sd->level = child->level + 1;
++		sched_domain_level_max = max(sched_domain_level_max, sd->level);
++		child->parent = sd;
++		sd->child = child;
++
++		if (!cpumask_subset(sched_domain_span(child),
++				    sched_domain_span(sd))) {
++			pr_err("BUG: arch topology borken\n");
++#ifdef CONFIG_SCHED_DEBUG
++			pr_err("     the %s domain not a subset of the %s domain\n",
++					child->name, sd->name);
++#endif
++			/* Fixup, ensure @sd has at least @child cpus. */
++			cpumask_or(sched_domain_span(sd),
++				   sched_domain_span(sd),
++				   sched_domain_span(child));
++		}
++
++	}
++	set_domain_attribute(sd, attr);
++
++	return sd;
++}
++
++/*
++ * Build sched domains for a given set of cpus and attach the sched domains
++ * to the individual cpus
++ */
++static int build_sched_domains(const struct cpumask *cpu_map,
++			       struct sched_domain_attr *attr)
++{
++	enum s_alloc alloc_state;
++	struct sched_domain *sd;
++	struct s_data d;
++	int i, ret = -ENOMEM;
++
++	alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
++	if (alloc_state != sa_rootdomain)
++		goto error;
++
++	/* Set up domains for cpus specified by the cpu_map. */
++	for_each_cpu(i, cpu_map) {
++		struct sched_domain_topology_level *tl;
++
++		sd = NULL;
++		for_each_sd_topology(tl) {
++			sd = build_sched_domain(tl, cpu_map, attr, sd, i);
++			if (tl == sched_domain_topology)
++				*per_cpu_ptr(d.sd, i) = sd;
++			if (tl->flags & SDTL_OVERLAP)
++				sd->flags |= SD_OVERLAP;
++			if (cpumask_equal(cpu_map, sched_domain_span(sd)))
++				break;
++		}
++	}
++
++	/* Calculate CPU capacity for physical packages and nodes */
++	for (i = nr_cpumask_bits-1; i >= 0; i--) {
++		if (!cpumask_test_cpu(i, cpu_map))
++			continue;
++
++		for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
++			claim_allocations(i, sd);
++		}
++	}
++
++	/* Attach the domains */
++	rcu_read_lock();
++	for_each_cpu(i, cpu_map) {
++		sd = *per_cpu_ptr(d.sd, i);
++		cpu_attach_domain(sd, d.rd, i);
++	}
++	rcu_read_unlock();
++
++	ret = 0;
++error:
++	__free_domain_allocs(&d, alloc_state, cpu_map);
++	return ret;
++}
++
++static cpumask_var_t *doms_cur;	/* current sched domains */
++static int ndoms_cur;		/* number of sched domains in 'doms_cur' */
++static struct sched_domain_attr *dattr_cur;
++				/* attribues of custom domains in 'doms_cur' */
++
++/*
++ * Special case: If a kmalloc of a doms_cur partition (array of
++ * cpumask) fails, then fallback to a single sched domain,
++ * as determined by the single cpumask fallback_doms.
++ */
++static cpumask_var_t fallback_doms;
++
++/*
++ * arch_update_cpu_topology lets virtualized architectures update the
++ * cpu core maps. It is supposed to return 1 if the topology changed
++ * or 0 if it stayed the same.
++ */
++int __weak arch_update_cpu_topology(void)
++{
++	return 0;
++}
++
++cpumask_var_t *alloc_sched_domains(unsigned int ndoms)
++{
++	int i;
++	cpumask_var_t *doms;
++
++	doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL);
++	if (!doms)
++		return NULL;
++	for (i = 0; i < ndoms; i++) {
++		if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) {
++			free_sched_domains(doms, i);
++			return NULL;
++		}
++	}
++	return doms;
++}
++
++void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
++{
++	unsigned int i;
++	for (i = 0; i < ndoms; i++)
++		free_cpumask_var(doms[i]);
++	kfree(doms);
++}
++
++/*
++ * Set up scheduler domains and groups. Callers must hold the hotplug lock.
++ * For now this just excludes isolated cpus, but could be used to
++ * exclude other special cases in the future.
++ */
++static int init_sched_domains(const struct cpumask *cpu_map)
++{
++	int err;
++
++	arch_update_cpu_topology();
++	ndoms_cur = 1;
++	doms_cur = alloc_sched_domains(ndoms_cur);
++	if (!doms_cur)
++		doms_cur = &fallback_doms;
++	cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map);
++	err = build_sched_domains(doms_cur[0], NULL);
++	register_sched_domain_sysctl();
++
++	return err;
++}
++
++/*
++ * Detach sched domains from a group of cpus specified in cpu_map
++ * These cpus will now be attached to the NULL domain
++ */
++static void detach_destroy_domains(const struct cpumask *cpu_map)
++{
++	int i;
++
++	rcu_read_lock();
++	for_each_cpu(i, cpu_map)
++		cpu_attach_domain(NULL, &def_root_domain, i);
++	rcu_read_unlock();
++}
++
++/* handle null as "default" */
++static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
++			struct sched_domain_attr *new, int idx_new)
++{
++	struct sched_domain_attr tmp;
++
++	/* fast path */
++	if (!new && !cur)
++		return 1;
++
++	tmp = SD_ATTR_INIT;
++	return !memcmp(cur ? (cur + idx_cur) : &tmp,
++			new ? (new + idx_new) : &tmp,
++			sizeof(struct sched_domain_attr));
++}
++
++/*
++ * Partition sched domains as specified by the 'ndoms_new'
++ * cpumasks in the array doms_new[] of cpumasks. This compares
++ * doms_new[] to the current sched domain partitioning, doms_cur[].
++ * It destroys each deleted domain and builds each new domain.
++ *
++ * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'.
++ * The masks don't intersect (don't overlap.) We should setup one
++ * sched domain for each mask. CPUs not in any of the cpumasks will
++ * not be load balanced. If the same cpumask appears both in the
++ * current 'doms_cur' domains and in the new 'doms_new', we can leave
++ * it as it is.
++ *
++ * The passed in 'doms_new' should be allocated using
++ * alloc_sched_domains.  This routine takes ownership of it and will
++ * free_sched_domains it when done with it. If the caller failed the
++ * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1,
++ * and partition_sched_domains() will fallback to the single partition
++ * 'fallback_doms', it also forces the domains to be rebuilt.
++ *
++ * If doms_new == NULL it will be replaced with cpu_online_mask.
++ * ndoms_new == 0 is a special case for destroying existing domains,
++ * and it will not create the default domain.
++ *
++ * Call with hotplug lock held
++ */
++void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
++			     struct sched_domain_attr *dattr_new)
++{
++	int i, j, n;
++	int new_topology;
++
++	mutex_lock(&sched_domains_mutex);
++
++	/* always unregister in case we don't destroy any domains */
++	unregister_sched_domain_sysctl();
++
++	/* Let architecture update cpu core mappings. */
++	new_topology = arch_update_cpu_topology();
++
++	n = doms_new ? ndoms_new : 0;
++
++	/* Destroy deleted domains */
++	for (i = 0; i < ndoms_cur; i++) {
++		for (j = 0; j < n && !new_topology; j++) {
++			if (cpumask_equal(doms_cur[i], doms_new[j])
++			    && dattrs_equal(dattr_cur, i, dattr_new, j))
++				goto match1;
++		}
++		/* no match - a current sched domain not in new doms_new[] */
++		detach_destroy_domains(doms_cur[i]);
++match1:
++		;
++	}
++
++	n = ndoms_cur;
++	if (doms_new == NULL) {
++		n = 0;
++		doms_new = &fallback_doms;
++		cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map);
++		WARN_ON_ONCE(dattr_new);
++	}
++
++	/* Build new domains */
++	for (i = 0; i < ndoms_new; i++) {
++		for (j = 0; j < n && !new_topology; j++) {
++			if (cpumask_equal(doms_new[i], doms_cur[j])
++			    && dattrs_equal(dattr_new, i, dattr_cur, j))
++				goto match2;
++		}
++		/* no match - add a new doms_new */
++		build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL);
++match2:
++		;
++	}
++
++	/* Remember the new sched domains */
++	if (doms_cur != &fallback_doms)
++		free_sched_domains(doms_cur, ndoms_cur);
++	kfree(dattr_cur);	/* kfree(NULL) is safe */
++	doms_cur = doms_new;
++	dattr_cur = dattr_new;
++	ndoms_cur = ndoms_new;
++
++	register_sched_domain_sysctl();
++
++	mutex_unlock(&sched_domains_mutex);
++}
++
++static int num_cpus_frozen;	/* used to mark begin/end of suspend/resume */
++
++/*
++ * Update cpusets according to cpu_active mask.  If cpusets are
++ * disabled, cpuset_update_active_cpus() becomes a simple wrapper
++ * around partition_sched_domains().
++ *
++ * If we come here as part of a suspend/resume, don't touch cpusets because we
++ * want to restore it back to its original state upon resume anyway.
++ */
++static int cpuset_cpu_active(struct notifier_block *nfb, unsigned long action,
++			     void *hcpu)
++{
++	switch (action) {
++	case CPU_ONLINE_FROZEN:
++	case CPU_DOWN_FAILED_FROZEN:
++
++		/*
++		 * num_cpus_frozen tracks how many CPUs are involved in suspend
++		 * resume sequence. As long as this is not the last online
++		 * operation in the resume sequence, just build a single sched
++		 * domain, ignoring cpusets.
++		 */
++		num_cpus_frozen--;
++		if (likely(num_cpus_frozen)) {
++			partition_sched_domains(1, NULL, NULL);
++			break;
++		}
++
++		/*
++		 * This is the last CPU online operation. So fall through and
++		 * restore the original sched domains by considering the
++		 * cpuset configurations.
++		 */
++
++	case CPU_ONLINE:
++		cpuset_update_active_cpus(true);
++		break;
++	default:
++		return NOTIFY_DONE;
++	}
++	return NOTIFY_OK;
++}
++
++static int cpuset_cpu_inactive(struct notifier_block *nfb, unsigned long action,
++			       void *hcpu)
++{
++	switch (action) {
++	case CPU_DOWN_PREPARE:
++		cpuset_update_active_cpus(false);
++		break;
++	case CPU_DOWN_PREPARE_FROZEN:
++		num_cpus_frozen++;
++		partition_sched_domains(1, NULL, NULL);
++		break;
++	default:
++		return NOTIFY_DONE;
++	}
++	return NOTIFY_OK;
++}
++
++#if defined(CONFIG_SCHED_SMT) || defined(CONFIG_SCHED_MC)
++/*
++ * Cheaper version of the below functions in case support for SMT and MC is
++ * compiled in but CPUs have no siblings.
++ */
++static bool sole_cpu_idle(int cpu)
++{
++	return rq_idle(cpu_rq(cpu));
++}
++#endif
++#ifdef CONFIG_SCHED_SMT
++static const cpumask_t *thread_cpumask(int cpu)
++{
++	return topology_sibling_cpumask(cpu);
++}
++/* All this CPU's SMT siblings are idle */
++static bool siblings_cpu_idle(int cpu)
++{
++	return cpumask_subset(thread_cpumask(cpu), &grq.cpu_idle_map);
++}
++#endif
++#ifdef CONFIG_SCHED_MC
++static const cpumask_t *core_cpumask(int cpu)
++{
++	return topology_core_cpumask(cpu);
++}
++/* All this CPU's shared cache siblings are idle */
++static bool cache_cpu_idle(int cpu)
++{
++	return cpumask_subset(core_cpumask(cpu), &grq.cpu_idle_map);
++}
++#endif
++
++enum sched_domain_level {
++	SD_LV_NONE = 0,
++	SD_LV_SIBLING,
++	SD_LV_MC,
++	SD_LV_BOOK,
++	SD_LV_CPU,
++	SD_LV_NODE,
++	SD_LV_ALLNODES,
++	SD_LV_MAX
++};
++
++void __init sched_init_smp(void)
++{
++	struct sched_domain *sd;
++	int cpu, other_cpu;
++
++	cpumask_var_t non_isolated_cpus;
++
++	alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
++	alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
++
++	sched_init_numa();
++
++	/*
++	 * There's no userspace yet to cause hotplug operations; hence all the
++	 * cpu masks are stable and all blatant races in the below code cannot
++	 * happen.
++	 */
++	mutex_lock(&sched_domains_mutex);
++	init_sched_domains(cpu_active_mask);
++	cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
++	if (cpumask_empty(non_isolated_cpus))
++		cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
++	mutex_unlock(&sched_domains_mutex);
++
++	hotcpu_notifier(sched_domains_numa_masks_update, CPU_PRI_SCHED_ACTIVE);
++	hotcpu_notifier(cpuset_cpu_active, CPU_PRI_CPUSET_ACTIVE);
++	hotcpu_notifier(cpuset_cpu_inactive, CPU_PRI_CPUSET_INACTIVE);
++
++	/* Move init over to a non-isolated CPU */
++	if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
++		BUG();
++	free_cpumask_var(non_isolated_cpus);
++
++	mutex_lock(&sched_domains_mutex);
++	grq_lock_irq();
++	/*
++	 * Set up the relative cache distance of each online cpu from each
++	 * other in a simple array for quick lookup. Locality is determined
++	 * by the closest sched_domain that CPUs are separated by. CPUs with
++	 * shared cache in SMT and MC are treated as local. Separate CPUs
++	 * (within the same package or physically) within the same node are
++	 * treated as not local. CPUs not even in the same domain (different
++	 * nodes) are treated as very distant.
++	 */
++	for_each_online_cpu(cpu) {
++		struct rq *rq = cpu_rq(cpu);
++
++		/* First check if this cpu is in the same node */
++		for_each_domain(cpu, sd) {
++			if (sd->level > SD_LV_NODE)
++				continue;
++			/* Set locality to local node if not already found lower */
++			for_each_cpu(other_cpu, sched_domain_span(sd)) {
++				if (rq->cpu_locality[other_cpu] > 3)
++					rq->cpu_locality[other_cpu] = 3;
++			}
++		}
++
++		/*
++		 * Each runqueue has its own function in case it doesn't have
++		 * siblings of its own allowing mixed topologies.
++		 */
++#ifdef CONFIG_SCHED_MC
++		for_each_cpu(other_cpu, core_cpumask(cpu)) {
++			if (rq->cpu_locality[other_cpu] > 2)
++				rq->cpu_locality[other_cpu] = 2;
++		}
++		if (cpumask_weight(core_cpumask(cpu)) > 1)
++			rq->cache_idle = cache_cpu_idle;
++#endif
++#ifdef CONFIG_SCHED_SMT
++		for_each_cpu(other_cpu, thread_cpumask(cpu))
++			rq->cpu_locality[other_cpu] = 1;
++		if (cpumask_weight(thread_cpumask(cpu)) > 1)
++			rq->siblings_idle = siblings_cpu_idle;
++#endif
++	}
++	grq_unlock_irq();
++	mutex_unlock(&sched_domains_mutex);
++
++	for_each_online_cpu(cpu) {
++		struct rq *rq = cpu_rq(cpu);
++		for_each_online_cpu(other_cpu) {
++			if (other_cpu <= cpu)
++				continue;
++			printk(KERN_DEBUG "BFS LOCALITY CPU %d to %d: %d\n", cpu, other_cpu, rq->cpu_locality[other_cpu]);
++		}
++	}
++}
++#else
++void __init sched_init_smp(void)
++{
++}
++#endif /* CONFIG_SMP */
++
++int in_sched_functions(unsigned long addr)
++{
++	return in_lock_functions(addr) ||
++		(addr >= (unsigned long)__sched_text_start
++		&& addr < (unsigned long)__sched_text_end);
++}
++
++void __init sched_init(void)
++{
++#ifdef CONFIG_SMP
++	int cpu_ids;
++#endif
++	int i;
++	struct rq *rq;
++
++	prio_ratios[0] = 128;
++	for (i = 1 ; i < NICE_WIDTH ; i++)
++		prio_ratios[i] = prio_ratios[i - 1] * 11 / 10;
++
++	raw_spin_lock_init(&grq.lock);
++	grq.nr_running = grq.nr_uninterruptible = grq.nr_switches = 0;
++	grq.niffies = 0;
++	grq.last_jiffy = jiffies;
++	raw_spin_lock_init(&grq.iso_lock);
++	grq.iso_ticks = 0;
++	grq.iso_refractory = false;
++	grq.noc = 1;
++#ifdef CONFIG_SMP
++	init_defrootdomain();
++	grq.qnr = grq.idle_cpus = 0;
++	cpumask_clear(&grq.cpu_idle_map);
++#else
++	uprq = &per_cpu(runqueues, 0);
++#endif
++	for_each_possible_cpu(i) {
++		rq = cpu_rq(i);
++		rq->grq_lock = &grq.lock;
++		rq->user_pc = rq->nice_pc = rq->softirq_pc = rq->system_pc =
++			      rq->iowait_pc = rq->idle_pc = 0;
++		rq->dither = false;
++#ifdef CONFIG_SMP
++		rq->sticky_task = NULL;
++		rq->last_niffy = 0;
++		rq->sd = NULL;
++		rq->rd = NULL;
++		rq->online = false;
++		rq->cpu = i;
++		rq_attach_root(rq, &def_root_domain);
++#endif
++		atomic_set(&rq->nr_iowait, 0);
++	}
++
++#ifdef CONFIG_SMP
++	cpu_ids = i;
++	/*
++	 * Set the base locality for cpu cache distance calculation to
++	 * "distant" (3). Make sure the distance from a CPU to itself is 0.
++	 */
++	for_each_possible_cpu(i) {
++		int j;
++
++		rq = cpu_rq(i);
++#ifdef CONFIG_SCHED_SMT
++		rq->siblings_idle = sole_cpu_idle;
++#endif
++#ifdef CONFIG_SCHED_MC
++		rq->cache_idle = sole_cpu_idle;
++#endif
++		rq->cpu_locality = kmalloc(cpu_ids * sizeof(int *), GFP_ATOMIC);
++		for_each_possible_cpu(j) {
++			if (i == j)
++				rq->cpu_locality[j] = 0;
++			else
++				rq->cpu_locality[j] = 4;
++		}
++	}
++#endif
++
++	for (i = 0; i < PRIO_LIMIT; i++)
++		INIT_LIST_HEAD(grq.queue + i);
++	/* delimiter for bitsearch */
++	__set_bit(PRIO_LIMIT, grq.prio_bitmap);
++
++#ifdef CONFIG_PREEMPT_NOTIFIERS
++	INIT_HLIST_HEAD(&init_task.preempt_notifiers);
++#endif
++
++	/*
++	 * The boot idle thread does lazy MMU switching as well:
++	 */
++	atomic_inc(&init_mm.mm_count);
++	enter_lazy_tlb(&init_mm, current);
++
++	/*
++	 * Make us the idle thread. Technically, schedule() should not be
++	 * called from this thread, however somewhere below it might be,
++	 * but because we are the idle thread, we just pick up running again
++	 * when this runqueue becomes "idle".
++	 */
++	init_idle(current, smp_processor_id());
++
++#ifdef CONFIG_SMP
++	zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT);
++	/* May be allocated at isolcpus cmdline parse time */
++	if (cpu_isolated_map == NULL)
++		zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
++	idle_thread_set_boot_cpu();
++#endif /* SMP */
++}
++
++#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
++static inline int preempt_count_equals(int preempt_offset)
++{
++	int nested = preempt_count() + rcu_preempt_depth();
++
++	return (nested == preempt_offset);
++}
++
++void __might_sleep(const char *file, int line, int preempt_offset)
++{
++	/*
++	 * Blocking primitives will set (and therefore destroy) current->state,
++	 * since we will exit with TASK_RUNNING make sure we enter with it,
++	 * otherwise we will destroy state.
++	 */
++	WARN_ONCE(current->state != TASK_RUNNING && current->task_state_change,
++			"do not call blocking ops when !TASK_RUNNING; "
++			"state=%lx set at [<%p>] %pS\n",
++			current->state,
++			(void *)current->task_state_change,
++			(void *)current->task_state_change);
++
++	___might_sleep(file, line, preempt_offset);
++}
++EXPORT_SYMBOL(__might_sleep);
++
++void ___might_sleep(const char *file, int line, int preempt_offset)
++{
++	static unsigned long prev_jiffy;	/* ratelimiting */
++
++	rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */
++	if ((preempt_count_equals(preempt_offset) && !irqs_disabled() &&
++	     !is_idle_task(current)) ||
++	    system_state != SYSTEM_RUNNING || oops_in_progress)
++		return;
++	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
++		return;
++	prev_jiffy = jiffies;
++
++	printk(KERN_ERR
++		"BUG: sleeping function called from invalid context at %s:%d\n",
++			file, line);
++	printk(KERN_ERR
++		"in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
++			in_atomic(), irqs_disabled(),
++			current->pid, current->comm);
++
++	if (task_stack_end_corrupted(current))
++		printk(KERN_EMERG "Thread overran stack, or stack corrupted\n");
++
++	debug_show_held_locks(current);
++	if (irqs_disabled())
++		print_irqtrace_events(current);
++#ifdef CONFIG_DEBUG_PREEMPT
++	if (!preempt_count_equals(preempt_offset)) {
++		pr_err("Preemption disabled at:");
++		print_ip_sym(current->preempt_disable_ip);
++		pr_cont("\n");
++	}
++#endif
++	dump_stack();
++}
++EXPORT_SYMBOL(___might_sleep);
++#endif
++
++#ifdef CONFIG_MAGIC_SYSRQ
++static inline void normalise_rt_tasks(void)
++{
++	struct task_struct *g, *p;
++	unsigned long flags;
++	struct rq *rq;
++	int queued;
++
++	read_lock(&tasklist_lock);
++	for_each_process_thread(g, p) {
++		/*
++		 * Only normalize user tasks:
++		 */
++		if (p->flags & PF_KTHREAD)
++			continue;
++
++		if (!rt_task(p) && !iso_task(p))
++			continue;
++
++		rq = task_grq_lock(p, &flags);
++		queued = task_queued(p);
++		if (queued)
++			dequeue_task(p);
++		__setscheduler(p, rq, SCHED_NORMAL, 0, false);
++		if (queued) {
++			enqueue_task(p, rq);
++			try_preempt(p, rq);
++		}
++
++		task_grq_unlock(&flags);
++	}
++	read_unlock(&tasklist_lock);
++}
++
++void normalize_rt_tasks(void)
++{
++	normalise_rt_tasks();
++}
++#endif /* CONFIG_MAGIC_SYSRQ */
++
++#if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB)
++/*
++ * These functions are only useful for the IA64 MCA handling, or kdb.
++ *
++ * They can only be called when the whole system has been
++ * stopped - every CPU needs to be quiescent, and no scheduling
++ * activity can take place. Using them for anything else would
++ * be a serious bug, and as a result, they aren't even visible
++ * under any other configuration.
++ */
++
++/**
++ * curr_task - return the current task for a given cpu.
++ * @cpu: the processor in question.
++ *
++ * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
++ *
++ * Return: The current task for @cpu.
++ */
++struct task_struct *curr_task(int cpu)
++{
++	return cpu_curr(cpu);
++}
++
++#endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */
++
++#ifdef CONFIG_IA64
++/**
++ * set_curr_task - set the current task for a given cpu.
++ * @cpu: the processor in question.
++ * @p: the task pointer to set.
++ *
++ * Description: This function must only be used when non-maskable interrupts
++ * are serviced on a separate stack.  It allows the architecture to switch the
++ * notion of the current task on a cpu in a non-blocking manner.  This function
++ * must be called with all CPU's synchronised, and interrupts disabled, the
++ * and caller must save the original value of the current task (see
++ * curr_task() above) and restore that value before reenabling interrupts and
++ * re-starting the system.
++ *
++ * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
++ */
++void set_curr_task(int cpu, struct task_struct *p)
++{
++	cpu_curr(cpu) = p;
++}
++
++#endif
++
++/*
++ * Use precise platform statistics if available:
++ */
++#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
++void task_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st)
++{
++	*ut = p->utime;
++	*st = p->stime;
++}
++EXPORT_SYMBOL_GPL(task_cputime_adjusted);
++
++void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st)
++{
++	struct task_cputime cputime;
++
++	thread_group_cputime(p, &cputime);
++
++	*ut = cputime.utime;
++	*st = cputime.stime;
++}
++
++void vtime_account_system_irqsafe(struct task_struct *tsk)
++{
++	unsigned long flags;
++
++	local_irq_save(flags);
++	vtime_account_system(tsk);
++	local_irq_restore(flags);
++}
++EXPORT_SYMBOL_GPL(vtime_account_system_irqsafe);
++
++#ifndef __ARCH_HAS_VTIME_TASK_SWITCH
++void vtime_task_switch(struct task_struct *prev)
++{
++	if (is_idle_task(prev))
++		vtime_account_idle(prev);
++	else
++		vtime_account_system(prev);
++
++	vtime_account_user(prev);
++	arch_vtime_task_switch(prev);
++}
++#endif
++
++#else
++/*
++ * Perform (stime * rtime) / total, but avoid multiplication overflow by
++ * losing precision when the numbers are big.
++ */
++static cputime_t scale_stime(u64 stime, u64 rtime, u64 total)
++{
++	u64 scaled;
++
++	for (;;) {
++		/* Make sure "rtime" is the bigger of stime/rtime */
++		if (stime > rtime) {
++			u64 tmp = rtime; rtime = stime; stime = tmp;
++		}
++
++		/* Make sure 'total' fits in 32 bits */
++		if (total >> 32)
++			goto drop_precision;
++
++		/* Does rtime (and thus stime) fit in 32 bits? */
++		if (!(rtime >> 32))
++			break;
++
++		/* Can we just balance rtime/stime rather than dropping bits? */
++		if (stime >> 31)
++			goto drop_precision;
++
++		/* We can grow stime and shrink rtime and try to make them both fit */
++		stime <<= 1;
++		rtime >>= 1;
++		continue;
++
++drop_precision:
++		/* We drop from rtime, it has more bits than stime */
++		rtime >>= 1;
++		total >>= 1;
++	}
++
++	/*
++	 * Make sure gcc understands that this is a 32x32->64 multiply,
++	 * followed by a 64/32->64 divide.
++	 */
++	scaled = div_u64((u64) (u32) stime * (u64) (u32) rtime, (u32)total);
++	return (__force cputime_t) scaled;
++}
++
++/*
++ * Adjust tick based cputime random precision against scheduler
++ * runtime accounting.
++ */
++static void cputime_adjust(struct task_cputime *curr,
++			   struct prev_cputime *prev,
++			   cputime_t *ut, cputime_t *st)
++{
++	cputime_t rtime, stime, utime, total;
++
++	stime = curr->stime;
++	total = stime + curr->utime;
++
++	/*
++	 * Tick based cputime accounting depend on random scheduling
++	 * timeslices of a task to be interrupted or not by the timer.
++	 * Depending on these circumstances, the number of these interrupts
++	 * may be over or under-optimistic, matching the real user and system
++	 * cputime with a variable precision.
++	 *
++	 * Fix this by scaling these tick based values against the total
++	 * runtime accounted by the CFS scheduler.
++	 */
++	rtime = nsecs_to_cputime(curr->sum_exec_runtime);
++
++	/*
++	 * Update userspace visible utime/stime values only if actual execution
++	 * time is bigger than already exported. Note that can happen, that we
++	 * provided bigger values due to scaling inaccuracy on big numbers.
++	 */
++	if (prev->stime + prev->utime >= rtime)
++		goto out;
++
++	if (total) {
++		stime = scale_stime((__force u64)stime,
++				    (__force u64)rtime, (__force u64)total);
++		utime = rtime - stime;
++	} else {
++		stime = rtime;
++		utime = 0;
++	}
++
++	/*
++	 * If the tick based count grows faster than the scheduler one,
++	 * the result of the scaling may go backward.
++	 * Let's enforce monotonicity.
++	 */
++	prev->stime = max(prev->stime, stime);
++	prev->utime = max(prev->utime, utime);
++
++out:
++	*ut = prev->utime;
++	*st = prev->stime;
++}
++
++void task_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st)
++{
++	struct task_cputime cputime = {
++		.sum_exec_runtime = tsk_seruntime(p),
++	};
++
++	task_cputime(p, &cputime.utime, &cputime.stime);
++	cputime_adjust(&cputime, &p->prev_cputime, ut, st);
++}
++EXPORT_SYMBOL_GPL(task_cputime_adjusted);
++
++/*
++ * Must be called with siglock held.
++ */
++void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st)
++{
++	struct task_cputime cputime;
++
++	thread_group_cputime(p, &cputime);
++	cputime_adjust(&cputime, &p->signal->prev_cputime, ut, st);
++}
++#endif
++
++void init_idle_bootup_task(struct task_struct *idle)
++{}
++
++#ifdef CONFIG_SCHED_DEBUG
++void proc_sched_show_task(struct task_struct *p, struct seq_file *m)
++{}
++
++void proc_sched_set_task(struct task_struct *p)
++{}
++#endif
++
++#ifdef CONFIG_SMP
++#define SCHED_LOAD_SHIFT	(10)
++#define SCHED_LOAD_SCALE	(1L << SCHED_LOAD_SHIFT)
++
++unsigned long default_scale_freq_power(struct sched_domain *sd, int cpu)
++{
++	return SCHED_LOAD_SCALE;
++}
++
++unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu)
++{
++	unsigned long weight = cpumask_weight(sched_domain_span(sd));
++	unsigned long smt_gain = sd->smt_gain;
++
++	smt_gain /= weight;
++
++	return smt_gain;
++}
++#endif
+Index: linux-4.4-ck1/include/uapi/linux/sched.h
+===================================================================
+--- linux-4.4-ck1.orig/include/uapi/linux/sched.h	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/include/uapi/linux/sched.h	2016-03-25 16:03:45.294350506 +1100
+@@ -37,9 +37,16 @@
+ #define SCHED_FIFO		1
+ #define SCHED_RR		2
+ #define SCHED_BATCH		3
+-/* SCHED_ISO: reserved but not implemented yet */
++/* SCHED_ISO: Implemented on BFS only */
+ #define SCHED_IDLE		5
++#ifdef CONFIG_SCHED_BFS
++#define SCHED_ISO		4
++#define SCHED_IDLEPRIO		SCHED_IDLE
++#define SCHED_MAX		(SCHED_IDLEPRIO)
++#define SCHED_RANGE(policy)	((policy) <= SCHED_MAX)
++#else /* CONFIG_SCHED_BFS */
+ #define SCHED_DEADLINE		6
++#endif /* CONFIG_SCHED_BFS */
+ 
+ /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */
+ #define SCHED_RESET_ON_FORK     0x40000000
+Index: linux-4.4-ck1/drivers/cpufreq/cpufreq_conservative.c
+===================================================================
+--- linux-4.4-ck1.orig/drivers/cpufreq/cpufreq_conservative.c	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/drivers/cpufreq/cpufreq_conservative.c	2016-03-25 16:03:45.294350506 +1100
+@@ -15,8 +15,8 @@
+ #include "cpufreq_governor.h"
+ 
+ /* Conservative governor macros */
+-#define DEF_FREQUENCY_UP_THRESHOLD		(80)
+-#define DEF_FREQUENCY_DOWN_THRESHOLD		(20)
++#define DEF_FREQUENCY_UP_THRESHOLD		(63)
++#define DEF_FREQUENCY_DOWN_THRESHOLD		(26)
+ #define DEF_FREQUENCY_STEP			(5)
+ #define DEF_SAMPLING_DOWN_FACTOR		(1)
+ #define MAX_SAMPLING_DOWN_FACTOR		(10)
+Index: linux-4.4-ck1/kernel/time/Kconfig
+===================================================================
+--- linux-4.4-ck1.orig/kernel/time/Kconfig	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/kernel/time/Kconfig	2016-03-25 16:03:45.294350506 +1100
+@@ -89,7 +89,7 @@ config NO_HZ_IDLE
+ config NO_HZ_FULL
+ 	bool "Full dynticks system (tickless)"
+ 	# NO_HZ_COMMON dependency
+-	depends on !ARCH_USES_GETTIMEOFFSET && GENERIC_CLOCKEVENTS
++	depends on !ARCH_USES_GETTIMEOFFSET && GENERIC_CLOCKEVENTS && !SCHED_BFS
+ 	# We need at least one periodic CPU for timekeeping
+ 	depends on SMP
+ 	depends on HAVE_CONTEXT_TRACKING
+Index: linux-4.4-ck1/kernel/sched/Makefile
+===================================================================
+--- linux-4.4-ck1.orig/kernel/sched/Makefile	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/kernel/sched/Makefile	2016-03-25 16:03:45.294350506 +1100
+@@ -11,11 +11,17 @@ ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER
+ CFLAGS_core.o := $(PROFILING) -fno-omit-frame-pointer
+ endif
+ 
++ifdef CONFIG_SCHED_BFS
++obj-y += bfs.o clock.o
++else
+ obj-y += core.o loadavg.o clock.o cputime.o
+ obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o
+ obj-y += wait.o completion.o idle.o
+-obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o
++obj-$(CONFIG_SMP) += cpudeadline.o
+ obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o
+-obj-$(CONFIG_SCHEDSTATS) += stats.o
+ obj-$(CONFIG_SCHED_DEBUG) += debug.o
+ obj-$(CONFIG_CGROUP_CPUACCT) += cpuacct.o
++endif
++obj-y += wait.o completion.o idle.o
++obj-$(CONFIG_SMP) += cpupri.o
++obj-$(CONFIG_SCHEDSTATS) += stats.o
+Index: linux-4.4-ck1/kernel/sched/bfs_sched.h
+===================================================================
+--- /dev/null	1970-01-01 00:00:00.000000000 +0000
++++ linux-4.4-ck1/kernel/sched/bfs_sched.h	2016-03-25 16:03:45.294350506 +1100
+@@ -0,0 +1,181 @@
++#include <linux/sched.h>
++#include <linux/cpuidle.h>
++#include <linux/stop_machine.h>
++
++#ifndef BFS_SCHED_H
++#define BFS_SCHED_H
++
++/*
++ * This is the main, per-CPU runqueue data structure.
++ * This data should only be modified by the local cpu.
++ */
++struct rq {
++	struct task_struct *curr, *idle, *stop;
++	struct mm_struct *prev_mm;
++
++	/* Pointer to grq spinlock */
++	raw_spinlock_t *grq_lock;
++
++	/* Stored data about rq->curr to work outside grq lock */
++	u64 rq_deadline;
++	unsigned int rq_policy;
++	int rq_time_slice;
++	u64 rq_last_ran;
++	int rq_prio;
++	bool rq_running; /* There is a task running */
++	int soft_affined; /* Running or queued tasks with this set as their rq */
++#ifdef CONFIG_SMT_NICE
++	struct mm_struct *rq_mm;
++	int rq_smt_bias; /* Policy/nice level bias across smt siblings */
++#endif
++	/* Accurate timekeeping data */
++	u64 timekeep_clock;
++	unsigned long user_pc, nice_pc, irq_pc, softirq_pc, system_pc,
++		iowait_pc, idle_pc;
++	atomic_t nr_iowait;
++
++#ifdef CONFIG_SMP
++	int cpu;		/* cpu of this runqueue */
++	bool online;
++	bool scaling; /* This CPU is managed by a scaling CPU freq governor */
++	struct task_struct *sticky_task;
++
++	struct root_domain *rd;
++	struct sched_domain *sd;
++	int *cpu_locality; /* CPU relative cache distance */
++#ifdef CONFIG_SCHED_SMT
++	bool (*siblings_idle)(int cpu);
++	/* See if all smt siblings are idle */
++#endif /* CONFIG_SCHED_SMT */
++#ifdef CONFIG_SCHED_MC
++	bool (*cache_idle)(int cpu);
++	/* See if all cache siblings are idle */
++#endif /* CONFIG_SCHED_MC */
++	u64 last_niffy; /* Last time this RQ updated grq.niffies */
++#endif /* CONFIG_SMP */
++#ifdef CONFIG_IRQ_TIME_ACCOUNTING
++	u64 prev_irq_time;
++#endif /* CONFIG_IRQ_TIME_ACCOUNTING */
++#ifdef CONFIG_PARAVIRT
++	u64 prev_steal_time;
++#endif /* CONFIG_PARAVIRT */
++#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
++	u64 prev_steal_time_rq;
++#endif /* CONFIG_PARAVIRT_TIME_ACCOUNTING */
++
++	u64 clock, old_clock, last_tick;
++	u64 clock_task;
++	bool dither;
++
++#ifdef CONFIG_SCHEDSTATS
++
++	/* latency stats */
++	struct sched_info rq_sched_info;
++	unsigned long long rq_cpu_time;
++	/* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
++
++	/* sys_sched_yield() stats */
++	unsigned int yld_count;
++
++	/* schedule() stats */
++	unsigned int sched_switch;
++	unsigned int sched_count;
++	unsigned int sched_goidle;
++
++	/* try_to_wake_up() stats */
++	unsigned int ttwu_count;
++	unsigned int ttwu_local;
++#endif /* CONFIG_SCHEDSTATS */
++#ifdef CONFIG_CPU_IDLE
++	/* Must be inspected within a rcu lock section */
++	struct cpuidle_state *idle_state;
++#endif
++};
++
++#ifdef CONFIG_SMP
++struct rq *cpu_rq(int cpu);
++#endif
++
++#ifndef CONFIG_SMP
++extern struct rq *uprq;
++#define cpu_rq(cpu)	(uprq)
++#define this_rq()	(uprq)
++#define raw_rq()	(uprq)
++#define task_rq(p)	(uprq)
++#define cpu_curr(cpu)	((uprq)->curr)
++#else /* CONFIG_SMP */
++DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
++#define this_rq()		this_cpu_ptr(&runqueues)
++#define raw_rq()		raw_cpu_ptr(&runqueues)
++#endif /* CONFIG_SMP */
++
++static inline u64 __rq_clock_broken(struct rq *rq)
++{
++	return READ_ONCE(rq->clock);
++}
++
++static inline u64 rq_clock(struct rq *rq)
++{
++	lockdep_assert_held(rq->grq_lock);
++	return rq->clock;
++}
++
++static inline u64 rq_clock_task(struct rq *rq)
++{
++	lockdep_assert_held(rq->grq_lock);
++	return rq->clock_task;
++}
++
++extern struct mutex sched_domains_mutex;
++
++#define rcu_dereference_check_sched_domain(p) \
++	rcu_dereference_check((p), \
++			      lockdep_is_held(&sched_domains_mutex))
++
++/*
++ * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
++ * See detach_destroy_domains: synchronize_sched for details.
++ *
++ * The domain tree of any CPU may only be accessed from within
++ * preempt-disabled sections.
++ */
++#define for_each_domain(cpu, __sd) \
++	for (__sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent)
++
++static inline void sched_ttwu_pending(void) { }
++
++static inline int task_on_rq_queued(struct task_struct *p)
++{
++	return p->on_rq;
++}
++
++#ifdef CONFIG_SMP
++
++extern void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask);
++
++#endif
++
++#ifdef CONFIG_CPU_IDLE
++static inline void idle_set_state(struct rq *rq,
++				  struct cpuidle_state *idle_state)
++{
++	rq->idle_state = idle_state;
++}
++
++static inline struct cpuidle_state *idle_get_state(struct rq *rq)
++{
++	WARN_ON(!rcu_read_lock_held());
++	return rq->idle_state;
++}
++#else
++static inline void idle_set_state(struct rq *rq,
++				  struct cpuidle_state *idle_state)
++{
++}
++
++static inline struct cpuidle_state *idle_get_state(struct rq *rq)
++{
++	return NULL;
++}
++#endif
++#endif /* BFS_SCHED_H */
+Index: linux-4.4-ck1/kernel/sched/stats.c
+===================================================================
+--- linux-4.4-ck1.orig/kernel/sched/stats.c	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/kernel/sched/stats.c	2016-03-25 16:03:45.294350506 +1100
+@@ -4,7 +4,11 @@
+ #include <linux/seq_file.h>
+ #include <linux/proc_fs.h>
+ 
++#ifndef CONFIG_SCHED_BFS
+ #include "sched.h"
++#else
++#include "bfs_sched.h"
++#endif
+ 
+ /*
+  * bump this up when changing the output format or the meaning of an existing
+Index: linux-4.4-ck1/arch/x86/Kconfig
+===================================================================
+--- linux-4.4-ck1.orig/arch/x86/Kconfig	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/arch/x86/Kconfig	2016-03-25 16:03:45.294350506 +1100
+@@ -877,10 +877,26 @@ config SCHED_SMT
+ 	depends on SMP
+ 	---help---
+ 	  SMT scheduler support improves the CPU scheduler's decision making
+-	  when dealing with Intel Pentium 4 chips with HyperThreading at a
++	  when dealing with Intel P4/Core 2 chips with HyperThreading at a
+ 	  cost of slightly increased overhead in some places. If unsure say
+ 	  N here.
+ 
++config SMT_NICE
++	bool "SMT (Hyperthreading) aware nice priority and policy support"
++	depends on SCHED_BFS && SCHED_SMT
++	default y
++	---help---
++	  Enabling Hyperthreading on Intel CPUs decreases the effectiveness
++	  of the use of 'nice' levels and different scheduling policies
++	  (e.g. realtime) due to sharing of CPU power between hyperthreads.
++	  SMT nice support makes each logical CPU aware of what is running on
++	  its hyperthread siblings, maintaining appropriate distribution of
++	  CPU according to nice levels and scheduling policies at the expense
++	  of slightly increased overhead.
++
++	  If unsure say Y here.
++
++
+ config SCHED_MC
+ 	def_bool y
+ 	prompt "Multi-core scheduler support"
+@@ -1961,7 +1977,7 @@ config HOTPLUG_CPU
+ config BOOTPARAM_HOTPLUG_CPU0
+ 	bool "Set default setting of cpu0_hotpluggable"
+ 	default n
+-	depends on HOTPLUG_CPU
++	depends on HOTPLUG_CPU && !SCHED_BFS
+ 	---help---
+ 	  Set whether default state of cpu0_hotpluggable is on or off.
+ 
+@@ -1990,7 +2006,7 @@ config BOOTPARAM_HOTPLUG_CPU0
+ config DEBUG_HOTPLUG_CPU0
+ 	def_bool n
+ 	prompt "Debug CPU0 hotplug"
+-	depends on HOTPLUG_CPU
++	depends on HOTPLUG_CPU && !SCHED_BFS
+ 	---help---
+ 	  Enabling this option offlines CPU0 (if CPU0 can be offlined) as
+ 	  soon as possible and boots up userspace with CPU0 offlined. User
+Index: linux-4.4-ck1/include/linux/sched/prio.h
+===================================================================
+--- linux-4.4-ck1.orig/include/linux/sched/prio.h	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/include/linux/sched/prio.h	2016-03-25 16:03:45.295350525 +1100
+@@ -19,8 +19,20 @@
+  */
+ 
+ #define MAX_USER_RT_PRIO	100
++
++#ifdef CONFIG_SCHED_BFS
++/* Note different MAX_RT_PRIO */
++#define MAX_RT_PRIO		(MAX_USER_RT_PRIO + 1)
++
++#define ISO_PRIO		(MAX_RT_PRIO)
++#define NORMAL_PRIO		(MAX_RT_PRIO + 1)
++#define IDLE_PRIO		(MAX_RT_PRIO + 2)
++#define PRIO_LIMIT		((IDLE_PRIO) + 1)
++#else /* CONFIG_SCHED_BFS */
+ #define MAX_RT_PRIO		MAX_USER_RT_PRIO
+ 
++#endif /* CONFIG_SCHED_BFS */
++
+ #define MAX_PRIO		(MAX_RT_PRIO + NICE_WIDTH)
+ #define DEFAULT_PRIO		(MAX_RT_PRIO + NICE_WIDTH / 2)
+ 
+Index: linux-4.4-ck1/drivers/cpufreq/intel_pstate.c
+===================================================================
+--- linux-4.4-ck1.orig/drivers/cpufreq/intel_pstate.c	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/drivers/cpufreq/intel_pstate.c	2016-03-25 16:03:45.295350525 +1100
+@@ -569,8 +569,13 @@ static void atom_set_pstate(struct cpuda
+ 	vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max);
+ 	vid = ceiling_fp(vid_fp);
+ 
+-	if (pstate > cpudata->pstate.max_pstate)
+-		vid = cpudata->vid.turbo;
++	if (pstate < cpudata->pstate.max_pstate)
++		cpu_scaling(cpudata->cpu);
++	else {
++		if (pstate > cpudata->pstate.max_pstate)
++			vid = cpudata->vid.turbo;
++		cpu_nonscaling(cpudata->cpu);
++	}
+ 
+ 	val |= vid;
+ 
+Index: linux-4.4-ck1/kernel/sched/idle.c
+===================================================================
+--- linux-4.4-ck1.orig/kernel/sched/idle.c	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/kernel/sched/idle.c	2016-03-25 16:03:45.295350525 +1100
+@@ -13,7 +13,11 @@
+ 
+ #include <trace/events/power.h>
+ 
++#ifdef CONFIG_SCHED_BFS
++#include "bfs_sched.h"
++#else
+ #include "sched.h"
++#endif
+ 
+ /**
+  * sched_idle_set_state - Record idle state for the current CPU.
+Index: linux-4.4-ck1/kernel/time/posix-cpu-timers.c
+===================================================================
+--- linux-4.4-ck1.orig/kernel/time/posix-cpu-timers.c	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/kernel/time/posix-cpu-timers.c	2016-03-25 16:03:45.295350525 +1100
+@@ -448,7 +448,7 @@ static void cleanup_timers(struct list_h
+  */
+ void posix_cpu_timers_exit(struct task_struct *tsk)
+ {
+-	add_device_randomness((const void*) &tsk->se.sum_exec_runtime,
++	add_device_randomness((const void*) &tsk_seruntime(tsk),
+ 						sizeof(unsigned long long));
+ 	cleanup_timers(tsk->cpu_timers);
+ 
+@@ -878,7 +878,7 @@ static void check_thread_timers(struct t
+ 	tsk_expires->virt_exp = expires_to_cputime(expires);
+ 
+ 	tsk_expires->sched_exp = check_timers_list(++timers, firing,
+-						   tsk->se.sum_exec_runtime);
++						   tsk_seruntime(tsk));
+ 
+ 	/*
+ 	 * Check for the special case thread timers.
+@@ -889,7 +889,7 @@ static void check_thread_timers(struct t
+ 			READ_ONCE(sig->rlim[RLIMIT_RTTIME].rlim_max);
+ 
+ 		if (hard != RLIM_INFINITY &&
+-		    tsk->rt.timeout > DIV_ROUND_UP(hard, USEC_PER_SEC/HZ)) {
++		    tsk_rttimeout(tsk) > DIV_ROUND_UP(hard, USEC_PER_SEC/HZ)) {
+ 			/*
+ 			 * At the hard limit, we just die.
+ 			 * No need to calculate anything else now.
+@@ -897,7 +897,7 @@ static void check_thread_timers(struct t
+ 			__group_send_sig_info(SIGKILL, SEND_SIG_PRIV, tsk);
+ 			return;
+ 		}
+-		if (tsk->rt.timeout > DIV_ROUND_UP(soft, USEC_PER_SEC/HZ)) {
++		if (tsk_rttimeout(tsk) > DIV_ROUND_UP(soft, USEC_PER_SEC/HZ)) {
+ 			/*
+ 			 * At the soft limit, send a SIGXCPU every second.
+ 			 */
+@@ -1144,7 +1144,7 @@ static inline int fastpath_timer_check(s
+ 		struct task_cputime task_sample;
+ 
+ 		task_cputime(tsk, &task_sample.utime, &task_sample.stime);
+-		task_sample.sum_exec_runtime = tsk->se.sum_exec_runtime;
++		task_sample.sum_exec_runtime = tsk_seruntime(tsk);
+ 		if (task_cputime_expired(&task_sample, &tsk->cputime_expires))
+ 			return 1;
+ 	}
+Index: linux-4.4-ck1/kernel/trace/trace_selftest.c
+===================================================================
+--- linux-4.4-ck1.orig/kernel/trace/trace_selftest.c	2016-03-25 16:03:45.297350563 +1100
++++ linux-4.4-ck1/kernel/trace/trace_selftest.c	2016-03-25 16:03:45.295350525 +1100
+@@ -1039,10 +1039,15 @@ static int trace_wakeup_test_thread(void
+ {
+ 	/* Make this a -deadline thread */
+ 	static const struct sched_attr attr = {
++#ifdef CONFIG_SCHED_BFS
++		/* No deadline on BFS, use RR */
++		.sched_policy = SCHED_RR,
++#else
+ 		.sched_policy = SCHED_DEADLINE,
+ 		.sched_runtime = 100000ULL,
+ 		.sched_deadline = 10000000ULL,
+ 		.sched_period = 10000000ULL
++#endif
+ 	};
+ 	struct wakeup_test_data *x = data;
+ 
diff --git a/patches_disabled/BLD-4.6.patch b/patches_disabled/BLD-4.6.patch
new file mode 100644
index 0000000..51cc1f7
--- /dev/null
+++ b/patches_disabled/BLD-4.6.patch
@@ -0,0 +1,708 @@
+diff --git a/init/Kconfig b/init/Kconfig
+index 0dfd09d..8d704e5 100644
+--- a/init/Kconfig
++++ b/init/Kconfig
+@@ -36,6 +36,15 @@ config BROKEN_ON_SMP
+ 	depends on BROKEN || !SMP
+ 	default y
+ 
++config BLD
++	bool "An alternate CPU load distribution technique for task scheduler"
++	depends on SMP
++	default y
++	help
++	  This is an alternate CPU load distribution technique based for task
++	  scheduler based on The Barbershop Load Distribution algorithm. Not
++	  suitable for NUMA, should work well on SMP.
++
+ config INIT_ENV_ARG_LIMIT
+ 	int
+ 	default 32 if !UML
+diff --git a/kernel/sched/bld.h b/kernel/sched/bld.h
+new file mode 100644
+index 0000000..f1f9fba
+--- /dev/null
++++ b/kernel/sched/bld.h
+@@ -0,0 +1,215 @@
++#ifdef CONFIG_BLD
++
++static DEFINE_RWLOCK(rt_list_lock);
++static LIST_HEAD(rt_rq_head);
++static LIST_HEAD(cfs_rq_head);
++static DEFINE_RWLOCK(cfs_list_lock);
++
++#ifdef CONFIG_FAIR_GROUP_SCHED
++static inline struct rq *rq_of_cfs(struct cfs_rq *cfs_rq)
++{
++	return cfs_rq->rq;
++}
++#else
++static inline struct rq *rq_of_cfs(struct cfs_rq *cfs_rq)
++{
++	return container_of(cfs_rq, struct rq, cfs);
++}
++#endif
++
++#ifdef CONFIG_RT_GROUP_SCHED
++static inline struct rq *rq_of_rt(struct rt_rq *rt_rq)
++{
++	return rt_rq->rq;
++}
++#else
++static inline struct rq *rq_of_rt(struct rt_rq *rt_rq)
++{
++	return container_of(rt_rq, struct rq, rt);
++}
++#endif
++
++static int select_cpu_for_wakeup(int task_type, struct cpumask *mask)
++{
++	int cpu = smp_processor_id(), i;
++	unsigned long load, varload;
++	struct rq *rq;
++
++	if (task_type) {
++		varload = ULONG_MAX;
++		for_each_cpu(i, mask) {
++			rq = cpu_rq(i);
++			load = rq->cfs.load.weight;
++			if (load < varload) {
++				varload = load;
++				cpu = i;
++			}
++		}
++	} else {
++		/* Here's an attempt to get a CPU within the mask where
++		 * we can preempt easily. To achieve this we tried to
++		 * maintain a lowbit, which indicate the lowest bit set on
++		 * array bitmap. Since all CPUs contains high priority
++		 * kernel threads therefore we eliminate 0, so it might not
++		 * be right every time, but it's just an indicator.
++		 */
++		varload = 1;
++
++		for_each_cpu(i, mask) {
++			rq = cpu_rq(i);
++			load = rq->rt.lowbit;
++			if (load >= varload) {
++				varload = load;
++				cpu = i;
++			}
++		}
++	}
++
++	return cpu;
++}
++
++static int bld_pick_cpu_cfs(struct task_struct *p, int sd_flags, int wake_flags)
++{
++	struct cfs_rq *cfs;
++	unsigned long flags;
++	unsigned int cpu = smp_processor_id();
++
++	read_lock_irqsave(&cfs_list_lock, flags);
++	list_for_each_entry(cfs, &cfs_rq_head, bld_cfs_list) {
++		cpu = cpu_of(rq_of_cfs(cfs));
++		if (cpu_online(cpu))
++			break;
++	}
++	read_unlock_irqrestore(&cfs_list_lock, flags);
++	return cpu;
++}
++
++static int bld_pick_cpu_rt(struct task_struct *p, int sd_flags, int wake_flags)
++{
++	struct rt_rq *rt;
++	unsigned long flags;
++	unsigned int cpu = smp_processor_id();
++
++	read_lock_irqsave(&rt_list_lock, flags);
++	list_for_each_entry(rt, &rt_rq_head, bld_rt_list) {
++		cpu = cpu_of(rq_of_rt(rt));
++		if (cpu_online(cpu))
++			break;
++	}
++	read_unlock_irqrestore(&rt_list_lock, flags);
++	return cpu;
++}
++
++static int bld_pick_cpu_domain(struct task_struct *p, int sd_flags, int wake_flags)
++{
++	unsigned int cpu = smp_processor_id(), want_affine = 0;
++	struct cpumask *tmpmask;
++
++	if (p->nr_cpus_allowed == 1)
++		return task_cpu(p);
++
++	if (sd_flags & SD_BALANCE_WAKE) {
++		if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) {
++			want_affine = 1;
++		}
++	}
++
++	if (want_affine)
++		tmpmask = tsk_cpus_allowed(p);
++	else
++		tmpmask = sched_domain_span(cpu_rq(task_cpu(p))->sd);
++
++	if (rt_task(p))
++		cpu = select_cpu_for_wakeup(0, tmpmask);
++	else
++		cpu = select_cpu_for_wakeup(1, tmpmask);
++
++	return cpu;
++}
++
++static void track_load_rt(struct rq *rq, struct task_struct *p)
++{
++	unsigned long flag;
++	int firstbit;
++	struct rt_rq *first;
++	struct rt_prio_array *array = &rq->rt.active;
++
++	first = list_entry(rt_rq_head.next, struct rt_rq, bld_rt_list);
++	firstbit = sched_find_first_bit(array->bitmap);
++
++	/* Maintaining rt.lowbit */
++	if (firstbit > 0 && firstbit <= rq->rt.lowbit)
++		rq->rt.lowbit = firstbit;
++
++	if (rq->rt.lowbit < first->lowbit) {
++		write_lock_irqsave(&rt_list_lock, flag);
++		list_del(&rq->rt.bld_rt_list);
++		list_add_tail(&rq->rt.bld_rt_list, &rt_rq_head);
++		write_unlock_irqrestore(&rt_list_lock, flag);
++	}
++}
++
++static int bld_get_cpu(struct task_struct *p, int sd_flags, int wake_flags)
++{
++	unsigned int cpu;
++
++	if (sd_flags == SD_BALANCE_WAKE || (sd_flags == SD_BALANCE_EXEC && (get_nr_threads(p) > 1)))
++		cpu = bld_pick_cpu_domain(p, sd_flags, wake_flags);
++	else {
++		if (rt_task(p))
++			cpu = bld_pick_cpu_rt(p, sd_flags, wake_flags);
++		else
++			cpu = bld_pick_cpu_cfs(p, sd_flags, wake_flags);
++	}
++
++	return cpu;
++}
++
++static void bld_track_load_activate(struct rq *rq, struct task_struct *p)
++{
++	unsigned long flag;
++	if (rt_task(p)) {
++		track_load_rt(rq, p);
++	} else {
++		if (rq->cfs.pos != 2) {
++			struct cfs_rq *last;
++			last = list_entry(cfs_rq_head.prev, struct cfs_rq, bld_cfs_list);
++			if (rq->cfs.load.weight >= last->load.weight) {
++				write_lock_irqsave(&cfs_list_lock, flag);
++				list_del(&rq->cfs.bld_cfs_list);
++				list_add_tail(&rq->cfs.bld_cfs_list, &cfs_rq_head);
++				rq->cfs.pos = 2; last->pos = 1;
++				write_unlock_irqrestore(&cfs_list_lock, flag);
++			}
++		}
++	}
++}
++
++static void bld_track_load_deactivate(struct rq *rq, struct task_struct *p)
++{
++	unsigned long flag;
++	if (rt_task(p)) {
++		track_load_rt(rq, p);
++	} else {
++		if (rq->cfs.pos != 0) {
++			struct cfs_rq *first;
++			first = list_entry(cfs_rq_head.next, struct cfs_rq, bld_cfs_list);
++			if (rq->cfs.load.weight <= first->load.weight) {
++				write_lock_irqsave(&cfs_list_lock, flag);
++				list_del(&rq->cfs.bld_cfs_list);
++				list_add(&rq->cfs.bld_cfs_list, &cfs_rq_head);
++				rq->cfs.pos = 0; first->pos = 1;
++				write_unlock_irqrestore(&cfs_list_lock, flag);
++			}
++		}
++	}
++}
++#else
++static inline void bld_track_load_activate(struct rq *rq, struct task_struct *p)
++{
++}
++
++static inline void bld_track_load_deactivate(struct rq *rq, struct task_struct *p)
++{
++}
++#endif /* CONFIG_BLD */
+diff --git a/kernel/sched/core.c b/kernel/sched/core.c
+index d1f7149..c3236de 100644
+--- a/kernel/sched/core.c
++++ b/kernel/sched/core.c
+@@ -24,6 +24,8 @@
+  *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri
+  *  2007-11-29  RT balancing improvements by Steven Rostedt, Gregory Haskins,
+  *              Thomas Gleixner, Mike Kravetz
++ *  2012-Feb	The Barbershop Load Distribution (BLD) algorithm - an alternate
++ *		CPU load distribution technique for kernel scheduler by Rakib Mullick.
+  */
+ 
+ #include <linux/kasan.h>
+@@ -86,6 +88,7 @@
+ #include "sched.h"
+ #include "../workqueue_internal.h"
+ #include "../smpboot.h"
++#include "bld.h"
+ 
+ #define CREATE_TRACE_POINTS
+ #include <trace/events/sched.h>
+@@ -713,6 +716,8 @@ static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
+ 	if (!(flags & ENQUEUE_RESTORE))
+ 		sched_info_queued(rq, p);
+ 	p->sched_class->enqueue_task(rq, p, flags);
++	if (!dl_task(p))
++		bld_track_load_activate(rq, p);
+ }
+ 
+ static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
+@@ -721,6 +726,8 @@ static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
+ 	if (!(flags & DEQUEUE_SAVE))
+ 		sched_info_dequeued(rq, p);
+ 	p->sched_class->dequeue_task(rq, p, flags);
++	if (!dl_task(p))
++		bld_track_load_deactivate(rq, p);
+ }
+ 
+ void activate_task(struct rq *rq, struct task_struct *p, int flags)
+@@ -1515,8 +1522,16 @@ int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
+ {
+ 	lockdep_assert_held(&p->pi_lock);
+ 
+-	if (p->nr_cpus_allowed > 1)
++	if (p->nr_cpus_allowed > 1) {
++#ifndef	CONFIG_BLD
+ 		cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags);
++#else
++		if(dl_task(p))
++			cpu = dl_sched_class.select_task_rq(p, cpu, sd_flags, wake_flags);
++		else
++			cpu = bld_get_cpu(p, sd_flags, wake_flags);
++#endif
++	}
+ 
+ 	/*
+ 	 * In order not to call set_task_cpu() on a blocking task we need
+@@ -1706,7 +1721,11 @@ void scheduler_ipi(void)
+ 	 */
+ 	preempt_fold_need_resched();
+ 
++#ifndef	CONFIG_BLD
+ 	if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick())
++#else
++	if (llist_empty(&this_rq()->wake_list))
++#endif
+ 		return;
+ 
+ 	/*
+@@ -1728,13 +1747,16 @@ void scheduler_ipi(void)
+ 	/*
+ 	 * Check if someone kicked us for doing the nohz idle load balance.
+ 	 */
++#ifndef	CONFIG_BLD
+ 	if (unlikely(got_nohz_idle_kick())) {
+ 		this_rq()->idle_balance = 1;
+ 		raise_softirq_irqoff(SCHED_SOFTIRQ);
+ 	}
++#endif
+ 	irq_exit();
+ }
+ 
++#ifndef	CONFIG_BLD
+ static void ttwu_queue_remote(struct task_struct *p, int cpu)
+ {
+ 	struct rq *rq = cpu_rq(cpu);
+@@ -1747,6 +1769,13 @@ static void ttwu_queue_remote(struct task_struct *p, int cpu)
+ 	}
+ }
+ 
++#endif
++
++bool cpus_share_cache(int this_cpu, int that_cpu)
++{
++	return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
++}
++
+ void wake_up_if_idle(int cpu)
+ {
+ 	struct rq *rq = cpu_rq(cpu);
+@@ -1770,18 +1799,13 @@ void wake_up_if_idle(int cpu)
+ out:
+ 	rcu_read_unlock();
+ }
+-
+-bool cpus_share_cache(int this_cpu, int that_cpu)
+-{
+-	return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
+-}
+ #endif /* CONFIG_SMP */
+ 
+ static void ttwu_queue(struct task_struct *p, int cpu)
+ {
+ 	struct rq *rq = cpu_rq(cpu);
+ 
+-#if defined(CONFIG_SMP)
++#if defined(CONFIG_SMP) && !defined(CONFIG_BLD)
+ 	if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
+ 		sched_clock_cpu(cpu); /* sync clocks x-cpu */
+ 		ttwu_queue_remote(p, cpu);
+@@ -2292,7 +2316,7 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p)
+ 	 * Silence PROVE_RCU.
+ 	 */
+ 	raw_spin_lock_irqsave(&p->pi_lock, flags);
+-	set_task_cpu(p, cpu);
++	__set_task_cpu(p, cpu);
+ 	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
+ 
+ #ifdef CONFIG_SCHED_INFO
+@@ -2837,7 +2861,14 @@ void sched_exec(void)
+ 	int dest_cpu;
+ 
+ 	raw_spin_lock_irqsave(&p->pi_lock, flags);
++#ifndef	CONFIG_BLD
+ 	dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0);
++#else
++	if (dl_task(p))
++		dest_cpu = task_cpu(p);
++	else
++		dest_cpu = bld_get_cpu(p, SD_BALANCE_EXEC, 0);
++#endif
+ 	if (dest_cpu == smp_processor_id())
+ 		goto unlock;
+ 
+@@ -2926,8 +2957,10 @@ void scheduler_tick(void)
+ 
+ #ifdef CONFIG_SMP
+ 	rq->idle_balance = idle_cpu(cpu);
++#ifndef	CONFIG_BLD
+ 	trigger_load_balance(rq);
+ #endif
++#endif
+ 	rq_last_tick_reset(rq);
+ }
+ 
+@@ -7359,6 +7392,15 @@ void __init sched_init(void)
+ #endif
+ 		init_rq_hrtick(rq);
+ 		atomic_set(&rq->nr_iowait, 0);
++#ifdef CONFIG_BLD
++		INIT_LIST_HEAD(&rq->cfs.bld_cfs_list);
++		list_add_tail(&rq->cfs.bld_cfs_list, &cfs_rq_head);
++		rq->cfs.pos = 0;
++
++		INIT_LIST_HEAD(&rq->rt.bld_rt_list);
++		list_add_tail(&rq->rt.bld_rt_list, &rt_rq_head);
++		rq->rt.lowbit = INT_MAX;
++#endif
+ 	}
+ 
+ 	set_load_weight(&init_task);
+@@ -7399,6 +7441,9 @@ void __init sched_init(void)
+ 	init_sched_fair_class();
+ 
+ 	scheduler_running = 1;
++#ifdef	CONFIG_BLD
++	printk(KERN_INFO "BLD: An Alternate CPU load distributor activated.\n");
++#endif
+ }
+ 
+ #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
+diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
+index e7dd0ec..555572f 100644
+--- a/kernel/sched/fair.c
++++ b/kernel/sched/fair.c
+@@ -4746,6 +4746,7 @@ static void task_waking_fair(struct task_struct *p)
+ 	record_wakee(p);
+ }
+ 
++#ifndef	CONFIG_BLD
+ #ifdef CONFIG_FAIR_GROUP_SCHED
+ /*
+  * effective_load() calculates the load change as seen from the root_task_group
+@@ -5248,6 +5249,7 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f
+ 
+ 	return new_cpu;
+ }
++#endif	/* CONFIG_BLD */
+ 
+ /*
+  * Called immediately before a task is migrated to a new cpu; task_cpu(p) and
+@@ -5552,6 +5554,7 @@ idle:
+ 	 * further scheduler activity on it and we're being very careful to
+ 	 * re-start the picking loop.
+ 	 */
++#ifndef	CONFIG_BLD
+ 	lockdep_unpin_lock(&rq->lock);
+ 	new_tasks = idle_balance(rq);
+ 	lockdep_pin_lock(&rq->lock);
+@@ -5565,7 +5568,7 @@ idle:
+ 
+ 	if (new_tasks > 0)
+ 		goto again;
+-
++#endif
+ 	return NULL;
+ }
+ 
+@@ -6226,8 +6229,9 @@ static unsigned long task_h_load(struct task_struct *p)
+ }
+ #endif
+ 
+-/********** Helpers for find_busiest_group ************************/
++#ifndef	CONFIG_BLD
+ 
++/********** Helpers for find_busiest_group ************************/
+ enum group_type {
+ 	group_other = 0,
+ 	group_imbalanced,
+@@ -6318,6 +6322,7 @@ static inline int get_sd_load_idx(struct sched_domain *sd,
+ 
+ 	return load_idx;
+ }
++#endif	/* CONFIG_BLD	*/
+ 
+ static unsigned long scale_rt_capacity(int cpu)
+ {
+@@ -6426,6 +6431,7 @@ void update_group_capacity(struct sched_domain *sd, int cpu)
+ 	sdg->sgc->capacity = capacity;
+ }
+ 
++#ifndef	CONFIG_BLD
+ /*
+  * Check whether the capacity of the rq has been noticeably reduced by side
+  * activity. The imbalance_pct is used for the threshold.
+@@ -7659,6 +7665,8 @@ static inline int on_null_domain(struct rq *rq)
+ 	return unlikely(!rcu_dereference_sched(rq->sd));
+ }
+ 
++#endif	/* CONFIG_BLD */
++
+ #ifdef CONFIG_NO_HZ_COMMON
+ /*
+  * idle load balancing details
+@@ -7666,12 +7674,39 @@ static inline int on_null_domain(struct rq *rq)
+  *   needed, they will kick the idle load balancer, which then does idle
+  *   load balancing for all the idle CPUs.
+  */
++#ifndef	CONFIG_BLD
+ static struct {
+ 	cpumask_var_t idle_cpus_mask;
+ 	atomic_t nr_cpus;
+ 	unsigned long next_balance;     /* in jiffy units */
+ } nohz ____cacheline_aligned;
+ 
++static inline void nohz_balance_exit_idle(int cpu)
++{
++	if (unlikely(test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)))) {
++		/*
++		 * Completely isolated CPUs don't ever set, so we must test.
++		 */
++		if (likely(cpumask_test_cpu(cpu, nohz.idle_cpus_mask))) {
++			cpumask_clear_cpu(cpu, nohz.idle_cpus_mask);
++			atomic_dec(&nohz.nr_cpus);
++		}
++		clear_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu));
++	}
++}
++
++static int sched_ilb_notifier(struct notifier_block *nfb,
++					unsigned long action, void *hcpu)
++{
++	switch (action & ~CPU_TASKS_FROZEN) {
++	case CPU_DYING:
++		nohz_balance_exit_idle(smp_processor_id());
++		return NOTIFY_OK;
++	default:
++		return NOTIFY_DONE;
++	}
++}
++
+ static inline int find_new_ilb(void)
+ {
+ 	int ilb = cpumask_first(nohz.idle_cpus_mask);
+@@ -7709,20 +7744,7 @@ static void nohz_balancer_kick(void)
+ 	smp_send_reschedule(ilb_cpu);
+ 	return;
+ }
+-
+-static inline void nohz_balance_exit_idle(int cpu)
+-{
+-	if (unlikely(test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)))) {
+-		/*
+-		 * Completely isolated CPUs don't ever set, so we must test.
+-		 */
+-		if (likely(cpumask_test_cpu(cpu, nohz.idle_cpus_mask))) {
+-			cpumask_clear_cpu(cpu, nohz.idle_cpus_mask);
+-			atomic_dec(&nohz.nr_cpus);
+-		}
+-		clear_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu));
+-	}
+-}
++#endif	/* CONFIG_BLD */
+ 
+ static inline void set_cpu_sd_state_busy(void)
+ {
+@@ -7764,6 +7786,7 @@ unlock:
+  */
+ void nohz_balance_enter_idle(int cpu)
+ {
++#ifndef	CONFIG_BLD
+ 	/*
+ 	 * If this cpu is going down, then nothing needs to be done.
+ 	 */
+@@ -7782,23 +7805,10 @@ void nohz_balance_enter_idle(int cpu)
+ 	cpumask_set_cpu(cpu, nohz.idle_cpus_mask);
+ 	atomic_inc(&nohz.nr_cpus);
+ 	set_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu));
+-}
+-
+-static int sched_ilb_notifier(struct notifier_block *nfb,
+-					unsigned long action, void *hcpu)
+-{
+-	switch (action & ~CPU_TASKS_FROZEN) {
+-	case CPU_DYING:
+-		nohz_balance_exit_idle(smp_processor_id());
+-		return NOTIFY_OK;
+-	default:
+-		return NOTIFY_DONE;
+-	}
++#endif
+ }
+ #endif
+ 
+-static DEFINE_SPINLOCK(balancing);
+-
+ /*
+  * Scale the max load_balance interval with the number of CPUs in the system.
+  * This trades load-balance latency on larger machines for less cross talk.
+@@ -7808,6 +7818,9 @@ void update_max_interval(void)
+ 	max_load_balance_interval = HZ*num_online_cpus()/10;
+ }
+ 
++#ifndef	CONFIG_BLD
++static DEFINE_SPINLOCK(balancing);
++
+ /*
+  * It checks each scheduling domain to see if it is due to be balanced,
+  * and initiates a balancing operation if so.
+@@ -8095,6 +8108,7 @@ void trigger_load_balance(struct rq *rq)
+ 		nohz_balancer_kick();
+ #endif
+ }
++#endif	/* CONFIG_BLD */
+ 
+ static void rq_online_fair(struct rq *rq)
+ {
+@@ -8531,7 +8545,9 @@ const struct sched_class fair_sched_class = {
+ 	.put_prev_task		= put_prev_task_fair,
+ 
+ #ifdef CONFIG_SMP
++#ifndef	CONFIG_BLD
+ 	.select_task_rq		= select_task_rq_fair,
++#endif
+ 	.migrate_task_rq	= migrate_task_rq_fair,
+ 
+ 	.rq_online		= rq_online_fair,
+@@ -8593,6 +8609,7 @@ void show_numa_stats(struct task_struct *p, struct seq_file *m)
+ 
+ __init void init_sched_fair_class(void)
+ {
++#ifndef	CONFIG_BLD
+ #ifdef CONFIG_SMP
+ 	open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
+ 
+@@ -8602,5 +8619,5 @@ __init void init_sched_fair_class(void)
+ 	cpu_notifier(sched_ilb_notifier, 0);
+ #endif
+ #endif /* SMP */
+-
++#endif /* BLD */
+ }
+diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
+index ec4f538d..4462bed 100644
+--- a/kernel/sched/rt.c
++++ b/kernel/sched/rt.c
+@@ -1375,6 +1375,7 @@ static void yield_task_rt(struct rq *rq)
+ #ifdef CONFIG_SMP
+ static int find_lowest_rq(struct task_struct *task);
+ 
++#ifndef	CONFIG_BLD
+ static int
+ select_task_rq_rt(struct task_struct *p, int cpu, int sd_flag, int flags)
+ {
+@@ -1430,6 +1431,7 @@ select_task_rq_rt(struct task_struct *p, int cpu, int sd_flag, int flags)
+ out:
+ 	return cpu;
+ }
++#endif	/* CONFIG_BLD */
+ 
+ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
+ {
+@@ -2335,7 +2337,9 @@ const struct sched_class rt_sched_class = {
+ 	.put_prev_task		= put_prev_task_rt,
+ 
+ #ifdef CONFIG_SMP
++#ifndef	CONFIG_BLD
+ 	.select_task_rq		= select_task_rq_rt,
++#endif
+ 
+ 	.set_cpus_allowed       = set_cpus_allowed_common,
+ 	.rq_online              = rq_online_rt,
+diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
+index ec2e8d2..aaab735 100644
+--- a/kernel/sched/sched.h
++++ b/kernel/sched/sched.h
+@@ -408,9 +408,8 @@ struct cfs_rq {
+ #endif /* CONFIG_FAIR_GROUP_SCHED */
+ #endif /* CONFIG_SMP */
+ 
+-#ifdef CONFIG_FAIR_GROUP_SCHED
+ 	struct rq *rq;	/* cpu runqueue to which this cfs_rq is attached */
+-
++#ifdef CONFIG_FAIR_GROUP_SCHED
+ 	/*
+ 	 * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
+ 	 * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
+@@ -434,6 +433,11 @@ struct cfs_rq {
+ 	struct list_head throttled_list;
+ #endif /* CONFIG_CFS_BANDWIDTH */
+ #endif /* CONFIG_FAIR_GROUP_SCHED */
++
++#ifdef CONFIG_BLD
++	struct list_head bld_cfs_list;
++	char pos;
++#endif
+ };
+ 
+ static inline int rt_bandwidth_enabled(void)
+@@ -479,12 +483,16 @@ struct rt_rq {
+ 	/* Nests inside the rq lock: */
+ 	raw_spinlock_t rt_runtime_lock;
+ 
++	struct rq *rq;
+ #ifdef CONFIG_RT_GROUP_SCHED
+ 	unsigned long rt_nr_boosted;
+ 
+-	struct rq *rq;
+ 	struct task_group *tg;
+ #endif
++#ifdef CONFIG_BLD
++	struct list_head bld_rt_list;
++	int lowbit;
++#endif
+ };
+ 
+ /* Deadline class' related fields in a runqueue */
diff --git a/patches/BLD-4.7.patch b/patches_disabled/BLD-4.8.patch
similarity index 87%
rename from patches/BLD-4.7.patch
rename to patches_disabled/BLD-4.8.patch
index e252c45..354d96e 100644
--- a/patches/BLD-4.7.patch
+++ b/patches_disabled/BLD-4.8.patch
@@ -1,9 +1,5 @@
- BLD changes for Linux kernel version 4.7
-
----
-
 diff --git a/init/Kconfig b/init/Kconfig
-index c02d897..edf8697 100644
+index cac3f09..4e49d16 100644
 --- a/init/Kconfig
 +++ b/init/Kconfig
 @@ -36,6 +36,15 @@ config BROKEN_ON_SMP
@@ -244,7 +240,7 @@ index 0000000..f1f9fba
 +}
 +#endif /* CONFIG_BLD */
 diff --git a/kernel/sched/core.c b/kernel/sched/core.c
-index 97ee9ac..b2ddabc 100644
+index 44817c6..f0f3321 100644
 --- a/kernel/sched/core.c
 +++ b/kernel/sched/core.c
 @@ -24,6 +24,8 @@
@@ -256,7 +252,7 @@ index 97ee9ac..b2ddabc 100644
   */
  
  #include <linux/kasan.h>
-@@ -86,6 +88,7 @@
+@@ -87,6 +89,7 @@
  #include "sched.h"
  #include "../workqueue_internal.h"
  #include "../smpboot.h"
@@ -264,7 +260,7 @@ index 97ee9ac..b2ddabc 100644
  
  #define CREATE_TRACE_POINTS
  #include <trace/events/sched.h>
-@@ -750,6 +753,8 @@ static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
+@@ -751,6 +754,8 @@ static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
  	if (!(flags & ENQUEUE_RESTORE))
  		sched_info_queued(rq, p);
  	p->sched_class->enqueue_task(rq, p, flags);
@@ -273,7 +269,7 @@ index 97ee9ac..b2ddabc 100644
  }
  
  static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
-@@ -758,6 +763,8 @@ static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
+@@ -759,6 +764,8 @@ static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
  	if (!(flags & DEQUEUE_SAVE))
  		sched_info_dequeued(rq, p);
  	p->sched_class->dequeue_task(rq, p, flags);
@@ -282,7 +278,7 @@ index 97ee9ac..b2ddabc 100644
  }
  
  void activate_task(struct rq *rq, struct task_struct *p, int flags)
-@@ -1587,11 +1594,17 @@ int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
+@@ -1588,11 +1595,17 @@ int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
  {
  	lockdep_assert_held(&p->pi_lock);
  
@@ -301,7 +297,7 @@ index 97ee9ac..b2ddabc 100644
  	/*
  	 * In order not to call set_task_cpu() on a blocking task we need
  	 * to rely on ttwu() to place the task on a valid ->cpus_allowed
-@@ -1794,7 +1807,11 @@ void scheduler_ipi(void)
+@@ -1795,7 +1808,11 @@ void scheduler_ipi(void)
  	 */
  	preempt_fold_need_resched();
  
@@ -313,7 +309,7 @@ index 97ee9ac..b2ddabc 100644
  		return;
  
  	/*
-@@ -1816,13 +1833,16 @@ void scheduler_ipi(void)
+@@ -1817,13 +1834,16 @@ void scheduler_ipi(void)
  	/*
  	 * Check if someone kicked us for doing the nohz idle load balance.
  	 */
@@ -330,7 +326,7 @@ index 97ee9ac..b2ddabc 100644
  static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags)
  {
  	struct rq *rq = cpu_rq(cpu);
-@@ -1836,6 +1856,7 @@ static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags)
+@@ -1837,6 +1857,7 @@ static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags)
  			trace_sched_wake_idle_without_ipi(cpu);
  	}
  }
@@ -338,7 +334,7 @@ index 97ee9ac..b2ddabc 100644
  
  void wake_up_if_idle(int cpu)
  {
-@@ -1872,7 +1893,7 @@ static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags)
+@@ -1873,7 +1894,7 @@ static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags)
  	struct rq *rq = cpu_rq(cpu);
  	struct pin_cookie cookie;
  
@@ -347,16 +343,7 @@ index 97ee9ac..b2ddabc 100644
  	if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
  		sched_clock_cpu(cpu); /* sync clocks x-cpu */
  		ttwu_queue_remote(p, cpu, wake_flags);
-@@ -2394,7 +2415,7 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p)
- 	 * Silence PROVE_RCU.
- 	 */
- 	raw_spin_lock_irqsave(&p->pi_lock, flags);
--	set_task_cpu(p, cpu);
-+	__set_task_cpu(p, cpu);
- 	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
- 
- #ifdef CONFIG_SCHED_INFO
-@@ -2941,7 +2962,14 @@ void sched_exec(void)
+@@ -2971,7 +2992,14 @@ void sched_exec(void)
  	int dest_cpu;
  
  	raw_spin_lock_irqsave(&p->pi_lock, flags);
@@ -371,7 +358,7 @@ index 97ee9ac..b2ddabc 100644
  	if (dest_cpu == smp_processor_id())
  		goto unlock;
  
-@@ -3030,8 +3058,10 @@ void scheduler_tick(void)
+@@ -3078,8 +3106,10 @@ void scheduler_tick(void)
  
  #ifdef CONFIG_SMP
  	rq->idle_balance = idle_cpu(cpu);
@@ -382,7 +369,7 @@ index 97ee9ac..b2ddabc 100644
  	rq_last_tick_reset(rq);
  }
  
-@@ -7262,7 +7292,9 @@ int sched_cpu_dying(unsigned int cpu)
+@@ -7313,7 +7343,9 @@ int sched_cpu_dying(unsigned int cpu)
  	raw_spin_unlock_irqrestore(&rq->lock, flags);
  	calc_load_migrate(rq);
  	update_max_interval();
@@ -392,7 +379,7 @@ index 97ee9ac..b2ddabc 100644
  	hrtick_clear(rq);
  	return 0;
  }
-@@ -7468,6 +7500,15 @@ void __init sched_init(void)
+@@ -7519,6 +7551,15 @@ void __init sched_init(void)
  #endif /* CONFIG_SMP */
  		init_rq_hrtick(rq);
  		atomic_set(&rq->nr_iowait, 0);
@@ -408,7 +395,7 @@ index 97ee9ac..b2ddabc 100644
  	}
  
  	set_load_weight(&init_task);
-@@ -7510,6 +7551,9 @@ void __init sched_init(void)
+@@ -7561,6 +7602,9 @@ void __init sched_init(void)
  	init_schedstats();
  
  	scheduler_running = 1;
@@ -419,10 +406,10 @@ index 97ee9ac..b2ddabc 100644
  
  #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
 diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
-index c8c5d2d..5b694b3 100644
+index 039de34..f823e5b 100644
 --- a/kernel/sched/fair.c
 +++ b/kernel/sched/fair.c
-@@ -4880,6 +4880,7 @@ static unsigned long cpu_avg_load_per_task(int cpu)
+@@ -4924,6 +4924,7 @@ static unsigned long cpu_avg_load_per_task(int cpu)
  	return 0;
  }
  
@@ -430,7 +417,7 @@ index c8c5d2d..5b694b3 100644
  #ifdef CONFIG_FAIR_GROUP_SCHED
  /*
   * effective_load() calculates the load change as seen from the root_task_group
-@@ -5411,6 +5412,7 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f
+@@ -5455,6 +5456,7 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f
  
  	return new_cpu;
  }
@@ -438,7 +425,7 @@ index c8c5d2d..5b694b3 100644
  
  /*
   * Called immediately before a task is migrated to a new cpu; task_cpu(p) and
-@@ -5741,6 +5743,7 @@ idle:
+@@ -5785,6 +5787,7 @@ idle:
  	 * further scheduler activity on it and we're being very careful to
  	 * re-start the picking loop.
  	 */
@@ -446,7 +433,7 @@ index c8c5d2d..5b694b3 100644
  	lockdep_unpin_lock(&rq->lock, cookie);
  	new_tasks = idle_balance(rq);
  	lockdep_repin_lock(&rq->lock, cookie);
-@@ -5754,7 +5757,7 @@ idle:
+@@ -5798,7 +5801,7 @@ idle:
  
  	if (new_tasks > 0)
  		goto again;
@@ -455,7 +442,7 @@ index c8c5d2d..5b694b3 100644
  	return NULL;
  }
  
-@@ -6415,8 +6418,9 @@ static unsigned long task_h_load(struct task_struct *p)
+@@ -6459,8 +6462,9 @@ static unsigned long task_h_load(struct task_struct *p)
  }
  #endif
  
@@ -466,7 +453,7 @@ index c8c5d2d..5b694b3 100644
  enum group_type {
  	group_other = 0,
  	group_imbalanced,
-@@ -6507,6 +6511,7 @@ static inline int get_sd_load_idx(struct sched_domain *sd,
+@@ -6551,6 +6555,7 @@ static inline int get_sd_load_idx(struct sched_domain *sd,
  
  	return load_idx;
  }
@@ -474,7 +461,7 @@ index c8c5d2d..5b694b3 100644
  
  static unsigned long scale_rt_capacity(int cpu)
  {
-@@ -6615,6 +6620,7 @@ void update_group_capacity(struct sched_domain *sd, int cpu)
+@@ -6659,6 +6664,7 @@ void update_group_capacity(struct sched_domain *sd, int cpu)
  	sdg->sgc->capacity = capacity;
  }
  
@@ -482,7 +469,7 @@ index c8c5d2d..5b694b3 100644
  /*
   * Check whether the capacity of the rq has been noticeably reduced by side
   * activity. The imbalance_pct is used for the threshold.
-@@ -7848,6 +7854,7 @@ static inline int on_null_domain(struct rq *rq)
+@@ -7892,6 +7898,7 @@ static inline int on_null_domain(struct rq *rq)
  {
  	return unlikely(!rcu_dereference_sched(rq->sd));
  }
@@ -490,7 +477,7 @@ index c8c5d2d..5b694b3 100644
  
  #ifdef CONFIG_NO_HZ_COMMON
  /*
-@@ -7856,12 +7863,39 @@ static inline int on_null_domain(struct rq *rq)
+@@ -7900,12 +7907,39 @@ static inline int on_null_domain(struct rq *rq)
   *   needed, they will kick the idle load balancer, which then does idle
   *   load balancing for all the idle CPUs.
   */
@@ -530,7 +517,7 @@ index c8c5d2d..5b694b3 100644
  static inline int find_new_ilb(void)
  {
  	int ilb = cpumask_first(nohz.idle_cpus_mask);
-@@ -7900,20 +7934,6 @@ static void nohz_balancer_kick(void)
+@@ -7944,20 +7978,6 @@ static void nohz_balancer_kick(void)
  	return;
  }
  
@@ -551,7 +538,7 @@ index c8c5d2d..5b694b3 100644
  static inline void set_cpu_sd_state_busy(void)
  {
  	struct sched_domain *sd;
-@@ -7930,6 +7950,8 @@ static inline void set_cpu_sd_state_busy(void)
+@@ -7974,6 +7994,8 @@ static inline void set_cpu_sd_state_busy(void)
  unlock:
  	rcu_read_unlock();
  }
@@ -560,7 +547,7 @@ index c8c5d2d..5b694b3 100644
  
  void set_cpu_sd_state_idle(void)
  {
-@@ -7954,6 +7976,7 @@ unlock:
+@@ -7998,6 +8020,7 @@ unlock:
   */
  void nohz_balance_enter_idle(int cpu)
  {
@@ -568,7 +555,7 @@ index c8c5d2d..5b694b3 100644
  	/*
  	 * If this cpu is going down, then nothing needs to be done.
  	 */
-@@ -7972,10 +7995,8 @@ void nohz_balance_enter_idle(int cpu)
+@@ -8016,10 +8039,8 @@ void nohz_balance_enter_idle(int cpu)
  	cpumask_set_cpu(cpu, nohz.idle_cpus_mask);
  	atomic_inc(&nohz.nr_cpus);
  	set_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu));
@@ -580,7 +567,7 @@ index c8c5d2d..5b694b3 100644
  
  /*
   * Scale the max load_balance interval with the number of CPUs in the system.
-@@ -7986,6 +8007,9 @@ void update_max_interval(void)
+@@ -8030,6 +8051,9 @@ void update_max_interval(void)
  	max_load_balance_interval = HZ*num_online_cpus()/10;
  }
  
@@ -590,7 +577,7 @@ index c8c5d2d..5b694b3 100644
  /*
   * It checks each scheduling domain to see if it is due to be balanced,
   * and initiates a balancing operation if so.
-@@ -8273,6 +8297,7 @@ void trigger_load_balance(struct rq *rq)
+@@ -8317,6 +8341,7 @@ void trigger_load_balance(struct rq *rq)
  		nohz_balancer_kick();
  #endif
  }
@@ -598,7 +585,7 @@ index c8c5d2d..5b694b3 100644
  
  static void rq_online_fair(struct rq *rq)
  {
-@@ -8288,7 +8313,6 @@ static void rq_offline_fair(struct rq *rq)
+@@ -8332,7 +8357,6 @@ static void rq_offline_fair(struct rq *rq)
  	/* Ensure any throttled groups are reachable by pick_next_task */
  	unthrottle_offline_cfs_rqs(rq);
  }
@@ -606,7 +593,7 @@ index c8c5d2d..5b694b3 100644
  #endif /* CONFIG_SMP */
  
  /*
-@@ -8716,7 +8740,9 @@ const struct sched_class fair_sched_class = {
+@@ -8791,7 +8815,9 @@ const struct sched_class fair_sched_class = {
  	.put_prev_task		= put_prev_task_fair,
  
  #ifdef CONFIG_SMP
@@ -616,7 +603,7 @@ index c8c5d2d..5b694b3 100644
  	.migrate_task_rq	= migrate_task_rq_fair,
  
  	.rq_online		= rq_online_fair,
-@@ -8777,6 +8803,7 @@ void show_numa_stats(struct task_struct *p, struct seq_file *m)
+@@ -8852,6 +8878,7 @@ void show_numa_stats(struct task_struct *p, struct seq_file *m)
  
  __init void init_sched_fair_class(void)
  {
@@ -624,7 +611,7 @@ index c8c5d2d..5b694b3 100644
  #ifdef CONFIG_SMP
  	open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
  
-@@ -8785,5 +8812,5 @@ __init void init_sched_fair_class(void)
+@@ -8860,5 +8887,5 @@ __init void init_sched_fair_class(void)
  	zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT);
  #endif
  #endif /* SMP */
@@ -662,10 +649,10 @@ index d5690b7..6f3589e 100644
  	.set_cpus_allowed       = set_cpus_allowed_common,
  	.rq_online              = rq_online_rt,
 diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
-index 898c0d2..720d524 100644
+index c64fc51..a1d329b 100644
 --- a/kernel/sched/sched.h
 +++ b/kernel/sched/sched.h
-@@ -415,9 +415,8 @@ struct cfs_rq {
+@@ -416,9 +416,8 @@ struct cfs_rq {
  #endif /* CONFIG_FAIR_GROUP_SCHED */
  #endif /* CONFIG_SMP */
  
@@ -676,7 +663,7 @@ index 898c0d2..720d524 100644
  	/*
  	 * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
  	 * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
-@@ -441,6 +440,11 @@ struct cfs_rq {
+@@ -442,6 +441,11 @@ struct cfs_rq {
  	struct list_head throttled_list;
  #endif /* CONFIG_CFS_BANDWIDTH */
  #endif /* CONFIG_FAIR_GROUP_SCHED */
@@ -688,7 +675,7 @@ index 898c0d2..720d524 100644
  };
  
  static inline int rt_bandwidth_enabled(void)
-@@ -486,12 +490,16 @@ struct rt_rq {
+@@ -487,12 +491,16 @@ struct rt_rq {
  	/* Nests inside the rq lock: */
  	raw_spinlock_t rt_runtime_lock;
  
diff --git a/patches/uksm-0.1.2.5-for-v4.7.patch b/patches_disabled/uksm-0.1.2.5-for-v4.8.patch
similarity index 97%
rename from patches/uksm-0.1.2.5-for-v4.7.patch
rename to patches_disabled/uksm-0.1.2.5-for-v4.8.patch
index 4ba4cbb..d308f87 100644
--- a/patches/uksm-0.1.2.5-for-v4.7.patch
+++ b/patches_disabled/uksm-0.1.2.5-for-v4.8.patch
@@ -78,7 +78,7 @@ index 0000000..8fce86f
 +2015-04-22 UKSM 0.1.2.4 Fix a race condition that can sometimes trigger anonying warnings.
 +2016-09-10 UKSM 0.1.2.5 Fix a bug in dedup ratio calculation.
 diff --git a/fs/exec.c b/fs/exec.c
-index 887c1c9..2bee16e 100644
+index 6fcfb3f..ef87e0f 100644
 --- a/fs/exec.c
 +++ b/fs/exec.c
 @@ -19,7 +19,7 @@
@@ -98,7 +98,7 @@ index 887c1c9..2bee16e 100644
  
  #include <asm/uaccess.h>
  #include <asm/mmu_context.h>
-@@ -1273,6 +1274,7 @@ void setup_new_exec(struct linux_binprm * bprm)
+@@ -1309,6 +1310,7 @@ void setup_new_exec(struct linux_binprm * bprm)
  	/* An exec changes our domain. We are no longer part of the thread
  	   group */
  	current->self_exec_id++;
@@ -107,7 +107,7 @@ index 887c1c9..2bee16e 100644
  	do_close_on_exec(current->files);
  }
 diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c
-index 8372046..82aa2f4 100644
+index b9a8c81..9765269 100644
 --- a/fs/proc/meminfo.c
 +++ b/fs/proc/meminfo.c
 @@ -89,6 +89,9 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
@@ -120,9 +120,9 @@ index 8372046..82aa2f4 100644
  #ifdef CONFIG_QUICKLIST
  		"Quicklists:     %8lu kB\n"
  #endif
-@@ -147,6 +150,9 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
+@@ -149,6 +152,9 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
  		K(global_page_state(NR_SLAB_UNRECLAIMABLE)),
- 		global_page_state(NR_KERNEL_STACK) * THREAD_SIZE / 1024,
+ 		global_page_state(NR_KERNEL_STACK_KB),
  		K(global_page_state(NR_PAGETABLE)),
 +#ifdef CONFIG_UKSM
 +		K(global_page_state(NR_UKSM_ZERO_PAGES)),
@@ -171,7 +171,7 @@ index d4458b6..172ceb9 100644
  
  static inline unsigned long my_zero_pfn(unsigned long addr)
 diff --git a/include/linux/ksm.h b/include/linux/ksm.h
-index 7ae216a..06861d8 100644
+index 481c8c4..5329b23 100644
 --- a/include/linux/ksm.h
 +++ b/include/linux/ksm.h
 @@ -19,21 +19,6 @@ struct mem_cgroup;
@@ -196,7 +196,7 @@ index 7ae216a..06861d8 100644
  
  static inline struct stable_node *page_stable_node(struct page *page)
  {
-@@ -64,6 +49,33 @@ struct page *ksm_might_need_to_copy(struct page *page,
+@@ -63,6 +48,33 @@ struct page *ksm_might_need_to_copy(struct page *page,
  int rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc);
  void ksm_migrate_page(struct page *newpage, struct page *oldpage);
  
@@ -230,7 +230,7 @@ index 7ae216a..06861d8 100644
  #else  /* !CONFIG_KSM */
  
  static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm)
-@@ -106,4 +118,6 @@ static inline void ksm_migrate_page(struct page *newpage, struct page *oldpage)
+@@ -105,4 +117,6 @@ static inline void ksm_migrate_page(struct page *newpage, struct page *oldpage)
  #endif /* CONFIG_MMU */
  #endif /* !CONFIG_KSM */
  
@@ -238,10 +238,10 @@ index 7ae216a..06861d8 100644
 +
  #endif /* __LINUX_KSM_H */
 diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
-index ca3e517..ae62e7d1 100644
+index 903200f..6c7d900 100644
 --- a/include/linux/mm_types.h
 +++ b/include/linux/mm_types.h
-@@ -357,6 +357,9 @@ struct vm_area_struct {
+@@ -358,6 +358,9 @@ struct vm_area_struct {
  	struct mempolicy *vm_policy;	/* NUMA policy for the VMA */
  #endif
  	struct vm_userfaultfd_ctx vm_userfaultfd_ctx;
@@ -252,20 +252,20 @@ index ca3e517..ae62e7d1 100644
  
  struct core_thread {
 diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
-index 02069c2..f7cce50 100644
+index 7f2ae99..89f7dd8 100644
 --- a/include/linux/mmzone.h
 +++ b/include/linux/mmzone.h
-@@ -153,6 +153,9 @@ enum zone_stat_item {
- 	WORKINGSET_NODERECLAIM,
- 	NR_ANON_TRANSPARENT_HUGEPAGES,
+@@ -138,6 +138,9 @@ enum zone_stat_item {
+ 	NUMA_OTHER,		/* allocation from other node */
+ #endif
  	NR_FREE_CMA_PAGES,
 +#ifdef CONFIG_UKSM
 +	NR_UKSM_ZERO_PAGES,
 +#endif
  	NR_VM_ZONE_STAT_ITEMS };
  
- /*
-@@ -817,7 +820,7 @@ static inline int is_highmem_idx(enum zone_type idx)
+ enum node_stat_item {
+@@ -869,7 +872,7 @@ static inline int is_highmem_idx(enum zone_type idx)
  }
  
  /**
@@ -513,10 +513,10 @@ index 0000000..825f05e
 +#endif /* !CONFIG_UKSM */
 +#endif /* __LINUX_UKSM_H */
 diff --git a/kernel/fork.c b/kernel/fork.c
-index aea4f4d..f93e114 100644
+index beb3172..569893a 100644
 --- a/kernel/fork.c
 +++ b/kernel/fork.c
-@@ -459,7 +459,7 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
+@@ -457,7 +457,7 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
  				goto fail_nomem;
  			charge = len;
  		}
@@ -525,7 +525,7 @@ index aea4f4d..f93e114 100644
  		if (!tmp)
  			goto fail_nomem;
  		*tmp = *mpnt;
-@@ -512,7 +512,7 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
+@@ -510,7 +510,7 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
  		__vma_link_rb(mm, tmp, rb_link, rb_parent);
  		rb_link = &tmp->vm_rb.rb_right;
  		rb_parent = &tmp->vm_rb;
@@ -535,17 +535,17 @@ index aea4f4d..f93e114 100644
  		retval = copy_page_range(mm, oldmm, mpnt);
  
 diff --git a/lib/Makefile b/lib/Makefile
-index ff6a7a6..ac0bb55 100644
+index 5dc77a8..b63a823 100644
 --- a/lib/Makefile
 +++ b/lib/Makefile
-@@ -20,7 +20,7 @@ KCOV_INSTRUMENT_dynamic_debug.o := n
- KCOV_INSTRUMENT_hweight.o := n
+@@ -17,7 +17,7 @@ KCOV_INSTRUMENT_debugobjects.o := n
+ KCOV_INSTRUMENT_dynamic_debug.o := n
  
  lib-y := ctype.o string.o vsprintf.o cmdline.o \
 -	 rbtree.o radix-tree.o dump_stack.o timerqueue.o\
 +	 rbtree.o radix-tree.o sradix-tree.o dump_stack.o timerqueue.o\
  	 idr.o int_sqrt.o extable.o \
- 	 sha1.o md5.o irq_regs.o argv_split.o \
+ 	 sha1.o chacha20.o md5.o irq_regs.o argv_split.o \
  	 flex_proportions.o ratelimit.o show_mem.o \
 diff --git a/lib/sradix-tree.c b/lib/sradix-tree.c
 new file mode 100644
@@ -1030,10 +1030,10 @@ index 0000000..8d06329
 +	return 0;
 +}
 diff --git a/mm/Kconfig b/mm/Kconfig
-index 3e2daef..165b60e 100644
+index be0ee11..64fd3bc 100644
 --- a/mm/Kconfig
 +++ b/mm/Kconfig
-@@ -332,6 +332,32 @@ config KSM
+@@ -340,6 +340,32 @@ config KSM
  	  See Documentation/vm/ksm.txt for more information: KSM is inactive
  	  until a program has madvised that an area is MADV_MERGEABLE, and
  	  root has set /sys/kernel/mm/ksm/run to 1 (if CONFIG_SYSFS is set).
@@ -1067,10 +1067,10 @@ index 3e2daef..165b60e 100644
  config DEFAULT_MMAP_MIN_ADDR
          int "Low address space to protect from user allocation"
 diff --git a/mm/Makefile b/mm/Makefile
-index 78c6f7d..7e7cd8a 100644
+index 2ca1faf..980c8dd 100644
 --- a/mm/Makefile
 +++ b/mm/Makefile
-@@ -63,7 +63,8 @@ obj-$(CONFIG_SPARSEMEM)	+= sparse.o
+@@ -66,7 +66,8 @@ obj-$(CONFIG_SPARSEMEM)	+= sparse.o
  obj-$(CONFIG_SPARSEMEM_VMEMMAP) += sparse-vmemmap.o
  obj-$(CONFIG_SLOB) += slob.o
  obj-$(CONFIG_MMU_NOTIFIER) += mmu_notifier.o
@@ -1081,10 +1081,10 @@ index 78c6f7d..7e7cd8a 100644
  obj-$(CONFIG_SLAB) += slab.o
  obj-$(CONFIG_SLUB) += slub.o
 diff --git a/mm/memory.c b/mm/memory.c
-index 9e04681..02200d3 100644
+index 793fe0f..0464507 100644
 --- a/mm/memory.c
 +++ b/mm/memory.c
-@@ -124,6 +124,28 @@ unsigned long highest_memmap_pfn __read_mostly;
+@@ -124,6 +124,25 @@ unsigned long highest_memmap_pfn __read_mostly;
  
  EXPORT_SYMBOL(zero_pfn);
  
@@ -1095,14 +1095,11 @@ index 9e04681..02200d3 100644
 +
 +static int __init setup_uksm_zero_page(void)
 +{
-+	unsigned long addr;
-+	addr = __get_free_pages(GFP_KERNEL | __GFP_ZERO, 0);
-+	if (!addr)
++	empty_uksm_zero_page = alloc_pages(__GFP_ZERO & ~__GFP_MOVABLE, 0);
++	if (!empty_uksm_zero_page)
 +		panic("Oh boy, that early out of memory?");
 +
-+	empty_uksm_zero_page = virt_to_page((void *) addr);
 +	SetPageReserved(empty_uksm_zero_page);
-+
 +	uksm_zero_pfn = page_to_pfn(empty_uksm_zero_page);
 +
 +	return 0;
@@ -1113,7 +1110,7 @@ index 9e04681..02200d3 100644
  /*
   * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
   */
-@@ -135,6 +157,7 @@ static int __init init_zero_pfn(void)
+@@ -135,6 +154,7 @@ static int __init init_zero_pfn(void)
  core_initcall(init_zero_pfn);
  
  
@@ -1121,7 +1118,7 @@ index 9e04681..02200d3 100644
  #if defined(SPLIT_RSS_COUNTING)
  
  void sync_mm_rss(struct mm_struct *mm)
-@@ -905,6 +928,11 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
+@@ -914,6 +934,11 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  		get_page(page);
  		page_dup_rmap(page, false);
  		rss[mm_counter(page)]++;
@@ -1133,7 +1130,7 @@ index 9e04681..02200d3 100644
  	}
  
  out_set_pte:
-@@ -1138,8 +1166,10 @@ again:
+@@ -1148,8 +1173,10 @@ again:
  			ptent = ptep_get_and_clear_full(mm, addr, pte,
  							tlb->fullmm);
  			tlb_remove_tlb_entry(tlb, pte, addr);
@@ -1145,7 +1142,7 @@ index 9e04681..02200d3 100644
  
  			if (!PageAnon(page)) {
  				if (pte_dirty(ptent)) {
-@@ -1995,8 +2025,10 @@ static inline void cow_user_page(struct page *dst, struct page *src, unsigned lo
+@@ -2010,8 +2037,10 @@ static inline void cow_user_page(struct page *dst, struct page *src, unsigned lo
  			clear_page(kaddr);
  		kunmap_atomic(kaddr);
  		flush_dcache_page(dst);
@@ -1157,15 +1154,15 @@ index 9e04681..02200d3 100644
  }
  
  static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
-@@ -2141,6 +2173,7 @@ static int wp_page_copy(struct mm_struct *mm, struct vm_area_struct *vma,
- 		new_page = alloc_zeroed_user_highpage_movable(vma, address);
+@@ -2154,6 +2183,7 @@ static int wp_page_copy(struct fault_env *fe, pte_t orig_pte,
+ 		new_page = alloc_zeroed_user_highpage_movable(vma, fe->address);
  		if (!new_page)
  			goto oom;
 +		uksm_cow_pte(vma, orig_pte);
  	} else {
- 		new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
- 		if (!new_page)
-@@ -2166,7 +2199,9 @@ static int wp_page_copy(struct mm_struct *mm, struct vm_area_struct *vma,
+ 		new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
+ 				fe->address);
+@@ -2180,7 +2210,9 @@ static int wp_page_copy(struct fault_env *fe, pte_t orig_pte,
  						mm_counter_file(old_page));
  				inc_mm_counter_fast(mm, MM_ANONPAGES);
  			}
@@ -1174,12 +1171,12 @@ index 9e04681..02200d3 100644
 +			uksm_unmap_zero_page(orig_pte);
  			inc_mm_counter_fast(mm, MM_ANONPAGES);
  		}
- 		flush_cache_page(vma, address, pte_pfn(orig_pte));
+ 		flush_cache_page(vma, fe->address, pte_pfn(orig_pte));
 diff --git a/mm/mmap.c b/mm/mmap.c
-index de2c176..ce60715 100644
+index ca9d91b..cf565b7 100644
 --- a/mm/mmap.c
 +++ b/mm/mmap.c
-@@ -43,6 +43,7 @@
+@@ -44,6 +44,7 @@
  #include <linux/userfaultfd_k.h>
  #include <linux/moduleparam.h>
  #include <linux/pkeys.h>
@@ -1187,7 +1184,7 @@ index de2c176..ce60715 100644
  
  #include <asm/uaccess.h>
  #include <asm/cacheflush.h>
-@@ -164,6 +165,7 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
+@@ -165,6 +166,7 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
  	if (vma->vm_file)
  		fput(vma->vm_file);
  	mpol_put(vma_policy(vma));
@@ -1206,13 +1203,13 @@ index de2c176..ce60715 100644
 +	uksm_remove_vma(vma);
 +
  	if (next && !insert) {
- 		struct vm_area_struct *exporter = NULL;
+ 		struct vm_area_struct *exporter = NULL, *importer = NULL;
  
 +		uksm_remove_vma(next);
  		if (end >= next->vm_end) {
  			/*
  			 * vma expands, overlapping all the next, and
-@@ -725,6 +734,7 @@ again:			remove_next = 1 + (end > next->vm_end);
+@@ -733,6 +742,7 @@ again:
  		end_changed = true;
  	}
  	vma->vm_pgoff = pgoff;
@@ -1220,14 +1217,13 @@ index de2c176..ce60715 100644
  	if (adjust_next) {
  		next->vm_start += adjust_next << PAGE_SHIFT;
  		next->vm_pgoff += adjust_next;
-@@ -795,16 +805,22 @@ again:			remove_next = 1 + (end > next->vm_end);
- 		 * up the code too much to do both in one go.
- 		 */
- 		next = vma->vm_next;
--		if (remove_next == 2)
-+		if (remove_next == 2) {
+@@ -806,16 +816,21 @@ again:
+ 		if (remove_next == 2) {
+ 			remove_next = 1;
+ 			end = next->vm_end;
 +			uksm_remove_vma(next);
  			goto again;
+-		}
 -		else if (next)
 +		} else if (next) {
  			vma_gap_update(next);
@@ -1246,7 +1242,7 @@ index de2c176..ce60715 100644
  	validate_mm(mm);
  
  	return 0;
-@@ -1196,6 +1212,9 @@ unsigned long do_mmap(struct file *file, unsigned long addr,
+@@ -1207,6 +1222,9 @@ unsigned long do_mmap(struct file *file, unsigned long addr,
  	vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) |
  			mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
  
@@ -1256,7 +1252,7 @@ index de2c176..ce60715 100644
  	if (flags & MAP_LOCKED)
  		if (!can_do_mlock())
  			return -EPERM;
-@@ -1534,6 +1553,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
+@@ -1545,6 +1563,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
  			allow_write_access(file);
  	}
  	file = vma->vm_file;
@@ -1264,7 +1260,7 @@ index de2c176..ce60715 100644
  out:
  	perf_event_mmap(vma);
  
-@@ -1575,6 +1595,7 @@ allow_write_and_free_vma:
+@@ -1586,6 +1605,7 @@ allow_write_and_free_vma:
  	if (vm_flags & VM_DENYWRITE)
  		allow_write_access(file);
  free_vma:
@@ -1272,7 +1268,7 @@ index de2c176..ce60715 100644
  	kmem_cache_free(vm_area_cachep, vma);
  unacct_error:
  	if (charged)
-@@ -2369,6 +2390,8 @@ static int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
+@@ -2391,6 +2411,8 @@ static int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
  	else
  		err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
  
@@ -1281,7 +1277,7 @@ index de2c176..ce60715 100644
  	/* Success. */
  	if (!err)
  		return 0;
-@@ -2639,6 +2662,7 @@ static int do_brk(unsigned long addr, unsigned long len)
+@@ -2669,6 +2691,7 @@ static int do_brk(unsigned long addr, unsigned long request)
  		return 0;
  
  	flags = VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
@@ -1289,7 +1285,7 @@ index de2c176..ce60715 100644
  
  	error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
  	if (offset_in_page(error))
-@@ -2696,6 +2720,7 @@ static int do_brk(unsigned long addr, unsigned long len)
+@@ -2726,6 +2749,7 @@ static int do_brk(unsigned long addr, unsigned long request)
  	vma->vm_flags = flags;
  	vma->vm_page_prot = vm_get_page_prot(flags);
  	vma_link(mm, vma, prev, rb_link, rb_parent);
@@ -1297,7 +1293,7 @@ index de2c176..ce60715 100644
  out:
  	perf_event_mmap(vma);
  	mm->total_vm += len >> PAGE_SHIFT;
-@@ -2734,6 +2759,12 @@ void exit_mmap(struct mm_struct *mm)
+@@ -2764,6 +2788,12 @@ void exit_mmap(struct mm_struct *mm)
  	/* mm's last user has gone, and its about to be pulled down */
  	mmu_notifier_release(mm);
  
@@ -1310,7 +1306,7 @@ index de2c176..ce60715 100644
  	if (mm->locked_vm) {
  		vma = mm->mmap;
  		while (vma) {
-@@ -2769,6 +2800,11 @@ void exit_mmap(struct mm_struct *mm)
+@@ -2799,6 +2829,11 @@ void exit_mmap(struct mm_struct *mm)
  		vma = remove_vma(vma);
  	}
  	vm_unacct_memory(nr_accounted);
@@ -1322,7 +1318,7 @@ index de2c176..ce60715 100644
  }
  
  /* Insert vm structure into process list sorted by address
-@@ -2878,6 +2914,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
+@@ -2908,6 +2943,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
  			new_vma->vm_ops->open(new_vma);
  		vma_link(mm, new_vma, prev, rb_link, rb_parent);
  		*need_rmap_locks = false;
@@ -1330,7 +1326,7 @@ index de2c176..ce60715 100644
  	}
  	return new_vma;
  
-@@ -3015,6 +3052,7 @@ static struct vm_area_struct *__install_special_mapping(
+@@ -3055,6 +3091,7 @@ static struct vm_area_struct *__install_special_mapping(
  	vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT);
  
  	perf_event_mmap(vma);
@@ -1339,7 +1335,7 @@ index de2c176..ce60715 100644
  	return vma;
  
 diff --git a/mm/rmap.c b/mm/rmap.c
-index 701b93f..64ba784 100644
+index 1ef3640..1c40463 100644
 --- a/mm/rmap.c
 +++ b/mm/rmap.c
 @@ -1110,9 +1110,9 @@ void page_move_anon_rmap(struct page *page, struct vm_area_struct *vma)
@@ -1356,10 +1352,10 @@ index 701b93f..64ba784 100644
  static void __page_set_anon_rmap(struct page *page,
 diff --git a/mm/uksm.c b/mm/uksm.c
 new file mode 100644
-index 0000000..039192f
+index 0000000..56852a5
 --- /dev/null
 +++ b/mm/uksm.c
-@@ -0,0 +1,5518 @@
+@@ -0,0 +1,5524 @@
 +/*
 + * Ultra KSM. Copyright (C) 2011-2012 Nai Xia
 + *
@@ -1558,7 +1554,8 @@ index 0000000..039192f
 +static struct sradix_tree_node *slot_tree_node_alloc(void)
 +{
 +	struct slot_tree_node *p;
-+	p = kmem_cache_zalloc(slot_tree_node_cachep, GFP_KERNEL);
++	p = kmem_cache_zalloc(slot_tree_node_cachep, GFP_KERNEL |
++			      __GFP_NORETRY | __GFP_NOWARN);
 +	if (!p)
 +		return NULL;
 +
@@ -2044,7 +2041,8 @@ index 0000000..039192f
 +static inline struct node_vma *alloc_node_vma(void)
 +{
 +	struct node_vma *node_vma;
-+	node_vma = kmem_cache_zalloc(node_vma_cache, GFP_KERNEL);
++	node_vma = kmem_cache_zalloc(node_vma_cache, GFP_KERNEL |
++				     __GFP_NORETRY | __GFP_NOWARN);
 +	if (node_vma) {
 +		INIT_HLIST_HEAD(&node_vma->rmap_hlist);
 +		INIT_HLIST_NODE(&node_vma->hlist);
@@ -2069,7 +2067,8 @@ index 0000000..039192f
 +	if (!vma_slot_cache)
 +		return NULL;
 +
-+	slot = kmem_cache_zalloc(vma_slot_cache, GFP_KERNEL);
++	slot = kmem_cache_zalloc(vma_slot_cache, GFP_KERNEL |
++				 __GFP_NORETRY | __GFP_NOWARN);
 +	if (slot) {
 +		INIT_LIST_HEAD(&slot->slot_list);
 +		INIT_LIST_HEAD(&slot->dedup_list);
@@ -2089,7 +2088,8 @@ index 0000000..039192f
 +{
 +	struct rmap_item *rmap_item;
 +
-+	rmap_item = kmem_cache_zalloc(rmap_item_cache, GFP_KERNEL);
++	rmap_item = kmem_cache_zalloc(rmap_item_cache, GFP_KERNEL |
++				      __GFP_NORETRY | __GFP_NOWARN);
 +	if (rmap_item) {
 +		/* bug on lowest bit is not clear for flag use */
 +		BUG_ON(is_addr(rmap_item));
@@ -2106,7 +2106,8 @@ index 0000000..039192f
 +static inline struct stable_node *alloc_stable_node(void)
 +{
 +	struct stable_node *node;
-+	node = kmem_cache_alloc(stable_node_cache, GFP_KERNEL | GFP_ATOMIC);
++	node = kmem_cache_alloc(stable_node_cache, GFP_KERNEL |
++				__GFP_NORETRY | __GFP_NOWARN);
 +	if (!node)
 +		return NULL;
 +
@@ -2124,7 +2125,8 @@ index 0000000..039192f
 +static inline struct tree_node *alloc_tree_node(struct list_head *list)
 +{
 +	struct tree_node *node;
-+	node = kmem_cache_zalloc(tree_node_cache, GFP_KERNEL | GFP_ATOMIC);
++	node = kmem_cache_zalloc(tree_node_cache, GFP_KERNEL |
++				 __GFP_NORETRY | __GFP_NOWARN);
 +	if (!node)
 +		return NULL;
 +
@@ -2241,8 +2243,8 @@ index 0000000..039192f
 +	void *expected_mapping;
 +
 +	page = pfn_to_page(stable_node->kpfn);
-+	expected_mapping = (void *)stable_node +
-+				(PAGE_MAPPING_ANON | PAGE_MAPPING_KSM);
++	expected_mapping = (void *)((unsigned long)stable_node |
++				    PAGE_MAPPING_KSM);
 +	rcu_read_lock();
 +	if (page->mapping != expected_mapping)
 +		goto stale;
@@ -2919,6 +2921,7 @@ index 0000000..039192f
 +				(page_to_pfn(kpage) == zero_pfn)) {
 +		entry = pte_mkspecial(entry);
 +		dec_mm_counter(mm, MM_ANONPAGES);
++		inc_zone_page_state(page, NR_UKSM_ZERO_PAGES);
 +	} else {
 +		get_page(kpage);
 +		page_add_anon_rmap(kpage, vma, addr, false);
@@ -3986,7 +3989,7 @@ index 0000000..039192f
 +		if (IS_ERR_OR_NULL(page))
 +			break;
 +		if (PageKsm(page)) {
-+			ret = handle_mm_fault(vma->vm_mm, vma, addr,
++			ret = handle_mm_fault(vma, addr,
 +					      FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE);
 +		} else
 +			ret = VM_FAULT_WRITE;
@@ -4634,7 +4637,6 @@ index 0000000..039192f
 +	if (find_zero_page_hash(hash_strength, *hash)) {
 +		if (!cmp_and_merge_zero_page(slot->vma, page)) {
 +			slot->pages_merged++;
-+			inc_zone_page_state(page, NR_UKSM_ZERO_PAGES);
 +
 +			/* For full-zero pages, no need to create rmap item */
 +			goto putpage;
@@ -6879,12 +6881,12 @@ index 0000000..039192f
 +#endif
 +
 diff --git a/mm/vmstat.c b/mm/vmstat.c
-index cb2a67b..912b86f 100644
+index 89cec42..188ce43 100644
 --- a/mm/vmstat.c
 +++ b/mm/vmstat.c
-@@ -733,6 +733,9 @@ const char * const vmstat_text[] = {
- 	"nr_anon_transparent_hugepages",
- 	"nr_free_cma",
+@@ -974,6 +974,9 @@ const char * const vmstat_text[] = {
+ 	"nr_dirtied",
+ 	"nr_written",
  
 +#ifdef CONFIG_UKSM
 +	"nr_uksm_zero_pages",