util/release/genrelnotes

#!/usr/bin/env bash
#
# This file is part of the coreboot project.
#
# Copyright 2015-2016 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.


# This script creates a list of commits between two releases, broken out into
# fairly inexact categories, based on the directories that files are in.  If
# a commit touched anything in the path that is checked earlier, it's counted
# as being in that category.
#
# Don't run this in your current working tree, it checks out different versions
# and can lose things.

# set -x # uncomment for debug

# Check for tools

if ! ( git --version && cloc --version && rename --version ) > /dev/null 2>&1
then
	echo "ERROR: cloc or git is not installed.  Exiting"
	exit 1
fi

if ! { cdup="$(git rev-parse --show-cdup 2>/dev/null)" && [ -z "${cdup}" ]; }
then
	echo "ERROR: This is not the top directory of a git repo.  Exiting."
	exit 1
fi

# Try to verify that the repo is clean before losing state.
if ! git diff-index --quiet --cached HEAD 2>/dev/null; then
	echo "ERROR: repo is not clean.  Exiting."
	exit 1
fi

# Verify the command line arguments
if [ "$1" == "--help" ] || [ -z "$1" ] || [ -z "$2" ]; then
	echo
	echo "Usage: $0 <old_version> <new_version> [release notes file]"
	echo "Old version should be a tag (4.1), a branch (origin/4.1), or a commit id"
	echo "New version can be 'HEAD' a branch (origin/master) a tag (4.2), or a commit id"
	echo "Logfile can be a new file or an existing file to update"
	echo "Example: \"$0 origin/4.1 4.2 rnotes.txt\""
	echo
	echo "Note that the script starts at the commit AFTER the old version."
	echo
	exit 1
else
	OLD_GIT_VERSION="$1"
	NEW_GIT_VERSION="$2"
	TOTAL_COMMITS=$(git log --pretty=oneline \
		"${OLD_GIT_VERSION}..${NEW_GIT_VERSION}" 2>/dev/null | wc -l)
fi

TOP=$(pwd)

if [ -n "$3" ]; then
	MAIN_LOGFILE="${TOP}/$3"
else
	MAIN_LOGFILE="${TOP}/relnotes.txt"
fi

# Figure out which logfile we're writing to.  If the specified logfile exists,
# we need to write to a temporary logfile, then append changes to the main
# logfile.
if [ -f "$MAIN_LOGFILE" ]; then
	LOGFILE="$(mktemp "LOGFILE.XXXX")"
	LOGFILE="${TOP}/$LOGFILE"
	UPDATE_MAIN_LOGFILE=1
else
	LOGFILE="$MAIN_LOGFILE"
fi

get_author_commit_count() {
	git log "${NEW_GIT_VERSION}" 2>/dev/null | grep -c "^Author: $1"
}

# Print and log the versions
log_versions() {
	echo "Log of commit $1 to commit $2"
	echo "Log of commit $1 to commit $2" >> "$LOGFILE"
	echo "Total commits: ${TOTAL_COMMITS}"
	echo "Total commits: ${TOTAL_COMMITS}" >> "$LOGFILE"
	echo
}

# Get the first commit id in the current tree
get_latest_commit_id() {
(
	cd "$1"
	git log 2>/dev/null | grep '^commit ' | head -1 | sed 's/commit //'
)
}

# Main get log function
_get_log() {
	local oldver="$1"
	local newver="$2"
	local title="$3"
	local paths="$4"
	local keywords="$5"

	# Leave ${paths} unquoted in the git commands
	# shellcheck disable=SC2086
	{ \
		if [ -n "$paths" ]; then \
			git log --abbrev-commit --pretty=oneline \
				"${oldver}..${newver}" -- ${paths} \
				2>/dev/null; \
		fi; \
		if [ -n "$keywords" ]; then \
			git log --abbrev-commit --pretty=oneline \
			"${oldver}..${newver}" 2>/dev/null \
			| grep -i "$keywords"; \
		fi \
	} | sort -t ' ' -k 2 | uniq
}

# Output to a new log, then compare to the first logfile, and only output
# non duplicated lines to the final file.
get_log_dedupe() {
	local title="$1"
	local paths="$2"
	local keywords="$3"
	local log
	local commits

	dedupe_tmpfile="$(mktemp "LOGFILE.XXXX")"

	log=$(_get_log "$OLD_GIT_VERSION" "$NEW_GIT_VERSION" \
		"$title" "$paths" "$keywords")

	echo "$log" > "$dedupe_tmpfile"

	log=$(grep -Fxv -f "$LOGFILE" "$dedupe_tmpfile")
	commits=$(echo "$log" | wc -l)

	#echo "$title: $paths $keywords" >> "$LOGFILE"
	printf "%s\n%s\n\n" "##### $title ($commits commits) #####" \
		"$log" >> "$LOGFILE"

	rm "$dedupe_tmpfile"
}

# get logs for the submodules
get_log_submodule() {
	local old_version="$1"
	local new_version="$2"
	local submodule_dir="$3"
	local log
	local commits

	printf "Submodule %s\n" "$submodule_dir"
	printf "commit %s to commit %s\n\n" "$old_version" "$new_version"

	(
	cd "${TOP}/$submodule_dir"
	log="$(_get_log "$old_version" "$new_version" "$submodule_dir" "." "")"
	commits=$(echo "$log" | wc -l)

	if [ -n "$log" ]; then
		printf "%s\n" "$submodule_dir ($commits commits)" >> "$LOGFILE"
		printf "\n" >> "$LOGFILE"
	fi
	)
}

find_areas() {
	local directory="$1"
	local filename="$2"
	local skip="$3"
	find "$directory" -name "$filename" | sed "s|/$filename||" | sed "s|/$directory||" | grep -v "$skip" | sort
}

# Make sure things get cleaned up if ctl-c is pressed while the old version
# is checked out and files are renamed.  This can be a real mess to clean
# up manually.
version_ctrl_c() {
	printf "\n** Trapped CTRL-C\n Cleaning up and exiting.\n"
	find 'src' -name 'gnumakefile' \
		-exec rename 's/gnumakefile/Makefile\.inc/' {} \;
	git checkout origin/master > /dev/null 2>&1
	git submodule update --init --checkout  > /dev/null 2>&1
	rm "$LOGFILE"
	exit 1;
}

# Calculate areas that have been added or removed based on file lists
show_diff () {
	local new
	local old

	new="$(comm -13 <(echo "$2") <(echo "$3"))"
	old="$(comm -23 <(echo "$2") <(echo "$3"))"

	# Allow running a postprocessor, given as 4th argument over the
	# resulting diff, provide context if it's old or new data
	if [ -n "$4" ]; then
		new=$(echo "$new" | $4 new | sort)
		old=$(echo "$old" | $4 old | sort)
	fi
	new="$(printf "$new" | sed 's/^/* /')"
	old="$(printf "$old" | sed 's/^/* /')"

	if [ -n "$new" ]; then
		printf "Added %s $1:\n-------------------\n%s\n\n" \
			"$(echo "$new" | wc -l)" "$new"  >> "$LOGFILE"
	fi
	if [ -n "$old" ]; then
		printf "Removed %s $1:\n-------------------\n%s\n\n" \
			"$(echo "$old" | wc -l)" "$old"  >> "$LOGFILE"
	fi
}

get_sloc () {
	# Because cloc works on extensions, and .inc identifies as pascal,
	# while we use it both for Makefile.inc and some files that are
	# really C, do three passes: everything but .inc files, all .inc files
	# that aren't Makefiles, all Makefile.inc, then combine them.

	local base=`mktemp`
	find src -name Makefile.inc > ${base}.mak

	cloc --progress-rate=0 --quiet \
		--script-lang="Bourne Shell",bash --exclude-ext=inc \
		--exclude-dir=vendorcode --out=${base} src
	cloc --progress-rate=0 --quiet \
		--exclude-list-file=${base}.mak --force-lang=c,inc \
		--exclude-dir=vendorcode --out=${base}.c src
	cloc --progress-rate=0 --quiet \
		--list-file=${base}.mak --force-lang=make,inc \
		--exclude-dir=vendorcode --out=${base}.m src
	cloc --progress-rate=0 --quiet --sum-reports \
		${base} ${base}.c ${base}.m --out ${base}.result

	echo
	cat ${base}.result.lang

	rm -f ${base}*
}

# Start collecting data from the old and new revisions.
# This is relatively disruptive to the tree, so trap on ctl-c so that
# things can be put back to normal
trap version_ctrl_c SIGINT

#check out old version and get information
printf -- "Finding old submodule versions...\n"
git checkout "$OLD_GIT_VERSION" > /dev/null 2>&1
git submodule update --init --checkout  > /dev/null 2>&1
for module in $(git submodule --quiet foreach pwd); do
	name="$(basename "$module" | sed 's/-/_/g')"
	version="${name^^}_OLD_VERSION"
	declare "$version"=$(get_latest_commit_id "$module")
done

printf "Logging directories in the old tree\n"
mainboard_list_old=$(grep -h "^[[:space:]]*config\>[[:space:]]*\<BOARD_" src/mainboard/*/*/Kconfig.name 2>/dev/null | sed "s,^.*\<BOARD_\([A-Z0-9_]*\)\>.*$,\1," | sort)
cpu_list_old=$(find_areas "src/cpu" "Kconfig" "intel/common")
soc_list_old=$(find_areas "src/soc" "Makefile.inc" "intel/common\|amd/common\|romstage")
northbridge_list_old=$(find_areas "src/northbridge" "Kconfig" "")
sio_list_old=$(find_areas "src/superio" "Makefile.inc" "")
southbridge_list_old=$(find_areas "src/southbridge" "Kconfig" "")

printf "Calculating old SLOC\n"
OLD_SLOC=$(get_sloc)

#check out new version and get information
printf -- "\nFinding new submodule versions...\n"
git checkout "$NEW_GIT_VERSION"  > /dev/null 2>&1
git submodule update --init --checkout  > /dev/null 2>&1

for module in $(git submodule --quiet foreach pwd); do
	name="$(basename "$module" | sed 's/-/_/g')"
	version="${name^^}_NEW_VERSION"
	declare "$version"=$(get_latest_commit_id "$module")
done

printf "Logging directories in the new tree\n"
mainboard_list_new=$(grep -h "^[[:space:]]*config\>[[:space:]]*\<BOARD_" src/mainboard/*/*/Kconfig.name 2>/dev/null | sed "s,^.*\<BOARD_\([A-Z0-9_]*\)\>.*$,\1," | sort)
cpu_list_new=$(find_areas "src/cpu" "Kconfig" "intel/common")
soc_list_new=$(find_areas "src/soc" "Makefile.inc" "intel/common\|amd/common\|romstage")
northbridge_list_new=$(find_areas "src/northbridge" "Kconfig" "")
sio_list_new=$(find_areas "src/superio" "Makefile.inc" "")
southbridge_list_new=$(find_areas "src/southbridge" "Kconfig" "")

printf "Calculating new SLOC\n"
NEW_SLOC=$(get_sloc)

git checkout origin/master  > /dev/null 2>&1
git submodule update --init --checkout  > /dev/null 2>&1
trap "" SIGINT
# Done collecting data from the old and new versions

# Start outputting to logfile
echo "Generating release notes from version ${OLD_GIT_VERSION} to ${NEW_GIT_VERSION}"
echo; echo "Main repo"
echo "Main repo" >> "$LOGFILE"
echo "------------------" >> "$LOGFILE"
log_versions "$(git log --pretty=%H \
	"${OLD_GIT_VERSION}..${NEW_GIT_VERSION}" 2>/dev/null | tail -1)" \
	"$(git log --pretty=%H \
	"${OLD_GIT_VERSION}..${NEW_GIT_VERSION}" 2>/dev/null | head -1 )"
echo "" >> "$LOGFILE"

### SPECIFIC AREAS FOR RELEASE ###
get_log_dedupe "cleanup" "" "spelling\|Use tabs\|transition away from device_t\|space [around\|before]\|code formatting\|commented code\|code cleanup\|capitalize\|unnecessary whitespace\|checkpatch\|remove unused\|remove.*not used"

get_log_dedupe "Google Kahlee / AMD Gardenia" "src/mainboard/google/kahlee src/mainboard/amd/gardenia" "kahlee\|gardenia"
get_log_dedupe "AMD Stoney Ridge" "src/soc/amd/stoneyridge" "stoney"

get_log_dedupe "Google Eve / Poppy / Fizz / Soraka / Nautilus / Intel KblRvp" "src/mainboard/google/eve src/mainboard/google/poppy src/mainboard/google/fizz src/mainboard/intel/kblrvp" "eve[ :]\|poppy\|fizz\|soraka\|nautilus\|kblrvp"
#get_log_dedupe "Intel Kunimitsu / Google Chell / Lars / Glados" "src/mainboard/google/lars src/mainboard/google/chell src/mainboard/google/glados src/mainboard/intel/kunimitsu" "chell\|lars\|kunimitsu"
get_log_dedupe "Purism SKL" "src/mainboard/purism/librem_skl" "librem13v2\|librem_skl"
get_log_dedupe "Intel Skylake / Kabylake" "src/soc/intel/skylake"  "skylake\|sky.lake\|kabylake\|kaby.lake"

get_log_dedupe "Google Cyan / Intel Strago" "src/mainboard/google/cyan src/mainboard/intel/strago" "cyan\|strago"
get_log_dedupe "Intel Braswell" "src/soc/intel/braswell" "braswell"

get_log_dedupe "Google Kevin / Gru / Bob / Scarlet / Nefario" "src/mainboard/google/gru" "kevin\|gru[^b]"
get_log_dedupe "Rockchip rk3399" "src/soc/rockchip/rk3399" "rk3399"

get_log_dedupe "Google Reef / Pyro / Sand / Snappy / Nasher" "src/mainboard/google/reef" "reef\|pyro\|sand[ :]\|snappy\|nasher"
#get_log_dedupe "Intel apollolake_rvp leafhill minnow3" "src/mainboard/intel/apollolake_rvp src/mainboard/intel/leafhill src/mainboard/intel/minnow3" "apollolake_rvp\|leafhill\|minnow3"
get_log_dedupe "Siemens mc_apl1" "src/mainboard/siemens/mc_apl1" "mc_apl1"
get_log_dedupe "Intel Apollolake" "src/soc/intel/apollolake" "apollolake\|apollo.lake"

get_log_dedupe "Intel CannonLake" "src/soc/intel/cannonlake src/mainboard/intel/cannonlake_rvp" "cannonlake"

get_log_dedupe "Intel Galileo" "src/mainboard/intel/galileo" "galileo"
get_log_dedupe "Intel Quark" "src/soc/intel/quark" "quark"

get_log_dedupe "Intel Baytrail" "src/soc/intel/baytrail" "baytrail"
#get_log_dedupe "Google Gale / Qualcomm QX ipq40xx" "src/mainboard/google/gale src/soc/qualcomm/ipq40xx" "gale\|ipq40"
#et_log_dedupe "Google Rotor / Marvell Mvmap2315" "src/soc/marvell/mvmap2315 src/mainboard/google/rotor" "marvell\|mvmap\|rotor"

get_log_dedupe "Gigabyte ga-g41m-es2l" "src/mainboard/gigabyte/ga-g41m-es2l" "es2l"
get_log_dedupe "Intel x4x northbridge / LGA775" "src/northbridge/intel/x4x src/cpu/intel/socket_LGA775" "x4x\|lga775"

get_log_dedupe "Intel Sandybridge / Ivybridge" "src/southbridge/intel/bd82x6x src/southbridge/intel/fsp_bd82x6x src/northbridge/intel/sandybridge src/northbridge/intel/fsp_sandybridge src/cpu/intel/fsp_model_206ax src/cpu/intel/model_206ax" "sandybridge\|ivybridge\|bd82x6x"

get_log_dedupe "Intel Common" "src/soc/intel/common src/southbridge/intel/common src/northbridge/intel/common" ""
get_log_dedupe "Amd Common" "src/soc/amd/common src/southbridge/amd/common" ""

get_log_dedupe "Nvidia" "src/southbridge/nvidia" ""
get_log_dedupe "Via" "src/northbridge/via src/southbridge/via" ""
get_log_dedupe "Broadcom" "src/southbridge/broadcom" ""
get_log_dedupe "Qualcomm" "src/soc/qualcomm" ""

get_log_dedupe "Intel vendorcode / FSP" "src/drivers/intel/fsp* src/vendorcode/intel" ""
get_log_dedupe "AMD vendorcode / AGESA / PI" "src/vendorcode/amd" ""
get_log_dedupe "Google vendorcode" "src/vendorcode/google"

get_log_dedupe "TPM" "src/drivers/i2c/tpm src/security/tpm" "tpm"
get_log_dedupe "Vboot" "src/vboot" "vboot"


### GENERAL AREAS FOR RELEASE ###
# shellcheck disable=SC2013
{
get_log_dedupe "ARM" \
	"$(for codedir in $(grep -rl "_ARM" --include=Kconfig | \
	grep -v 'src/mainboard\|payloads/\|drivers/\|vendorcode/\|console' ); \
	do dirname "$codedir"; done | grep -v '^src$')"

get_log_dedupe "RISC-V" \
	"$(for codedir in $(grep -rl "_RISCV" --include=Kconfig | grep -v 'payloads/\|drivers/\|vendorcode/\|console' ); do dirname "$codedir"; done | grep -v '^src$')" \
	"riscv\|risc-v\|sifive"
}

get_log_dedupe "X86 intel" \
	"src/cpu/intel src/soc/intel src/northbridge/intel \
	src/southbridge/intel src/include/intel src/drivers/intel"

get_log_dedupe "X86 amd" \
	"src/cpu/amd src/northbridge/amd src/southbridge/amd src/include/amd src/soc/amd" \
	"agesa\|binarypi\|binary.pi"

get_log_dedupe "X86 common" \
	"src/arch/x86 src/cpu/x86 src/include/x86 src/include/pc80"

get_log_dedupe "Mainboards" "src/mainboard/"

# Next, print all the rest of the specific areas
get_log_dedupe "ACPI"		"src/acpi/"
get_log_dedupe "Console"	"src/console src/include/console"
get_log_dedupe "SuperIO"	"src/superio src/include/superio"
get_log_dedupe "EC"		"src/ec"
get_log_dedupe "Drivers"	"src/drivers"
get_log_dedupe "Devices"	"src/device src/include/device"

# 5th, print the generic areas - This goes late so that the specific
# area changes will catch any commits in these areas first.
get_log_dedupe "Toolchain"	"util/crossgcc"
get_log_dedupe "cbfstool"	"util/cbfstool"
get_log_dedupe "Lint tools"	"util/lint"
get_log_dedupe "Lib"		"src/lib"
get_log_dedupe "Commonlib"	"src/commonlib"
get_log_dedupe "Include"	"src/include"
get_log_dedupe "Utilities"	"util"
get_log_dedupe "Payloads"	"payloads"
get_log_dedupe "Vendorcode"	"src/vendorcode"
get_log_dedupe "Documentation"	"Documentation README"

# Then look at areas that are usually outside the mainboards and architectures
get_log_dedupe "Build system" \
	"Makefile Makefile.inc toolchain.inc src/Kconfig src/cpu/Makefile.inc"

get_log_dedupe "Submodules" "3rdparty util/nvidia/cbootimage"

get_log_dedupe "Maintainers" "MAINTAINERS" ""

# Finally, get anything that was missed above
get_log_dedupe "MISC" "."

# Replace VENDOR_DEVICE from stdin with their nice names on stdout
real_mainboard_names() {
	local tree_version=$1 # "old" or "new"
	local git_version_var=${tree_version^^}_GIT_VERSION
	local git_version=${!git_version_var}
	local line

	while read line; do
		local file="$(git grep -l "^[[:space:]]*config\>[[:space:]]*\<BOARD_${line}\$" ${git_version} -- src/mainboard/\*/\*/Kconfig.name | sed "s,^${git_version}:,,")"
		if [ -z "$file" ]; then
			echo "This shouldn't happen: couldn't find Kconfig for $line"
			exit 1
		fi
		local vendor="$(git grep \" ${git_version} -- $(echo ${file} | cut -d/ -f1-3,5) | cut -d\" -f2)"
		git grep -h -A1 "^[[:space:]]*config\>[[:space:]]*\<BOARD_${line}\$" ${git_version} -- ${file} | grep \" |cut -d\" -f2 |sed "s,^,${vendor} ,"
	done
}

# Show areas that have been added or removed
show_diff "mainboards"   "$mainboard_list_old"   "$mainboard_list_new"    real_mainboard_names
show_diff "processors"   "$cpu_list_old"         "$cpu_list_new"
show_diff "socs"         "$soc_list_old"         "$soc_list_new"
show_diff "northbridges" "$northbridge_list_old" "$northbridge_list_new"
show_diff "southbridges" "$southbridge_list_old" "$southbridge_list_new"
show_diff "sios"         "$sio_list_old"         "$sio_list_new"

# Log submodules
printf "Submodules\n----------\n" >> "$LOGFILE"
for module in $(git submodule --quiet foreach pwd); do
	name=$(basename "$module")
	# shellcheck disable=SC2001
	path=${module#$PWD}
	old_version=$(echo "${name^^}_OLD_VERSION" | sed 's/-/_/g')
	new_version=$(echo "${name^^}_NEW_VERSION" | sed 's/-/_/g')
	get_log_submodule "${!old_version}" "${!new_version}" "$path"
done

printf "\nrepo statistics\n-------------------\n" >> "$LOGFILE"
before_names="$(mktemp "OLDNAMES.XXXX")"
after_names="$(mktemp "NEWNAMES.XXXX")"
NEW_AUTHORS=$(git log --pretty=%an "${OLD_GIT_VERSION}" 2>/dev/null | sort | \
	uniq > "$before_names" && \
	git log --pretty=%an "${NEW_GIT_VERSION}" 2>/dev/null | \
	sort | uniq > "$after_names" && \
	grep -Fxv -c -f "$before_names" "$after_names")
NEW_AUTHOR_LIST=$( grep -Fxv -f "$before_names" "$after_names")
rm -f "$before_names" "$after_names"
{
	printf -- "- Total commits: %s\n" "$TOTAL_COMMITS"
	printf -- "- Total authors: %s\n" \
		"$(git log "${OLD_GIT_VERSION}..${NEW_GIT_VERSION}" 2>/dev/null | \
		grep -e '^Author:' | sed 's/.*Author: //' | sed 's/ <.*.>//' | \
		sort | uniq | wc -l)"
	printf -- "- New authors: %s\n\nNew Authors:\n%s\n" "$NEW_AUTHORS" \
		"$NEW_AUTHOR_LIST"
} >> "$LOGFILE"

printf "Getting developer list\n"
printf "\n%-40s:  %5s\n" "Developer" "Commits" >> "$LOGFILE"
git log "${OLD_GIT_VERSION}..${NEW_GIT_VERSION}" 2>/dev/null | grep '^Author: ' | \
	sed 's|Author: ||' | sed 's|\s<.*||' | sort | uniq | \
	while read -r line; do
		 printf "%-40s:  %5s %5s\n" "$line" \
		 	"$(git log "${OLD_GIT_VERSION}" 2>/dev/null | \
		 	grep -c "^Author: ${line} <")" \
		 	"$(git log "${NEW_GIT_VERSION}" 2>/dev/null | \
		 	grep -c "^Author: ${line} <")" >> "$LOGFILE";
	 done

printf "\nOld SLOC (%s)\n%s" "$NOW" "$OLD_SLOC" >> "$LOGFILE"
printf "\nNew SLOC (%s)\n%s" "$NOW" "$NEW_SLOC" >> "$LOGFILE"

# Add the collected data to the top of the existing logfile for parsing
if [ -n "$UPDATE_MAIN_LOGFILE" ]; then
	tmpfile="$(mktemp "LOGFILE.XXXX")"
	grep -Fxv -f "$MAIN_LOGFILE" "$LOGFILE" > "$tmpfile"
	printf "\n\n" >> "$tmpfile"
	cat "$MAIN_LOGFILE" >> "$tmpfile"
	mv "$tmpfile" "$MAIN_LOGFILE"
	rm -f "$LOGFILE"
fi

printf "Done.\n"