Martin Roth | 298b007 | 2021-11-10 20:33:44 -0700 | [diff] [blame] | 1 | #!/bin/bash |
| 2 | set -e |
| 3 | |
| 4 | VERSION="1.00" |
| 5 | |
| 6 | PROGRAM=$0 |
| 7 | PROGNAME="$(basename "${PROGRAM}")" |
| 8 | |
| 9 | MODIFIED_FILES=() |
| 10 | CLEAN_DIR_LIST=(configs Documentation payloads spd src util) |
| 11 | KEEP_FILES=(util/kconfig/) |
| 12 | REQUIRED_MAKEFILES="util/testing\|util/crossgcc\|payloads/coreinfo\|payloads/nvramcui\|payloads/libpayload\|payloads/external/tint\|util/amdfwtool\|util/ectool\|util/futility\|util/intelmetool\|util/inteltool\|util/intelvbttool\|til/post\|util/superiotool" |
| 13 | VERBOSE= |
| 14 | |
| 15 | # Text STYLE variables |
| 16 | BOLD="\033[1m" |
| 17 | RED='\033[38;5;9m' |
| 18 | GREEN='\033[38;5;2m' |
| 19 | NO_COLOR='\033[0m' |
| 20 | |
| 21 | ################################################################################ |
| 22 | |
| 23 | usage() { |
| 24 | cat << EOF |
| 25 | The ${PROGNAME} script is used to create a git patch that removes all files |
| 26 | not used in a single build. It does this by creating a temporary directory |
| 27 | and configuring it to show the last time a file was accessed. It then sets |
| 28 | the time on all files back to midnight on 2021-01-01 and then does a full |
| 29 | build. Because all Kconfig and Makefiles are accessed during the built, |
| 30 | it then creates a new Kconfig file containing all of the old Kconfigs. |
| 31 | The next step is to delete all of the files that have an access time still |
| 32 | set to 2021. The final step of the cleaning process is to recursively remove |
| 33 | any Makefile that is alone in a directory by itself. The script then makes |
| 34 | a commit and creates a patch. |
| 35 | |
| 36 | Usage: ${PROGNAME} [options] |
| 37 | |
| 38 | Options: |
| 39 | -b | --blddir <dir> Set /tmp/<dir> as the build directory |
| 40 | -h | --help Print usage and exit |
| 41 | -D | --debug Print debug information. Use -DD to show all commands |
| 42 | -V | --version Print the version and exit |
| 43 | --nocolor Don't print color codes |
| 44 | EOF |
| 45 | } |
| 46 | |
| 47 | _echo_color() { |
| 48 | local color="$1" |
| 49 | local text="$2" |
| 50 | local newline="$3" |
| 51 | if [[ ${newline} == "0" ]]; then |
| 52 | printf "${color}%s${NO_COLOR}" "${text}" |
| 53 | else |
| 54 | printf "${color}%s${NO_COLOR}\n" "${text}" |
| 55 | fi |
| 56 | } |
| 57 | |
| 58 | _echo_error() { |
| 59 | _echo_color "${RED}" "$*" 1 >&2 |
| 60 | } |
| 61 | |
| 62 | show_version() { |
| 63 | echo |
| 64 | _echo_color "${BOLD}${GREEN}" "${PROGNAME} version ${VERSION}" |
| 65 | echo |
| 66 | } |
| 67 | |
| 68 | get_args() { |
| 69 | if ! args="$(getopt -l version,help,debug,nocolor,blddir: -o b:DhV -- "$@")"; then |
| 70 | usage |
| 71 | exit 1 |
| 72 | fi |
| 73 | |
| 74 | eval set -- "${args}" |
| 75 | |
| 76 | while true; do |
| 77 | case "$1" in |
| 78 | -b | --blddir) |
| 79 | shift |
| 80 | BLD_DIR="/tmp/$1" |
| 81 | ;; |
| 82 | -D | --debug) |
| 83 | # -d prints extra debug info |
| 84 | # -dd prints all script steps |
| 85 | if [ -n "${VERBOSE}" ]; then |
| 86 | set -x |
| 87 | else |
| 88 | VERBOSE="V=1" |
| 89 | fi |
| 90 | ;; |
| 91 | -h | --help) |
| 92 | usage |
| 93 | exit 0 |
| 94 | ;; |
| 95 | --nocolor) |
| 96 | BOLD="" |
| 97 | RED="" |
| 98 | GREEN="" |
| 99 | NO_COLOR="" |
| 100 | ;; |
| 101 | -V | --version) exit 0 ;; |
| 102 | --) |
| 103 | shift |
| 104 | break |
| 105 | ;; |
| 106 | *) |
| 107 | _echo_error "Unknown argument '$1'" |
| 108 | usage |
| 109 | exit 1 |
| 110 | ;; |
| 111 | esac |
| 112 | shift |
| 113 | done |
| 114 | |
| 115 | if [[ -n $1 ]]; then |
| 116 | _echo_error "Unknown command '$1'" |
| 117 | usage |
| 118 | exit 1 |
| 119 | fi |
| 120 | |
| 121 | BLD_DIR="${BLD_DIR:-$(mktemp -d)}" |
| 122 | } |
| 123 | |
| 124 | recursively_rm_dir_onlyfile() { |
| 125 | local dir=$1 |
| 126 | local beforecount |
| 127 | local aftercount |
| 128 | |
| 129 | while true; do |
| 130 | if [[ ! -d ${dir} ]]; then |
| 131 | break |
| 132 | fi |
| 133 | beforecount="$(find "${dir}" | wc -l)" |
| 134 | while read -r file; do |
| 135 | # Don't delete any of the makefiles required for building. |
| 136 | if echo "${file}" | grep -q "${REQUIRED_MAKEFILES}"; then |
| 137 | break |
| 138 | fi |
| 139 | # Remove the directory if a makefile is the only file present. |
| 140 | if [[ "$(cd "${file}" && find . -maxdepth 1 | grep -v "./Makefile")" == "." ]]; then |
| 141 | rm -rf "${file}" |
| 142 | fi |
| 143 | done < <(find "${dir}" -depth -type d) |
| 144 | if [[ ! -d ${dir} ]]; then |
| 145 | break |
| 146 | fi |
| 147 | find "${dir}" -type d -empty -delete |
| 148 | if [[ ! -d ${dir} ]]; then |
| 149 | break |
| 150 | fi |
| 151 | aftercount="$(find "${dir}" | wc -l)" |
| 152 | if [[ ${aftercount} -eq ${beforecount} ]]; then |
| 153 | break |
| 154 | fi |
| 155 | done |
| 156 | } |
| 157 | |
| 158 | verify_atime_enabled() { |
| 159 | local testfile |
| 160 | # Make sure the build directory is mounted correctly |
| 161 | if [ ! -d "${BLD_DIR}" ]; then |
| 162 | mkdir "${BLD_DIR}" |
| 163 | fi |
| 164 | if ! grep -q "${BLD_DIR}" /proc/mounts; then |
| 165 | echo "Mounting the ${BLD_DIR} directory with atime enabled" |
| 166 | sudo mount -t tmpfs -o rw,relatime tmpfs "${BLD_DIR}" |
| 167 | elif ! grep "${BLD_DIR}" /proc/mounts | grep -q relatime; then |
| 168 | echo "Remounting the ${BLD_DIR} directory with relatime enabled" |
| 169 | sudo mount -o remount,relatime "${BLD_DIR}" |
| 170 | fi |
| 171 | |
| 172 | testfile="$(mktemp -p "${BLD_DIR}")" |
| 173 | touch -a --date="2020-01-01 00:00:00" "${testfile}" |
| 174 | if ! stat "${testfile}" | grep -q "Access: 2020-01-01"; then |
| 175 | _echo_error "Error: could not set access time." |
| 176 | sudo umount "${BLD_DIR}" |
| 177 | rm -rf "${BLD_DIR}" |
| 178 | exit 1 |
| 179 | fi |
| 180 | rm -f "${testfile}" |
| 181 | } |
| 182 | |
| 183 | update_codebase() { |
| 184 | local tempconfig |
| 185 | tempconfig="$(mktemp)" |
| 186 | if [ ! -f "${BLD_DIR}/COPYING" ]; then |
| 187 | echo "Downloading coreboot tree" |
| 188 | git clone https://review.coreboot.org/coreboot.git "${BLD_DIR}" |
| 189 | make -C "${BLD_DIR}" build/xcompile |
| 190 | fi |
| 191 | |
| 192 | # Start from a completely clean tree or we'll miss anything that |
| 193 | # doesn't need to be rebuilt. Save the config if it exists. |
| 194 | if [[ -f .config ]]; then |
| 195 | mv .config "${tempconfig}" |
| 196 | fi |
| 197 | _echo_color "${GREEN}" "Cleaning coreboot tree" |
| 198 | make -s -C "${BLD_DIR}" distclean |
| 199 | if [[ -f ${tempconfig} ]]; then |
| 200 | mv "${tempconfig}" .config |
| 201 | fi |
| 202 | |
| 203 | # force a refresh of all submodules |
| 204 | _echo_color "${GREEN}" "Refreshing all submodules..." |
| 205 | git submodule update --recursive --remote --init --checkout |
| 206 | } |
| 207 | |
| 208 | save_kconfig() ( |
| 209 | cd "${BLD_DIR}" && util/lint/kconfig_lint -w -p -o kconfig.tmp |
| 210 | ) |
| 211 | |
| 212 | update_times() { |
| 213 | _echo_color "${GREEN}" "Updating access time of all files" |
| 214 | git ls-files | xargs touch -a -m -t 202001010000 |
| 215 | if ! stat "${BLD_DIR}/COPYING" | grep -q "Access: 2020-01-01"; then |
| 216 | _echo_error "Error: could not set access time." |
| 217 | _echo_error " One of the following processes may be accessing it." |
| 218 | fuser -uvm "${BLD_DIR}/COPYING" |
| 219 | exit 1 |
| 220 | fi |
| 221 | } |
| 222 | |
| 223 | mark_files_to_keep() { |
| 224 | for file in "${KEEP_FILES[@]}"; do |
| 225 | find "${BLD_DIR}/${file}" -depth -exec touch {} \; |
| 226 | done |
| 227 | } |
| 228 | |
| 229 | build_platform() { |
| 230 | local extra_text="$1" |
| 231 | _echo_color "${GREEN}" "Building platform ${extra_text}" |
| 232 | if [[ ! -f "${BLD_DIR}/.config" ]]; then |
| 233 | if [[ -n ${CONFIG_FILE} ]]; then |
| 234 | cp "${CONFIG_FILE}" "${BLD_DIR}/.config" |
| 235 | fi |
| 236 | echo "CONFIG_PAYLOAD_NONE=y" >>"${BLD_DIR}/.config" |
| 237 | fi |
| 238 | |
| 239 | make -C "${BLD_DIR}" -s clean UPDATED_SUBMODULES=1 BUILD_TIMELESS=1 |
| 240 | make -C "${BLD_DIR}" -s olddefconfig |
| 241 | make -C "${BLD_DIR}" -s UPDATED_SUBMODULES=1 BUILD_TIMELESS=1 ${VERBOSE} |
| 242 | HASH="$(sha256sum build/coreboot.rom)" |
| 243 | make -C "${BLD_DIR}" -s clean UPDATED_SUBMODULES=1 BUILD_TIMELESS=1 |
| 244 | } |
| 245 | |
| 246 | show_modified() { |
| 247 | readarray MODIFIED_FILES < <(find "${BLD_DIR}" -atime -1 -type f -path ./.git -prune) |
| 248 | echo "Files changed: ${#MODIFIED_FILES[@]}" |
| 249 | } |
| 250 | |
| 251 | remove_kconfigs() { |
| 252 | # Dump all Kconfigs into a single file so that directories |
| 253 | # can be removed, while maintaining the entire Kconfig |
| 254 | # structure. |
| 255 | find "${BLD_DIR}/src" -name 'Kconfig*' -delete |
| 256 | mv "${BLD_DIR}/kconfig.tmp" "${BLD_DIR}/src/Kconfig" |
| 257 | } |
| 258 | |
| 259 | remove_unused() { |
| 260 | local dir |
| 261 | # Most files can be removed simply by looking at the time, but |
Martin Roth | 659f97c | 2024-01-18 19:06:14 -0700 | [diff] [blame] | 262 | # all Kconfig and Makefiles in the entire tree are accessed |
Martin Roth | 298b007 | 2021-11-10 20:33:44 -0700 | [diff] [blame] | 263 | # whether they're used or not. |
| 264 | remove_kconfigs |
| 265 | |
| 266 | echo |
| 267 | _echo_color "${GREEN}" "Checking access time and removing unused files in:" |
| 268 | for dir in "${CLEAN_DIR_LIST[@]}"; do |
| 269 | printf "%s\n" "${BLD_DIR}/${dir}" |
| 270 | # find and remove all files without updated times. |
| 271 | find "${BLD_DIR}/${dir}" -atime +5 -type f -delete |
| 272 | |
| 273 | recursively_rm_dir_onlyfile "${BLD_DIR}/${dir}" |
| 274 | done |
| 275 | printf "\n\n" |
| 276 | } |
| 277 | |
| 278 | create_patch() { |
| 279 | _echo_color "${GREEN}" "Creating patch" |
| 280 | ( |
| 281 | cd "${BLD_DIR}" |
| 282 | git add -A |
| 283 | git commit -m "remove unused files" --no-verify |
| 284 | git format-patch HEAD^ |
| 285 | ) |
| 286 | } |
| 287 | |
| 288 | main() { |
| 289 | show_version |
| 290 | get_args "$@" |
| 291 | |
| 292 | verify_atime_enabled |
| 293 | update_codebase |
| 294 | save_kconfig |
| 295 | update_times |
| 296 | mark_files_to_keep |
| 297 | build_platform "to mark used files" |
| 298 | OLDHASH="${HASH}" |
| 299 | HASH="" |
| 300 | #show_modified |
| 301 | remove_unused |
| 302 | create_patch |
| 303 | build_platform "to verify the build still works" |
| 304 | NEWHASH="${HASH}" |
| 305 | |
| 306 | echo |
| 307 | _echo_color "${GREEN}" "Checksums:" |
| 308 | echo "Old: ${OLDHASH}" |
| 309 | echo "New: ${NEWHASH}" |
| 310 | } |
| 311 | |
| 312 | main "$@" |