Martin Roth | ce005da | 2016-04-01 14:30:33 -0600 | [diff] [blame] | 1 | #!/usr/bin/perl |
| 2 | |
| 3 | # |
| 4 | # This file is part of the coreboot project. |
| 5 | # |
| 6 | # Copyright (C) 2015 Google, Inc. |
| 7 | # |
| 8 | # This program is free software; you can redistribute it and/or modify |
| 9 | # it under the terms of the GNU General Public License as published by |
| 10 | # the Free Software Foundation; version 2 of the License. |
| 11 | # |
| 12 | # This program is distributed in the hope that it will be useful, |
| 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 15 | # GNU General Public License for more details. |
| 16 | # |
| 17 | |
| 18 | package gerrit_stats; |
| 19 | |
| 20 | # To install any needed modules install the cpanm app, and use it to install the required modules: |
| 21 | # sudo cpan App::cpanminus |
| 22 | # sudo /usr/local/bin/cpanm JSON::Util Net::OpenSSH DateTime Devel::Size |
| 23 | |
| 24 | use strict; |
| 25 | use warnings; |
| 26 | use English qw( -no_match_vars ); |
| 27 | use File::Find; |
| 28 | use File::Path; |
| 29 | use Getopt::Long; |
| 30 | use Getopt::Std; |
| 31 | use JSON::Util; |
| 32 | use Net::OpenSSH; |
| 33 | use Data::Dumper qw(Dumper); |
| 34 | use DateTime; |
| 35 | use Devel::Size qw(size total_size); |
| 36 | |
| 37 | my $old_version; |
| 38 | my $new_version; |
| 39 | my $infodir="$ENV{'HOME'}/.commit_info/" . `git config -l | grep remote.origin.url | sed 's|.*@||' | sed 's|:.*||'`; |
| 40 | chomp($infodir); |
| 41 | my $URL_WITH_USER; |
| 42 | my $SKIP_GERRIT_CHECK; |
| 43 | my $print_commit_list = 1; |
| 44 | |
| 45 | #disable print buffering |
| 46 | $OUTPUT_AUTOFLUSH = 1; |
| 47 | binmode STDOUT, ":utf8"; |
| 48 | |
| 49 | Main(); |
| 50 | |
| 51 | #------------------------------------------------------------------------------- |
| 52 | # Main |
| 53 | #------------------------------------------------------------------------------- |
| 54 | sub Main { |
| 55 | check_arguments(); |
| 56 | |
| 57 | my %submitters = (); |
| 58 | my %authors = (); |
| 59 | my %owners = (); |
| 60 | my %reviewers = (); |
| 61 | my %author_added = (); |
| 62 | my %author_removed = (); |
| 63 | my $total_added = 0; |
| 64 | my $total_removed = 0; |
| 65 | my $number_of_commits = 0; |
| 66 | my $number_of_submitters = 0; |
| 67 | my $submit_epoch = ""; |
| 68 | my $first_submit_epoch = ""; |
| 69 | if (!$URL_WITH_USER) { |
| 70 | get_user() |
| 71 | } |
| 72 | |
| 73 | # make sure the versions exist |
| 74 | check_versions(); |
| 75 | |
| 76 | #fetch patches if needed. Get ids of first and last commits |
| 77 | my @commits = `git log --pretty=%h "$old_version..$new_version" 2>/dev/null`; |
| 78 | get_commits(@commits); |
| 79 | my $last_commit_id = $commits[0]; |
| 80 | my $first_commit_id = $commits[@commits - 1]; |
| 81 | chomp $last_commit_id; |
| 82 | chomp $first_commit_id; |
| 83 | |
| 84 | print "Statistics from commit $first_commit_id to commit $last_commit_id\n"; |
| 85 | print "Patch, Date, Owner, Author, Submitter, Inserted lines, Deleted lines, Subject, Reviewers\n"; |
| 86 | |
| 87 | #loop through all commits |
| 88 | for my $commit_id (@commits) { |
| 89 | $commit_id =~ s/^\s+|\s+$//g; |
| 90 | |
| 91 | my $submitter = ""; |
| 92 | my %patch_reviewers = (); |
| 93 | my $info; |
| 94 | my $owner; |
| 95 | my $author; |
| 96 | my $author_email; |
| 97 | my $inserted_lines = 0; |
| 98 | my $deleted_lines = 0; |
| 99 | my $subject; |
| 100 | |
| 101 | $number_of_commits++; |
| 102 | print "\"$commit_id\", "; |
| 103 | |
| 104 | #read the data file for the current commit |
| 105 | if (-f "$infodir/$commit_id" && -s "$infodir/$commit_id" > 20) { |
| 106 | open( my $HANDLE, "<", "$infodir/$commit_id" ) or die "Error: could not open file '$infodir/$commit_id'\n"; |
| 107 | $info = <$HANDLE>; |
| 108 | close $HANDLE; |
| 109 | |
| 110 | my $commit_info = JSON::Util->decode($info); |
| 111 | |
| 112 | #get the easy data |
| 113 | $owner = $commit_info->{'owner'}{'name'}; |
| 114 | if (! $owner) { |
| 115 | $owner = $commit_info->{'owner'}{'username'}; |
| 116 | } |
| 117 | if (! $owner) { |
| 118 | $owner = ""; |
| 119 | } |
| 120 | $author = $commit_info->{'currentPatchSet'}{'author'}{'name'}; |
| 121 | $author_email = $commit_info->{'currentPatchSet'}{'author'}{'email'}; |
| 122 | if (! $author) { |
| 123 | $author = $commit_info->{'currentPatchSet'}{'author'}{'username'}; |
| 124 | } |
| 125 | |
| 126 | $inserted_lines = $commit_info->{'currentPatchSet'}{'sizeInsertions'}; |
| 127 | $deleted_lines = $commit_info->{'currentPatchSet'}{'sizeDeletions'}; |
| 128 | $subject = $commit_info->{'subject'}; |
| 129 | |
| 130 | #get the patch's submitter |
| 131 | my $approvals = $commit_info->{'currentPatchSet'}{'approvals'}; |
| 132 | for my $approval (@$approvals) { |
| 133 | if ($approval->{'type'} eq "SUBM") { |
| 134 | $submit_epoch = $approval->{'grantedOn'}; |
| 135 | $submitter = $approval->{'by'}{'name'}; |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | #get all the reviewers for all patch revisions |
| 140 | my $patchsets = $commit_info->{'patchSets'}; |
| 141 | for my $patch (@$patchsets) { |
| 142 | if (! $author) { |
| 143 | $author = $patch->{'author'}{'name'}; |
| 144 | } |
| 145 | my $approvals = $patch->{'approvals'}; |
| 146 | for my $approval (@$approvals) { |
| 147 | |
| 148 | if ( (! $submitter) && ($approval->{'type'} eq "SUBM")) { |
| 149 | $submit_epoch = $approval->{'grantedOn'}; |
| 150 | $submitter = $approval->{'by'}{'name'}; |
| 151 | } |
| 152 | |
| 153 | if ($approval->{'type'} eq "Code-Review") { |
| 154 | my $patch_reviewer = $approval->{'by'}{'name'}; |
| 155 | if ($patch_reviewer) { |
| 156 | if (exists $patch_reviewers{$patch_reviewer}) { |
| 157 | $patch_reviewers{$patch_reviewer}++; |
| 158 | } else { |
| 159 | $patch_reviewers{$patch_reviewer} = 1; |
| 160 | } |
| 161 | } |
| 162 | } |
| 163 | } |
| 164 | } |
| 165 | |
| 166 | } else { |
| 167 | # get the info from git |
| 168 | my $logline = `git log --pretty="%ct@@@%s@@@%an@@@%aE@@@%cn" $commit_id^..$commit_id --`; |
| 169 | $logline =~ m/^(.*)@@@(.*)@@@(.*)@@@(.*)@@@(.*)\n/; |
| 170 | ($submit_epoch, $subject, $author, $author_email, $submitter) = ($1, $2, $3, $4, $5); |
| 171 | $owner = $author; |
| 172 | $logline = `git log --pretty= --shortstat $commit_id^..$commit_id --`; |
| 173 | if ($logline =~ m/\s+(\d+)\s+insertion/) { |
| 174 | $inserted_lines = $1; |
| 175 | } |
| 176 | if ($logline =~ m/\s+(\d+)\s+deletion/) { |
| 177 | $deleted_lines = $1 * -1; |
| 178 | } |
| 179 | my @loglines = `git log $commit_id^..$commit_id -- | grep '\\sReviewed-by:'`; |
| 180 | for my $line (@loglines){ |
| 181 | if ($line =~ m/.*:\s+(.*)\s</) { |
| 182 | my $patch_reviewer = $1; |
| 183 | if ($patch_reviewer) { |
| 184 | if (exists $patch_reviewers{$patch_reviewer}) { |
| 185 | $patch_reviewers{$patch_reviewer}++; |
| 186 | } else { |
| 187 | $patch_reviewers{$patch_reviewer} = 1; |
| 188 | } |
| 189 | } |
| 190 | } |
| 191 | } |
| 192 | |
| 193 | } |
| 194 | |
| 195 | # Not entirely certain why this is needed, but for a number of patches have been submitted |
| 196 | # the submit time in gerrit is set to April 9, 2015. |
| 197 | if ($submit_epoch == 1428586219){ |
| 198 | my $logline = `git log --pretty="%ct" $commit_id^..$commit_id --`; |
| 199 | $logline =~ m/^(.*)\n/; |
| 200 | $submit_epoch = $1; |
| 201 | } |
| 202 | |
| 203 | #add the count and owner to the submitter hash |
| 204 | if (exists $submitters{$submitter}) { |
| 205 | $submitters{$submitter}++; |
| 206 | } else { |
| 207 | $submitters{$submitter} = 1; |
| 208 | $number_of_submitters++; |
| 209 | } |
| 210 | |
| 211 | #create a readable date |
| 212 | my $dt = DateTime->from_epoch(epoch => $submit_epoch); |
| 213 | $dt->set_time_zone( 'Europe/Paris' ); |
| 214 | my $submit_time = $dt->strftime('%Y/%m/%d %H:%M:%S'); |
| 215 | if (!$first_submit_epoch) { |
| 216 | $first_submit_epoch = $submit_epoch; |
| 217 | } |
| 218 | |
| 219 | #create the list of reviewers to print |
| 220 | my $reviewerlist = ""; |
| 221 | foreach my $reviewer (keys %patch_reviewers) { |
| 222 | if ($reviewerlist eq "") { |
| 223 | $reviewerlist = $reviewer; |
| 224 | } else { |
| 225 | $reviewerlist .= ", $reviewer"; |
| 226 | } |
| 227 | |
| 228 | if (exists $reviewers{$reviewer}) { |
| 229 | $reviewers{$reviewer}++; |
| 230 | } else { |
| 231 | $reviewers{$reviewer} = 1; |
| 232 | } |
| 233 | } |
| 234 | if (! $reviewerlist) { |
| 235 | $reviewerlist = "-" |
| 236 | } |
| 237 | |
| 238 | if ($print_commit_list) { |
| 239 | print "$submit_time, $owner, $author, $submitter, $inserted_lines, $deleted_lines, \"$subject\", \"$reviewerlist\"\n"; |
| 240 | } else { |
| 241 | print "$number_of_commits\n"; |
| 242 | } |
| 243 | $total_added += $inserted_lines; |
| 244 | $total_removed += $deleted_lines; |
| 245 | if (exists $owners{$owner}) { |
| 246 | $owners{$owner}++; |
| 247 | } else { |
| 248 | $owners{$owner} = 1; |
| 249 | } |
| 250 | |
| 251 | if (exists $authors{$author}{"num"}) { |
| 252 | $authors{$author}{"num"}++; |
| 253 | $author_added{$author} += $inserted_lines; |
| 254 | $author_removed{$author} += $deleted_lines; |
| 255 | $authors{$author}{"earliest_commit"}=$submit_time; |
| 256 | } else { |
| 257 | $authors{$author}{"num"} = 1; |
| 258 | $authors{$author}{"latest_commit"}=$submit_time; |
| 259 | $authors{$author}{"earliest_commit"}=$submit_time; |
| 260 | $author_added{$author} = $inserted_lines; |
| 261 | $author_removed{$author} = $deleted_lines; |
| 262 | } |
| 263 | if (! exists $authors{$author}{email} && $author_email) { |
| 264 | $authors{$author}{email} = "$author_email"; |
| 265 | } |
| 266 | } |
| 267 | my $Days = ($first_submit_epoch - $submit_epoch) / 86400; |
| 268 | if (($first_submit_epoch - $submit_epoch) % 86400) { |
| 269 | $Days += 1; |
| 270 | } |
| 271 | |
| 272 | print "- Total Commits: $number_of_commits\n"; |
| 273 | printf "- Average Commits per day: %.2f\n", $number_of_commits / $Days; |
| 274 | print "- Total lines added: $total_added\n"; |
| 275 | print "- Total lines removed: $total_removed\n"; |
| 276 | print "- Total difference: " . ($total_added + $total_removed) . "\n\n"; |
| 277 | |
| 278 | print "=== Authors - Number of commits ===\n"; |
| 279 | my $number_of_authors = 0; |
| 280 | foreach my $author (sort { $authors{$b}{num} <=> $authors{$a}{num} } (keys %authors) ) { |
| 281 | if (! exists $authors{$author}{"email"}) { |
| 282 | $authors{$author}{"email"} = "-"; |
| 283 | } |
| 284 | printf "%-25s %5d %-40s (%2.2f%%) {%s / %s}\n",$author, $authors{$author}{"num"}, $authors{$author}{"email"}, $authors{$author}{"num"} / $number_of_commits * 100, $authors{$author}{"latest_commit"}, $authors{$author}{"earliest_commit"}; |
| 285 | $number_of_authors++; |
| 286 | } |
| 287 | print "Total Authors: $number_of_authors\n\n"; |
| 288 | |
| 289 | print "=== Authors - Lines added ===\n"; |
| 290 | foreach my $author (sort { $author_added{$b} <=> $author_added{$a} } (keys %author_added) ) { |
| 291 | if ($author_added{$author}) { |
| 292 | printf "%-25s %5d (%2.3f%%)\n",$author, $author_added{$author}, $author_added{$author} / $total_added * 100; |
| 293 | } |
| 294 | } |
| 295 | print "\n"; |
| 296 | |
| 297 | print "=== Authors - Lines removed ===\n"; |
| 298 | foreach my $author (sort { $author_removed{$a} <=> $author_removed{$b} } (keys %author_removed) ) { |
| 299 | if ($author_removed{$author}) { |
| 300 | printf "%-25s %5d (%2.3f%%)\n",$author,$author_removed{$author} * -1, $author_removed{$author} / $total_removed * 100; |
| 301 | } |
| 302 | } |
| 303 | print "\n"; |
| 304 | |
| 305 | print "=== Reviewers - Number of patches reviewed ===\n"; |
| 306 | my $number_of_reviewers = 0; |
| 307 | foreach my $reviewer (sort { $reviewers{$b} <=> $reviewers{$a} } (keys %reviewers) ) { |
| 308 | printf "%-25s %5d (%2.3f%%)\n",$reviewer, $reviewers{$reviewer}, $reviewers{$reviewer} / $number_of_commits * 100; |
| 309 | $number_of_reviewers++; |
| 310 | } |
| 311 | print "Total Reviewers: $number_of_reviewers\n\n"; |
| 312 | |
| 313 | print "=== Submitters - Number of patches submitted ===\n"; |
| 314 | foreach my $submitter (sort { $submitters{$b} <=> $submitters{$a} } (keys %submitters) ) { |
| 315 | printf "%-25s %5d (%2.3f%%)\n",$submitter, $submitters{$submitter}, $submitters{$submitter} / $number_of_commits * 100; |
| 316 | } |
| 317 | print "Total Submitters: $number_of_submitters\n\n"; |
| 318 | |
| 319 | print "Commits, Ave, Added, Removed, Diff, Authors, Reviewers, Submitters\n"; |
| 320 | printf "$number_of_commits, %.2f, $total_added, $total_removed, " . ($total_added + $total_removed) . ", $number_of_authors, $number_of_reviewers, $number_of_submitters\n", $number_of_commits / $Days; |
| 321 | } |
| 322 | |
| 323 | #------------------------------------------------------------------------------- |
| 324 | #------------------------------------------------------------------------------- |
| 325 | sub check_versions { |
| 326 | `git cat-file -e $old_version^{commit} 2>/dev/null`; |
| 327 | if (${^CHILD_ERROR_NATIVE}){ |
| 328 | print "Error: Old version ($old_version) does not exist.\n"; |
| 329 | exit 1; |
| 330 | } |
| 331 | |
| 332 | `git cat-file -e $new_version^{commit} 2>/dev/null`; |
| 333 | if (${^CHILD_ERROR_NATIVE}){ |
| 334 | print "Error: New version ($new_version) does not exist.\n"; |
| 335 | exit 1; |
| 336 | } |
| 337 | } |
| 338 | |
| 339 | #------------------------------------------------------------------------------- |
| 340 | #------------------------------------------------------------------------------- |
| 341 | sub get_user { |
| 342 | my $url=`git config -l | grep remote.origin.url`; |
| 343 | |
| 344 | if ($url =~ /.*url=ssh:\/\/(\w+@[a-zA-Z][a-zA-Z0-9\.]+:\d+)/) |
| 345 | { |
| 346 | $URL_WITH_USER = $1; |
| 347 | } else { |
| 348 | print "Error: Could not get a ssh url with a username from gitconfig.\n"; |
| 349 | print " use the -u option to set a url.\n"; |
| 350 | exit 1; |
| 351 | } |
| 352 | } |
| 353 | |
| 354 | #------------------------------------------------------------------------------- |
| 355 | #------------------------------------------------------------------------------- |
| 356 | sub get_commits { |
| 357 | my @commits = @_; |
| 358 | my $submit_time = ""; |
| 359 | if (defined $SKIP_GERRIT_CHECK) { |
| 360 | return; |
| 361 | } |
| 362 | my $ssh = Net::OpenSSH->new("$URL_WITH_USER", ); |
| 363 | $ssh->error and die "Couldn't establish SSH connection to $URL_WITH_USER:". $ssh->error; |
| 364 | |
| 365 | print "Using URL: ssh://$URL_WITH_USER\n"; |
| 366 | |
| 367 | if (! -d $infodir) { |
| 368 | mkpath($infodir) |
| 369 | } |
| 370 | |
| 371 | for my $commit_id (@commits) { |
| 372 | $commit_id =~ s/^\s+|\s+$//g; |
| 373 | $submit_time = ""; |
| 374 | my $gerrit_review; |
| 375 | |
| 376 | # Quit if we've reeached the last coreboot commit supporting these queries |
| 377 | if ($commit_id =~ /^7309709/) { |
| 378 | last; |
| 379 | } |
| 380 | |
| 381 | if (-f "$infodir/$commit_id") { |
| 382 | $gerrit_review = 1; |
| 383 | } else { |
| 384 | $gerrit_review = `git log $commit_id^..$commit_id | grep '\\sReviewed-on:\\s'`; |
| 385 | } |
| 386 | |
| 387 | if ($gerrit_review && $commit_id && (! -f "$infodir/$commit_id") ) { |
| 388 | print "Downloading $commit_id"; |
| 389 | my @info = $ssh->capture("gerrit query --format=JSON --comments --files --current-patch-set --all-approvals --submit-records --dependencies commit:$commit_id"); |
| 390 | $ssh->error and die "remote ls command failed: " . $ssh->error; |
| 391 | |
| 392 | my $commit_info = JSON::Util->decode($info[0]); |
| 393 | my $rowcount = $commit_info->{'rowCount'}; |
| 394 | if (defined $rowcount && ($rowcount eq "0")) { |
| 395 | print " - no gerrit commit for that id.\n"; |
| 396 | open( my $HANDLE, ">", "$infodir/$commit_id" ) or die "Error: could not open file '$infodir/$commit_id'\n"; |
| 397 | print $HANDLE "No gerrit commit"; |
| 398 | close $HANDLE; |
| 399 | next; |
| 400 | } |
| 401 | my $approvals = $commit_info->{'currentPatchSet'}{'approvals'}; |
| 402 | |
| 403 | for my $approval (@$approvals) { |
| 404 | if ($approval->{'type'} eq "SUBM") { |
| 405 | $submit_time = $approval->{'grantedOn'} |
| 406 | } |
| 407 | } |
| 408 | my $dt=""; |
| 409 | if ($submit_time) { |
| 410 | $dt = DateTime->from_epoch(epoch => $submit_time); |
| 411 | } else { |
| 412 | print " - no submit time for that id.\n"; |
| 413 | open( my $HANDLE, ">", "$infodir/$commit_id" ) or die "Error: could not open file '$infodir/$commit_id'\n"; |
| 414 | print $HANDLE "No submit time"; |
| 415 | close $HANDLE; |
| 416 | |
| 417 | next; |
| 418 | } |
| 419 | |
| 420 | open( my $HANDLE, ">", "$infodir/$commit_id" ) or die "Error: could not open file '$infodir/$commit_id'\n"; |
| 421 | print $HANDLE $info[0]; |
| 422 | close $HANDLE; |
| 423 | |
| 424 | $dt->set_time_zone( 'Europe/Paris' ); |
| 425 | print " - submit time: " . $dt->strftime('%Y/%m/%d %H:%M:%S') . "\n"; |
| 426 | } elsif ($commit_id && (! -f "$infodir/$commit_id")) { |
| 427 | print "No gerrit commit for $commit_id\n"; |
| 428 | open( my $HANDLE, ">", "$infodir/$commit_id" ) or die "Error: could not open file '$infodir/$commit_id'\n"; |
| 429 | print $HANDLE "No gerrit commit"; |
| 430 | close $HANDLE; |
| 431 | } |
| 432 | } |
| 433 | print "\n"; |
| 434 | } |
| 435 | |
| 436 | #------------------------------------------------------------------------------- |
| 437 | # check_arguments parse the command line arguments |
| 438 | #------------------------------------------------------------------------------- |
| 439 | sub check_arguments { |
| 440 | my $show_usage = 0; |
| 441 | GetOptions( |
| 442 | 'help|?' => sub { usage() }, |
| 443 | 'url|u=s' => \$URL_WITH_USER, |
| 444 | 'skip|s' => \$SKIP_GERRIT_CHECK, |
| 445 | ); |
| 446 | # strip ssh:// from url if passed in. |
| 447 | if (defined $URL_WITH_USER) { |
| 448 | $URL_WITH_USER =~ s|ssh://||; |
| 449 | } |
| 450 | if (@ARGV) { |
| 451 | ($old_version, $new_version) = @ARGV; |
| 452 | } else { |
| 453 | usage(); |
| 454 | } |
| 455 | } |
| 456 | |
| 457 | #------------------------------------------------------------------------------- |
| 458 | # usage - Print the arguments for the user |
| 459 | #------------------------------------------------------------------------------- |
| 460 | sub usage { |
| 461 | print "gerrit_stats <options> [Old version] [New version]\n"; |
| 462 | print "Old version should be a tag (4.1), a branch (origin/4.1), or a commit id\n"; |
| 463 | print "New version can be 'HEAD' a branch (origin/master) a tag (4.2), or a commit id\n"; |
| 464 | print " Options:\n"; |
| 465 | print " u | url [url] url with username.\n"; |
| 466 | print "Example: \"$0 -u Gaumless\@review.coreboot.org:29418 origin/4.1 4.2\"\n"; |
| 467 | exit(0); |
| 468 | } |
| 469 | |
| 470 | 1; |