#!/usr/bin/perl -w

# Copyright (C) 2010, 2011 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGE.

# Features to add:
#   - Command line option to run a single test.
#   - Command line option to run all tests in a suite.

use strict;
use warnings;

use File::Basename;
use FindBin;
use Getopt::Long qw(:config pass_through);
use IPC::Open3;
use lib $FindBin::Bin;
use webkitdirs;
use Term::ANSIColor qw(:constants);

sub buildTestTool();
sub dumpAllTests();
sub populateTests();
sub runAllTests();
sub runAllTestsInSuite($);
sub runTest($$);

# Timeout for individual test, in sec
my $timeout = 10;

my $showHelp = 0;
my $verbose = 0;
my $dump = 0;

my $programName = basename($0);
my $usage = <<EOF;
Usage: $programName [options]
  --help                Show this help message
  -v|--verbose          Verbose output
  -d|--dump-tests       Dump the names of testcases without running them
EOF

GetOptions(
    'help' => \$showHelp,
    'verbose|v' => \$verbose,
    'dump|d' => \$dump,
);

if ($showHelp) {
   print STDERR $usage;
   exit 1;
}

setConfiguration();
buildTestTool();
setPathForRunningWebKitApp(\%ENV);
my %testsToRun = populateTests();

if ($dump) {
    dumpAllTests();
    exit 0;
}

if (runAllTests()) {
    exit 1;
}

sub dumpAllTests()
{
    print "Dumping test cases\n";
    print "------------------\n";
    for my $suite (keys %testsToRun) {
        print $suite . ":\n";
        print map { "   " . $_ . "\n" } @{ $testsToRun{$suite} };
    }
    print "------------------\n";
}

sub runAllTests()
{
    my $anyFailures = 0;
    for my $suite (keys %testsToRun) {
        my $failed = runAllTestsInSuite($suite);
        if ($failed) {
            $anyFailures = 1;
        }
    }
    return $anyFailures;
}

sub runAllTestsInSuite($)
{
    my ($suite) = @_;
    print "Suite: $suite\n";

    my $anyFailures = 0;
    for my $test (@{$testsToRun{$suite}}) {
        my $failed = runTest($suite, $test);
        if ($failed) {
            $anyFailures = 1;
        }
    }
   
    return $anyFailures;
}

sub runTest($$)
{
    my ($suite, $testName) = @_;
    my $test = $suite . "." . $testName;

    my $gtestArg = "--gtest_filter=" . $test;

    print "    Test: $testName -> ";

    my $result = 0;
    my $timedOut = 0;
    if (isAppleMacWebKit()) {
        my $productDir = productDir();
        $ENV{DYLD_FRAMEWORK_PATH} = $productDir;
        $ENV{WEBKIT_UNSET_DYLD_FRAMEWORK_PATH} = "YES";
        my $apiTesterPath = "$productDir/TestWebKitAPI";

        local *DEVNULL;
        my ($childIn, $childOut, $childErr);
        if ($verbose) {
            $childOut = ">&STDOUT";
            $childErr = ">&STDERR";
        } else {
            open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
            $childOut = ">&DEVNULL";
            $childErr = ">&DEVNULL";
        }

        my $pid;
        if (architecture()) {
            $pid = open3($childIn, $childOut, $childErr, "arch", "-" . architecture(), $apiTesterPath, $gtestArg, @ARGV) or die "Failed to run test: $test.";
        } else {
            $pid = open3($childIn, $childOut, $childErr, $apiTesterPath, $gtestArg, @ARGV) or die "Failed to run test: $test.";
        }

        close($childIn);
        close($childOut);
        close($childErr);
        close(DEVNULL) unless ($verbose);
        eval {
            local $SIG{ALRM} = sub { die "alarm\n" };
            alarm $timeout;
            waitpid($pid, 0);
            alarm 0;
            $result = $?;
        };
        if ($@) {
            die unless $@ eq "alarm\n";
            kill SIGTERM, $pid or kill SIGKILL, $pid;
            $timedOut = 1;
        };

     } elsif (isAppleWinWebKit()) {
        my $apiTesterNameSuffix;
        if (configurationForVisualStudio() ne "Debug_All") {
            $apiTesterNameSuffix = "";
        } else {
            $apiTesterNameSuffix = "_debug";
        }
        my $apiTesterPath = File::Spec->catfile(productDir(), "TestWebKitAPI$apiTesterNameSuffix.exe");
        eval {
            local $SIG{ALRM} = sub { die "alarm\n" };
            alarm $timeout;
            $result = system { $apiTesterPath } $apiTesterPath, $gtestArg, @ARGV;
            alarm 0;
        };
        if ($@) {
            die unless $@ eq "alarm\n";
            $timedOut = 1;
        };
    } else {
        die "run-api-tests is not supported on this platform.\n"
    }

    if ($timedOut) {
        print BOLD YELLOW, "Timeout", RESET, "\n";
    } elsif (!$result) {
        print BOLD GREEN, "Passed", RESET, "\n";
    } else {
        print BOLD RED, "Failed", RESET, "\n";
    }
    return $timedOut || $result;
}

sub populateTests()
{
    my @tests;
    my $timedOut;

    if (isAppleMacWebKit()) {
        my $productDir = productDir();
        $ENV{DYLD_FRAMEWORK_PATH} = $productDir;
        $ENV{WEBKIT_UNSET_DYLD_FRAMEWORK_PATH} = "YES";
        my $apiTesterPath = "$productDir/TestWebKitAPI";

        local *DEVNULL;
        my ($childIn, $childOut, $childErr);
        if ($verbose) {
            $childErr = ">&STDERR";
        } else {
            open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
            $childErr = ">&DEVNULL";
        }

        my $pid;
        if (architecture()) {
            $pid = open3($childIn, $childOut, $childErr, "arch", "-" . architecture(), $apiTesterPath, "--gtest_list_tests") or die "Failed to build list of tests!";
        } else {
            $pid = open3($childIn, $childOut, $childErr, $apiTesterPath, "--gtest_list_tests") or die "Failed to build list of tests!";
        }

        close($childIn);
        @tests = <$childOut>;
        close($childOut);
        close($childErr);
        close(DEVNULL) unless ($verbose);

        waitpid($pid, 0);
        my $result = $?;

        if ($result) {
            print STDERR "Failed to build list of tests!\n";
            exit exitStatus($result);
        }
    } elsif (isAppleWinWebKit()) {
        my $apiTesterNameSuffix;
        if (configurationForVisualStudio() ne "Debug_All") {
            $apiTesterNameSuffix = "";
        } else {
            $apiTesterNameSuffix = "_debug";
        }
        my $apiTesterPath = File::Spec->catfile(productDir(), "TestWebKitAPI$apiTesterNameSuffix.exe");
        open(TESTS, "-|", $apiTesterPath, "--dump-tests") or die $!;
        @tests = <TESTS>;
        close(TESTS) or die $!;
    } else {
        die "run-api-tests is not supported on this platform.\n"
    }

    my %keyedTests = ();
    my $suite;
    for my $test (@tests) {
       $test =~ s/[\r\n]*$//;
       if ($test =~ m/\.$/) {
          $test =~ s/\.$//;
          $suite = $test;
       } else {
          $test =~ s/^\s*//;
          push @{$keyedTests{$suite}}, $test;
        }
    }
 
    return %keyedTests;
}


sub buildTestTool()
{
    my $originalCwd = getcwd();

    chdirWebKit();

    my $buildTestTool = "build-api-tests";
    print STDERR "Running $buildTestTool\n";

    local *DEVNULL;
    my ($childIn, $childOut, $childErr);
    if ($verbose) {
        # When not quiet, let the child use our stdout/stderr.
        $childOut = ">&STDOUT";
        $childErr = ">&STDERR";
    } else {
        open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
        $childOut = ">&DEVNULL";
        $childErr = ">&DEVNULL";
    }

    my @args = argumentsForConfiguration();
    my $buildProcess = open3($childIn, $childOut, $childErr, "Tools/Scripts/$buildTestTool", @args) or die "Failed to run " . $buildTestTool;

    close($childIn);
    close($childOut);
    close($childErr);
    close(DEVNULL) unless ($verbose);

    waitpid($buildProcess, 0);
    my $buildResult = $?;

    if ($buildResult) {
        print STDERR "Compiling TestWebKitAPI failed!\n";
        exit exitStatus($buildResult);
    }

    chdir $originalCwd;
}
