ci-regression.sh

#!/bin/bash +x

set -e  # exit if any command fails

if [ -z "$1" ]; then
  echo "No directory specified to check out master for regression tests."
  echo "Aborting!"
  exit 1
fi
export TEST_HARNESS_MASTER_DIR="$1"

if [ -n "$2" ]; then
  MAKE_JOBS=$2
  echo "Make parallelism set to '$MAKE_JOBS'"
else
  MAKE_JOBS=2  # all machines are dual-core or better as of 2015, so sayeth the king
  echo "Make parallelism defaulted to 2."
fi

GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2)
if [ -z "$(git status | grep detached)" ]; then
  GIT_DETACHED="FALSE"
else
  GIT_DETACHED="TRUE"
fi

if [ -d "${TEST_HARNESS_MASTER_DIR}" ]; then
  read -p "Remove existing directory ${TEST_HARNESS_MASTER_DIR} (y/n)? " CONT
  if [ "$CONT" = "y" ]; then
    rm -rf "${TEST_HARNESS_MASTER_DIR}"
  fi
fi

mkdir -p "${PWD}/test-harness-out"

echo "Copying ${PWD} to ${TEST_HARNESS_MASTER_DIR}"
cp -p -r "${PWD}" "${TEST_HARNESS_MASTER_DIR}"

PREVIOUS_PWD=${PWD}
pushd "${TEST_HARNESS_MASTER_DIR}"
make all -j$MAKE_JOBS

TESTS_LIST=(`./test-runner --gtest_list_tests | awk '/\.$/{ TS=$1 }  /  / { print TS $1 }'`)
TOTAL_TESTS=${#TESTS_LIST[@]}
TESTS_TO_RUN=TOTAL_TESTS/TOTAL_AGENTS
FIRST_TEST=TESTS_TO_RUN-AGENT_N*TESTS_TO_RUN
if [[ $AGENT_N -eq $TOTAL_AGENTS ]]; then
  printf -v FILTER_ARG "%s:" "${TESTS_LIST[@]:$FIRST_TEST:$((TESTS_TO_RUN+TOTAL_TESTS%TOTAL_AGENTS))}"
else
  printf -v FILTER_ARG "%s:" "${TESTS_LIST[@]:$FIRST_TEST:$TESTS_TO_RUN}"
fi

if [ "$TRAVIS_OS_NAME" == "windows" ]; then
  ./test-runner --gtest_filter=$FILTER_ARG-Regression.image_load_save_test --gtest_output=xml:test-harness-out/TestReport${AGENT_N}.xml
else
  ./test-runner --gtest_filter=$FILTER_ARG --gtest_output=xml:test-harness-out/TestReport${AGENT_N}.xml
fi

if [[ "$TRAVIS" -eq "true" ]]; then
  # upload coverage report before running regression tests
  bash <(curl -s https://codecov.io/bash) -f "*.info" -t "$_CODECOV_UPLOAD_TOKEN"
fi
# move output to safe space until we can compare
mv ./test-harness-out ${PREVIOUS_PWD}
# clean before we checkout
make clean

if [[ "${PWD}" == "${TEST_HARNESS_MASTER_DIR}" ]]; then
  git stash

  if [[ -n "$TRAVIS_PULL_REQUEST_SHA" ]] && [[ -n "$TRAVIS_BRANCH" ]]; then
    echo "This appears to be a Travis pull request integration run; checking out '$TRAVIS_BRANCH' for the comparison."
    git checkout "$TRAVIS_BRANCH"
  elif [[ -n "$TRAVIS_COMMIT_RANGE" ]]; then
    prev=${TRAVIS_COMMIT_RANGE%%.*}
    echo "This appears to be a Travis push integration run; checking out '$prev' for the comparison."
    git checkout "$prev"
  elif [[ "${GIT_BRANCH}" == "master" && "${GIT_DETACHED}" == "FALSE" ]]; then
    echo "You appear to be on branch master with no changes. Checking out HEAD~1 for the comparison"
    git checkout HEAD~1
  else
    echo "You appear to be on branch ${GIT_BRANCH}. Checking out branch master for the comparison"
    git checkout master
  fi
  git clean -f -d

  # re-install deps incase they've changed
  echo "Reinstalling deps"
  ./CI/install_emake_deps.sh && ./CI/split_jobs.sh install

  echo "Rebuilding plugin and harness from last commit..."
  make all -j$MAKE_JOBS
  echo "Generating regression comparison images..."
  mkdir -p "${PWD}/test-harness-out"
  if [ "$TRAVIS_OS_NAME" == "windows" ]; then
    ./test-runner --gtest_filter=$FILTER_ARG-Regression.image_load_save_test
  else
    ./test-runner --gtest_filter=$FILTER_ARG
  fi

  popd
else
  echo "Failed to change directory to ${TEST_HARNESS_MASTER_DIR}. Something is horribly wrong..."
  echo "Aborting!"
  exit 1
fi

./CommandLine/testing/GitHub-ImageDiff.sh