#!/bin/bash
#-----------------------------------------------------------------------------
# File        : erlang_automated_build
# Author      : Gordon Guthrie <gordonguthrie@backawinner.gg>
# Description : This is an automated build script for Erlang development
#               Please run the script with the following options for details
#
#               ./erlang_automated_build -h
#
# Created     : 10 December 2005 by Gordon Guthrie
#
# License     : This script is licensed under the GPL V2.0
#               but may be released under any later version at your discretion
#
                VERSION="1.0"
#
#-----------------------------------------------------------------------------
#
#    This program is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 2, or (at your option)
#    any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Boston, MA 02111.
#
#    A copy of this license can be found at:
#    http://www.gnu.org/licenses/info/GPLv2orLater.html
#
#-----------------------------------------------------------------------------

function displayscripthelp
{
cat <<EOF
Introduction
------------
This script performs the daily build for an Erlang system. It has
ONLY been tested on SuSE 9.2.

Purpose
-------
This script can perform the following build tasks:
* take a CVS extract of code
* compile that source code and check the results
* run a test suite
* run the dialyzer
* run tsunami
* graphs all or some of the previous results over time
* copy the graphs to some other location

How To Use This Script
----------------------
The following steps should be followed to implement this script.

1  Run the configuration test

   ./erlang_automated_build -c

   This will attempt to autodetect the components it needs and report back
   success or failure. It will also dump out the values of all the hardwired
   values in this script. You have the option of either editing them here or
   overriding them with command line options.

2  Decide on how you wish to implement this script

   This script can be run in a variety of ways:
   * a monolithic installation  - all steps happen on the same machine
   * a distributed installation - different steps happen on different machines

   The graphing options can be configured such that distributed applications
   produce integrated output.

3  Ensure that you can CVS your code down

   You need to set up CVS to work without having to put in a password

   Run the following at the command line to just run the CVS part to test
   if it is working

   erlang_automated_build -B -C -F -G -I -v

   This script is set up to use an ssh batch wrapper script which
   is in /src and is called erl_auto_bld_ssh_wrapper

   The location of this file is held in the user editable variable:
   * CVS_RSH_SUBSTITUTE

4  Set up a pre-compilation script file to prepare the build

   This script will run an external batch file prior to executing
   the compile - this file needs to exist (even if it does nothing).
   This facility exists in case your CVS needs unpacking or any other
   changes prior to running the compile phase

   The path & filename of this script is in the user editable variable section:
   * PREPARE_BLD_SCRIPT
   It can be overridden with the option:
   * -P PATH/FILE

5  Configure the Erlang helper function

   There is a helper function which you need to set up:
   * compile_code.erl (see src directory)

   This module has the fuctions you wish to compile hard coded in it and
   must be appropriately edited - or replace it with one of your own.

   The return from this file is used in the graphing:
   * every line that contains 'OK: ' will count as 1 successful compile
   * everyline that contains 'Compile failure: ' will count as 1 failed compile

   The path to this file is in the user editable variable section:
   * HELPER_FUN_LOC
   It can be overridden with the option:
   * -H PATH/FILE

6  Configure the test server helper function

   You need to install and compile the test server helper function:
   * ts_stub.erl (see the src directory)

   It needs to go in the test_server directory with all the other
   ts_*.erl/ts_*.beam files

   This function is needed coz the erl batch command dont take lists as a
   parameter.It doesnt need to be edited. It is passed the test suite spec
   on the command line

7  Set up a pre-tsunami script

   This script will run an external batch file prior to executing
   the load test - this file needs to exist (even if it does nothing).
   This facility exists to start your test system or insert test data or
   any other changes prior to running the load tester

   The pre-tsunami script is started as a background shell with no contact
   with this script. The pre-tsunami script signals to this script via a
   canary - a file whose existance (or not signals) that
   erlang_automated_build can continue and kick off tsunami. An example
   of this shown in the following files available in /doc/examples/src:
   * pre-tsunami.script
   * start_app.erl
 
   The path and filename of this script is in user editable variable section:
   * PRE_TSUNAMI_SCRIPT
   It can be overridden with the option:
   * -Q PATH/FILE

   The path and filename of the canary is in the user editable variable section:
   * WHERES_THE_CANARY
   It can be overridden with the option
   * -C PATH/FILE

8  Create your load testing script

   The path and filename of this script is help in the user editable variable:
   * TSUNAMI_XML_FILE
   It can be overridden with the option:
   * -R PATH/FILE

9  Set up a post-tsunami script

   This script can clean up after your tsunami load testing (ie kill the 
   system under test)

   An example of a post-tsunami script can be found in /doc/examples/src:
   * post-tsunami.script

   The path and filename of this script is help in the user editable variable:
   * POST_TSUNAMI_SCRIPT
   It can be overridden with the option:
   * -S PATH/FILE

10 Set up a location to copy your results to

   This programme will run a series of secure copies with the format:
   * scp [options] results_files username@host:/path/to/directory

   IDX-Tsnumi expects its graphs to be in subdirectory called /images
   be sure to create that as well

   The user editable variable SCP_LOC contains:
   * 'username@host:/path/to/directory'
   It can be overridden with:
   * -Z username@host:/path/to/directory

   scp will use a private key to copy the data.

   The user editable variable SCP_PRIV_KEY contains:
   * /path/to/private.key
   It can be overridden with
   * -z /path/to/private.key

11 Set up the general configuration of the script

   There is a large section of this script marked as user editable
   which contains a series of variable definitions that define where various
   things are to be found. The values of these should be set edited. Explicit
   file locations are specified so that the script can easily run as a cron job
  
   In future version of this script the user editable section will be
   'copy'n'pastable'

When It Will Fail
-----------------

This script contains a Y3K bug - find it and win a coconut!

Prerequesites
-------------

This script requires the following to be installed and working:
* the Erlang base system
  (http://www.erlang.org)
* a CVS client
  (http://savannah.nongnu.org/projects/cvs/)
* the Erlang Test server (and a suite of tests)
  (http://www.erlang.org/project/test_server/)
* the Dialyzer
  (http://www.it.uu.se/research/group/hipe/dialyzer/)
* Tsunami
  (http://www.idealx.org/prj/idx-tsunami/ (french))
  (http://www.idealx.org/prj/idx-tsunami/index.en.html (english))
* Gnuplot
  (http://www.gnuplot.info/)
* Secure Shell
  (http://www.openssh.com/)

Not all the components are needed to run all the parts of the script
but if components are missing then it will not be possible to run the
appropriate section.

Core Activities
---------------
This script performs the following core activities:
* downloads source code from CVS
* compiles the source code
  - runs a prepare build script
  - checks the results
  - graphs the result over time (ie multiple invocations of the script)
  - copies the results to a location
* runs a test suite
  - graphs the results over time (ie multiple invocations of the script)
  - copies the results to a location
* runs the dialyzer
  -  graphs the results over time (ie multiple invocations of the script)
  - copies the results to a location
* runs tsunami
  - runs a pre-tsunami script to set up the system for load testing
  - graphs the results over time (ie multiple invocations of the script)
  - copies the results to a location
* e-mails a defined user list on failure (or success)

Options
-------

erlang_automated_build takes the following options:

    General Options
    ---------------
    -c   checks for prerequisites and dumps the configuration
    -h	 displays this information
    -v   runs in verbose mode
    -d   runs in debug mode (ie super verbose)
    -V   gives the version

    Overriding User Edited Variables
    --------------------------------
    -E [DIR]                 location of Erlang installation
    -L [DIR]                 local CVSROOT
                             (ie where CVS will download the source code to)
    -P [PATH/FILE]           location of prepare build script
    -H [DIR]                 location of compile_code.erl function
                             (the helper function)
    -Q [PATH/FILE]           location of pre-tsunami load script
    -W [PATH/FILE]           location of the canary
    -R [PATH/FILE]           location of tsunami xml file
    -S [PATH/FILE]           location of post-tsunami load script
    -X [e-mail addresses]    who to notify on success
                             a list of e-mail adresses seperated by a semi-colon 
    -Y [e-mail addresses]    who to notify on failure
                             a list of e-mail adresses seperated by a semi-colon
    -Z [user@host:/path/]    an alternate place to copy the results to
    -z [PATH/FILE]           an alternative private key to use for the scp copy

    Output Options
    --------------
    -r [DIR]   change the location where reports are to output to

    Flow Of Control Options
    -----------------------
    -A   dont cvs new code down
    -B   dont compile the code
    -C   dont run the test suite
    -F   dont run the dialyser
    -G   dont run tsunami
    -I   dont run gnuplot
    -J   dont copy the output anywhere
    -K   dont backup the code before CVSing down a new version
         delete it instead

EOF
}
STARTTIME=`date`
################################################################################
#                                                                              #
# STEP 1 set up the global variables                                           #
#                                                                              #
################################################################################

# Firstly set up the variables that can be set in options
DUMPCONFIG="false"
SCRIPTHELP="false"
VERBOSE="false"
INVALID="false"
DEBUG="false"
SHOW_VERSION="false"
SKIP_CVS="false"
SKIP_COMPILE="false"
SKIP_TEST_SUITE="false"
SKIP_DIALYZER="false"
SKIP_TSUNAMI="false"
SKIP_GNUPLOT="false"
SKIP_SECURE_COPY="false"
SKIP_PRECVS_BACKUP="false"

# Now set up variables that are used in the body of the kirk
HAS_ANY_STEP_FAILED="false"
PREPARE_TO_EXIT="false"
SENDMAIL_TO=""
SENDMAIL_SUBJECT=""
SENDMAIL_BODY=""

################################################################################
#  _    _                 ______    _ _ _   
# | |  | |               |  ____|  | (_) |  
# | |  | |___  ___ _ __  | |__   __| |_| |_ 
# | |  | / __|/ _ \ '__| |  __| / _` | | __|
# | |__| \__ \  __/ |    | |___| (_| | | |_ 
#  \____/|___/\___|_|    |______\__,_|_|\__|
#
# All the user editable constants are in this section
#
# In the event of a script update this section can be copied over wholesale
#
# The standard locations of components
ERLANG_LOC="/usr/local/lib/erlang"
DOWNLOAD_LOC="/opt"
LOCAL_CVSROOT="$DOWNLOAD_LOC/erlang_apps"
TEST_SERVER_LOC="/opt/erlang_apps/test_output/test_server"
DIALYZER_LOC="/opt/dialyzer"
TSUNAMI_LOC="$ERLANG_LOC/lib/tsunami-1.1.0"
TSU_ANALYSE_LOC="/usr/local/lib/idx-tsunami/bin"
# A script to run after the CVS download is complete
PREPARE_BLD_SCRIPT="/opt/erlang_apps/myapp/prepare_build"
# A function that is called to perform the erlang compile
HELPER_FUN_LOC="$LOCAL_CVSROOT/myapp/ebin"
# A script to run before running tsunami
PRE_TSUNAMI_SCRIPT="/opt/erlang_apps/myapp/pre-tsunami.script"
WHERES_THE_CANARY="/tmp"
# The tsunami xml file to rung
TSUNAMI_XML_FILE="/opt/erlang_apps/myapp/conf/tsunami_load.xml"
POST_TSUNAMI_SCRIPT="/opt/erlang_apps/myapp/post-tsunami.script"
# The format of SCP_LOC is username@host:/path/to/directory/
# needs final slash
SCP_LOC="username@host:/path/to/dir/under/webroot/"
# SCP will use a private key to copy data up  - it needs to be specified
SCP_PRIV_KEY="/opt/mykey.priv"
SENDMAIL_LOC="/usr/lib"

# Where reports will be run to
REPORT_LOC="/tmp/eab_reports"

# Now set up some other constants
SYSTEM_NAME="MyApp"
# List of e-mail addresses seperated by a semi-colon
NOTIFY_ON_FAIL="user1@hostname; user2@hostname"
# List of e-mail addresses seperated by a semi-colon
NOTIFY_ON_SUCCESS="user1@hostname; user2@hostname; user3@hostname"
DAILY_BUILD_FROM="erlang_automated_build_robot@hostname"
CVS_COMMAND_LINE='cvs -d :ext:user@host:/CVSROOT co myapp'
CVS_RSH_SUBSTITUTE="/opt/erl_auto_bld_ssh_wrapper" # You might want to put ssh in here or rsh, whatever...
TEST_SUITE_SPEC="myapp"

#
# Dialyzer offset - this should be at least -1 but you may have hooky
# code in the scan path - use this variable to bodge that
DIALYZER_OFFSET=5

# The tsunami load script can take a long time to run
# depends on you! - there is a 'comfort' write out that polls regularly so
# that you know it is still working and hasn't hung. This variable controls
# how often if polls in seconds
#
# It is recommended that you set it to:
# * 10 to 15 in development
# * 3600 or above in production
SLEEP_TIME=3
#  ______    _ _ _     ______           _ 
# |  ____|  | (_) |   |  ____|         | |
# | |__   __| |_| |_  | |__   _ __   __| |
# |  __| / _` | | __| |  __| | '_ \ / _` |
# | |___| (_| | | |_  | |____| | | | (_| |
# |______\__,_|_|\__| |______|_| |_|\__,_|
#
################################################################################

# When are we running this script?
TIMESTAMP=$(date +"%Y_%m_%d_%H:%M:%S")
REPORT_TIME_FORMAT=$(date +"%Y/%m/%d/%H/%M")

# Now I'm going to knock up some derivative variables for convenience
#
# Compile Results Files
#
RL_TOTAL_COMPS=$REPORT_LOC/compile_total_files.txt
RL_TOTAL_COMP_FAILS=$REPORT_LOC/compile_fails.txt
RL_TOTAL_COMP_SUCCS=$REPORT_LOC/compile_success.txt
#
# Test Results Files
#
RL_TOTAL_TEST_CASES=$REPORT_LOC/test_total.txt
RL_TOTAL_OK_TESTS=$REPORT_LOC/test_ok.txt
RL_TOTAL_FAILED_TESTS=$REPORT_LOC/test_fail.txt
RL_TOTAL_SKIPPED_TESTS=$REPORT_LOC/test_skipped.txt
RL_TEST_PERC_COV=$REPORT_LOC/test_perc_covered.txt
RL_LINES_COV=$REPORT_LOC/lines_cov.txt
RL_LINES_NOT_COV=$REPORT_LOC/lines_not_cov.txt
RL_TOTAL_LINES=$REPORT_LOC/lines_total.txt
#
# Dialyzer Results File
#
RL_DIALYZER=$REPORT_LOC/dialyzer_totals.txt
################################################################################
#                                                                              #
# STEP 3 Define the functions                                                  #
#                                                                              #
################################################################################
function fix_me
{
    echo "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%"
    echo "%%%                 ______ _        __  __                    %%%"
    echo "%%%                |  ____(_)      |  \/  |                   %%%"
    echo "%%%                | |__   ___  __ | \  / | ___               %%%"
    echo "%%%                |  __| | \ \/ / | |\/| |/ _ \              %%%"
    echo "%%%                | |    | |>  <  | |  | |  __/              %%%"
    echo "%%%                |_|    |_/_/\_\ |_|  |_|\___|              %%%"
    echo "%%%                                                           %%%"
    echo "%%%                    Alls seems clear here!                 %%%"
    echo "%%%                                                           %%%"
    echo "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%"
}

function print_GPL
{
echo "#-----------------------------------------------------------------------------"
echo "#"
echo "#    This program is free software; you can redistribute it and/or modify"
echo "#    it under the terms of the GNU General Public License as published by"
echo "#    the Free Software Foundation; either version 2, or (at your option)"
echo "#    any later version."
echo "#"
echo "#    This program is distributed in the hope that it will be useful,"
echo "#    but WITHOUT ANY WARRANTY; without even the implied warranty of"
echo "#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the"
echo "#    GNU General Public License for more details."
echo "#"
echo "#    You should have received a copy of the GNU General Public License"
echo "#    along with this program; if not, write to the Free Software"
echo "#    Foundation, Inc., 59 Temple Place, Boston, MA 02111."
echo "#"
echo "#    A copy of this license can be found at:"
echo "#    http://www.gnu.org/licenses/info/GPLv2orLater.html"
echo "#"
echo "#-----------------------------------------------------------------------------"
}

function check_components   
{
    if (test -f "$ERLANG_LOC/bin/erl") ; then
       echo "* Erlang                           seems OK..."
    else
       echo "* missing erl                      -- check your Erlang/OTP installation"
    fi

    if (test -f "$TEST_SERVER_LOC/ts.erl") ; then
       echo "* Test Server                      seems OK..."
    else
       echo "* missing ts.erl                   -- check your Test Server installation"
    fi

    if (test -f "$DIALYZER_LOC/dialyzer") ; then
       echo "* Dialyzer                         seems OK..."
    else
       echo "* missing dialyzer                 -- check your Dialyzer installation"
    fi

    if (test -f "$TSUNAMI_LOC/ebin/tsunami.app") ; then
       echo "* Tsuanmi                          seems OK..."
    else
       echo "* missing tsunami.app              -- check your Tsunami installation"
    fi

    if (test -f "$TSU_ANALYSE_LOC/analyse_msg.pl") ; then
       echo "* Tsuanmi analyse_msg.pl           seems OK..."
    else
       echo "* missing analyse_msg.pl           -- check your Tsunami installation"
    fi

    if (test -f "$PREPARE_BLD_SCRIPT") ; then
       echo "* Prepare build script             seems OK..."
    else
       echo "* missing prepare build script     -- check your daily_build installation"
    fi

    if (test -f "$HELPER_FUN_LOC/compile_code.beam") ; then
       echo "* Helper Fun                       seems OK..."
    else
       echo "* missing compile_code.erl         -- check your daily_build installation"
    fi

    if (test -f "$PRE_TSUNAMI_SCRIPT") ; then
       echo "* Pre-tsunami load script          seems OK..."
    else
       echo "* missing pre-tsunami load script  -- check your daily_build installation"
    fi

    if (test -f "$TSUNAMI_XML_FILE") ; then
       echo "* Tsunami xml file                 seems OK..."
    else
       echo "* Tsunami xml file                 -- check your daily_build installation"
    fi

    if (test -f "$POST_TSUNAMI_SCRIPT") ; then
       echo "* Post-tsunami load script         seems OK..."
    else
       echo "* missing post-tsunami load script -- check your daily_build installation"
    fi

    if (test -f "$SCP_PRIV_KEY") ; then
       echo "* scp private key                  seems OK..."
    else
       echo "* missing scp private key          -- check your daily_build installation"
    fi

    if (test -f "$SENDMAIL_LOC/sendmail") ; then
       echo "* Sendmail                         seems OK..."
    else
       echo "* missing sendmail                 -- check your sendmail installation"
    fi

    echo ""
    echo "Check if the following are installed:"
    echo ""
    echo "cvs:     "
    echo `cvs --version`
    echo ""
    echo "ssh:     " 
    echo `ssh -V`
    # ssh throws a random line feed in -V for
    # "OpenSSH_3.9p1, OpenSSL 0.9.7d 17 Mar 2004" who knows why?
    echo "Gnuplot: " 
    echo `gnuplot -V`
    echo ""
}

function dump_config
{
    write_config
    echo ""
    echo "Now test if the various things are where we think they are..."
    echo ""
    check_components
}
function write_config
{
    echo ""
    echo "Dumping the configuration of the erlang_automated_build script"
    echo ""
    echo "* ERLANG_LOC          is $ERLANG_LOC"
    echo "* LOCAL_CVSROOT       is $LOCAL_CVSROOT"
    echo "* TEST_SERVER_LOC     is $TEST_SERVER_LOC"
    echo "* DIALYZER_LOC        is $DIALYZER_LOC"
    echo "* TSUNAMI_LOC         is $TSUNAMI_LOC"
    echo "* TSU_ANALYSE_LOC     is $TSU_ANALYSE_LOC"
    echo "* PREPARE_BLD_SCRIPT  is $PREPARE_BLD_SCRIPT"
    echo "* HELPER_FUN_LOC      is $HELPER_FUN_LOC"
    echo "* PRE_TSUNAMI_SCRIPT  is $PRE_TSUNAMI_SCRIPT"
    echo "* WHERES_THE_CANARY   is $WHERES_THE_CANARY"
    echo "* TSUNAMI_XML_FILE    is $TSUNAMI_XML_FILE"
    echo "* POST_TSUNAMI_SCRIPT is $POST_TSUNAMI_SCRIPT"
    echo "* SCP_LOC             is $SCP_LOC"
    echo "* SCP_PRIV_KEY        is $SCP_PRIV_KEY"
    echo "* SENDMAIL_LOC        is $SENDMAIL_LOC"
    echo "* REPORT_LOC          is $REPORT_LOC"
    echo "* SYSTEM_NAME         is $SYSTEM_NAME"
    echo "* NOTIFY_ON_FAIL      is $NOTIFY_ON_FAIL"
    echo "                        (to get no failure e-mails set to empty string)"
    echo "* NOTIFY_ON_SUCCESS   is $NOTIFY_ON_SUCCESS"
    echo "                        (to get no failure e-mails set to empty string)"
    echo "* DAILY_BUILD_FROM    is $DAILY_BUILD_FROM"
    echo "* CVS_COMMAND_LINE    is $CVS_COMMAND_LINE"
    echo "* CVS_RSH_SUBSTITUTE  is $CVS_RSH_SUBSTITUTE"
    echo "* TEST_SUITE_SPEC     is $TEST_SUITE_SPEC"
}

function send_emails
{
if [ $DEBUG = "true" ]; then
    echo "e-mail from    : $DAILY_BUILD_FROM"
    echo "e-mail to      : $SENDMAIL_TO"
    echo "e-mail subject : $SENDMAIL_SUBJECT"
    echo "e-mail body    : $SENDMAIL_BODY"
fi
(
echo "From: $DAILY_BUILD_FROM"
echo "To: $SENDMAIL_TO"
echo "Subject: $SENDMAIL_SUBJECT"
echo ""
echo -e "$SENDMAIL_BODY"
echo ""
echo ""; echo ""
echo "Submitted on $(date)"
) | $SENDMAIL_LOC/sendmail -t
}

################################################################################
#                                                                              #
# STEP 4 Handle the options                                                    #
#                                                                              #
################################################################################

while getopts "chvdVE:L:H:P:Q:W:R:S:r:X:Y:Z:z:ABCFGIJK" OPT; do
    case $OPT in
	c )     DUMPCONFIG="true";;
	h )	SCRIPTHELP="true";;
	v )	VERBOSE="true";;
        d )     DEBUG="true";;
        V )     SHOW_VERSION="true";;
        E )     ERLANG_LOC=$OPTARG;;
        L )     LOCAL_CVSROOT=$OPTARG;;
	H )     HELPER_FUN_LOC=$OPTARG;;
	P )     PREPARE_BLD_SCRIPT=$OPTARG;;
	Q )     PRE_TSUNAMI_SCRIPT=$OPTARG;;
	W )     WHERES_THE_CANARY=$OPTARG;;
	R )     TSUNAMI_XML_FILE=$OPTARG;;
	S )     POST_TSUNAMI_SCRIPT=$OPTARG;;
	r )     REPORT_LOC=$OPTARG;;
	X )     NOTIFY_ON_SUCCESS=$OPTARG;;
	Y )     NOTIFY_ON_FAIL=$OPTARG;;
	Z )     SCP_LOC=$OPTARG;;
	z )     SCP_PRIV_KEY=$OPTARG;;
	A )     SKIP_CVS="true";;
        B )     SKIP_COMPILE="true";;
        C )     SKIP_TEST_SUITE="true";;
        F )     SKIP_DIALYZER="true";;
        G )     SKIP_TSUNAMI="true";;
        I )     SKIP_GNUPLOT="true";;
	J )     SKIP_SECURE_COPY="true";;
	K )     SKIP_PRECVS_BACKUP="true";;
	\? )    INVALID="true";;
    esac
done

if [ $VERBOSE = "true" ]; then
    print_GPL
fi

if [ $DEBUG = "true" ]; then
    echo "* TIMESTAMP           is $TIMESTAMP"
    echo "* DUMPCONFIG          is $DUMPCONFIG"
    echo "* SCRIPTHELP          is $SCRIPTHELP"
    echo "* VERBOSE             is $VERBOSE"
    echo "* DEBUG               is $DEBUG"
    echo "* SHOW_VERSION        is $SHOW_VERSION"
    echo "* PREPARE_TO_EXIT     is $PREPARE_TO_EXIT"
    echo "* ERLANG_LOC          is $ERLANG_LOC"
    echo "* LOCAL_CVSROOT       is $LOCAL_CVSROOT"
    echo "* TEST_SERVER_LOC     is $TEST_SERVER_LOC"
    echo "* DIALYZER_LOC        is $DIALYZER_LOC"
    echo "* TSUNAMI_LOC         is $TSUNAMI_LOC"
    echo "* TSU_ANALYSE_LOC     is $TSU_ANALYSE_LOC"
    echo "* PREPARE_BLD_SCRIPT  is $PREPARE_BLD_SCRIPT"
    echo "* HELPER_FUN_LOC      is $HELPER_FUN_LOC"
    echo "* PRE_TSUNAMI_SCRIPT  is $PRE_TSUNAMI_SCRIPT"
    echo "* WHERES_THE_CANARY   is $WHERES_THE_CANARY"
    echo "* TSUNAMI_XML_FILE    is $TSUNAMI_XML_FILE"
    echo "* POST_TSUNAMI_SCRIPT is $POST_TSUNAMI_SCRIPT"
    echo "* SCP_LOC             is $SCP_LOC"
    echo "* SCP_PRIV_KEY        is $SCP_PRIV_KEY"
    echo "* SENDMAIL_LOC        is $SENDMAIL_LOC"
    echo "* REPORT_LOC          is $REPORT_LOC"
    echo "* INVALID             is $INVALID"
    echo "* NOTIFY_ON_FAIL      is $NOTIFY_ON_FAIL"
    echo "* NOTIFY_ON_SUCCESS   is $NOTIFY_ON_SUCCESS"
    echo "* DAILY_BUILD_FROM    is $DAILY_BUILD_FROM"
    echo "* CVS_COMMAND_LINE    is $CVS_COMMAND_LINE"
    echo "* CVS_RSH_SUBSTITUTE  is $CVS_RSH_SUBSTITUTE"
    echo "* TEST_SUITE_SPEC     is $TEST_SUITE_SPEC"
    echo "* SKIP_CVS            is $SKIP_CVS"
    echo "* SKIP_COMPILE        is $SKIP_COMPILE"
    echo "* SKIP_TEST_SUITE     is $SKIP_TEST_SUITE"
    echo "* SKIP_DIALYZER       is $SKIP_DIALYZER"
    echo "* SKIP_TSUNAMI        is $SKIP_TSUNAMI"
    echo "* SKIP_GNUPLOT        is $SKIP_GNUPLOT"
    echo "* SKIP_SECURE_COPY    is $SKIP_SECURE_COPY"
    echo "* SKIP_PRECVS_BACKUP  is $SKIP_PRECVS_BACKUP"
fi

if [ $DUMPCONFIG = "true" ]; then
    dump_config
    PREPARE_TO_EXIT="true"
fi

if [ $SCRIPTHELP = "true" ]; then
    displayscripthelp
    PREPARE_TO_EXIT="true"
fi

if [ $SHOW_VERSION = "true" ]; then
    echo "The version of erlang_automated_build is $VERSION"
    echo "erlang_automated_build is released under the GPL V2.0"
    PREPARE_TO_EXIT="true"
fi

if [ $INVALID = "true" ]; then
	echo "Invalid arguments in erlang_automated_build"
	displayscripthelp
	echo "Invalid arguments in erlang_automated_build"
	exit 2
fi

if [ $PREPARE_TO_EXIT = "true" ]; then
    exit 1
fi

# Run dump_config and check components anyway if verbose is on
# Provided we haven't dumped them already
if [ $VERBOSE = "true" ] || [ $DEBUG = "true" ]; then
  if [ $DUMPCONFIG = "false" ]; then
    dump_config
  fi
fi
    #fix_me

################################################################################
#                                                                              #
# STEP 5 Do some housekeeping stuff just 'cos you need to do it                #
#                                                                              #
################################################################################

# Now create the reports directory - this is really only going to be done once
if ! [ -d $REPORT_LOC ]; then
    mkdir $REPORT_LOC
    # Bit minging but now test for success
    if ! [ -d $REPORT_LOC ]; then
        echo "Cannot create the reports directory $REPORTS_LOC!"
        exit 1
    fi
fi
REPORTS_DIR=$REPORT_LOC/$TIMESTAMP
mkdir $REPORTS_DIR
if ! [ -d $REPORTS_DIR ]; then
    echo "Cannot create the reports directory $REPORTS_DIR"
    exit 1
fi

################################################################################
#                                                                              #
# STEP 6 Delete the current build and cvs down the current build               #
#                                                                              #
################################################################################
if [ $SKIP_CVS = "false" ]; then

# Firstly check if there is an existing installation
# If there is back it up...
    if [ $SKIP_PRECVS_BACKUP = "false" ]; then
	if (test -d "$LOCAL_CVSROOT") ; then
	    CVS_BACKUP_LOC="$LOCAL_CVSROOT-backup-$TIMESTAMP"
	    if [ $VERBOSE = "true" ] || [ $DEBUG = "true" ]; then
		echo "CVS_BACKUP_LOC is $CVS_BACKUP_LOC"
	    fi
	    RETURN=`mv $LOCAL_CVSROOT $CVS_BACKUP_LOC`
            # Now test if the backup has been created
	    if (test -d "$CVS_BACKUP_LOC") ; then 
		if [ $VERBOSE = "true" ] || [ $DEBUG = "true" ]; then
		    echo "CVS_BACKUP_LOC is $CVS_BACKUP_LOC"
		fi
	    else
		echo "CVS backup has not been created, script terminating!"
		HAS_ANY_STEP_FAILED="true"
		SENDMAIL_TO="$NOTIFY_ON_FAIL"
		SENDMAIL_SUBJECT="$SYSTEM_NAME CVS backup has not happened"
		SENDMAIL_BODY="The $SYSTEM_NAME automated build has failed"
		send_emails
		exit 2
	    fi
      
	    if [ $VERBOSE = "true" ] || [ $DEBUG = "true" ]; then
		echo ""
		echo "Current CVS tree backed up to $LOCAL_CVSROOT_backup_$TIMESTAMP"
	    fi
	else
	    echo "No local CVSROOT at $LOCAL_CVSROOT"
	fi
    else
	# If the CVS root exists just delete it
	if (test -d "$LOCAL_CVSROOT") ; then
	    if (test -w "$LOCAL_CVSROOT"); then
		rm -r $LOCAL_CVSROOT
	    else
		echo ""
		echo "CVSROOT exists but can't be deleted, script terminating!"
		HAS_ANY_STEP_FAILED="true"
		SENDMAIL_TO="$NOTIFY_ON_FAIL"
		SENDMAIL_SUBJECT="$SYSTEM_NAME CVS has not been deleted"
		SENDMAIL_BODY="The $SYSTEM_NAME automated build has failed"
		send_emails
		exit 2
	    fi
       fi
    fi

    # We are going to substitute the value of $CVS_RSH here
    # cos we want CVS to log in with a single user name and public key
    OLD_CVS_RSH=$CVS_RSH
    CVS_RSH=$CVS_RSH_SUBSTITUTE
    export CVS_RSH
    # Now wheech down the cvs you want
    cd $DOWNLOAD_LOC
    RETURN=`$CVS_COMMAND_LINE`
    #echo "Return is $RETURN"
    # Now reset the old value of CVS_RSH
    CVS_RSH=$OLD_CVS_RSH
    export CVS_RSH
fi

################################################################################
#                                                                              #
# STEP 7 Runs a compile script                                                 #
#                                                                              #
################################################################################
if [ $SKIP_COMPILE = "false" ]; then
    RETURN=`$PREPARE_BLD_SCRIPT`
    #echo "The prepare build script has returned $RETURN"
    cd $HELPER_FUN_LOC
    # Need to fully specify the path if you want to run it in cron (which we do)
    RETURN=`$ERLANG_LOC/bin/erl -noshell -s compile_code start -s init stop`
    MUNGED_RETURN=`echo $RETURN | grep -o "Termination Status: ok"`
    NO_OF_SUCCESSFUL_COMPILES=`echo -e $RETURN | grep -o "OK: " | wc -l`
    NO_OF_UNSUCCESSFUL_COMPILES=`echo -e $RETURN | grep -o "Compile failure: " | wc -l`
   if [ $DEBUG = "true" ] || [ $VERBOSE = "true" ]; then
        echo "No of successful compiles is $NO_OF_SUCCESSFUL_COMPILES"
        echo "No of unsuccessful compiles is $NO_OF_UNSUCCESSFUL_COMPILES"
   fi    
    if [ "$MUNGED_RETURN" != "Termination Status: ok" ]; then
	echo "Compilation failure!,"
	if [ -n "$NOTIFY_ON_FAIL" ]; then
	    SENDMAIL_TO="$NOTIFY_ON_FAIL"
	    SENDMAIL_SUBJECT="$SYSTEM_NAME Compile Failure"
	    SENDMAIL_BODY="The $SYSTEM_NAME automated build has failed\n\n"
	    SENDMAIL_BODY="$SENDMAIL_BODY\nNo of successfully compiled modules   is $NO_OF_SUCCESSFUL_COMPILES\n"
	    SENDMAIL_BODY="$SENDMAIL_BODY\nNo of unsuccessfully compiled modules is $NO_OF_UNSUCCESSFUL_COMPILES\n"
	    send_emails
	fi
	PREPARE_TO_EXIT="true"
    else
	echo "Compilation Success!"
	if [ -n "$NOTIFY_ON_SUCCESS" ]; then
	    SENDMAIL_SUBJECT="$SYSTEM_NAME Compile Success"
	    SENDMAIL_BODY="The $SYSTEM_NAME automated compile has succeded\n\n"
	    SENDMAIL_BODY="$SENDMAIL_BODY\n    No of successfully compiled modules is   $NO_OF_SUCCESSFUL_COMPILES\n\n"
	fi
    fi
fi

################################################################################
#                                                                              #
# STEP 8 Copies the compilee results to the base file for the graphing tool    #
#                                                                              #
################################################################################
if [ $SKIP_COMPILE = "false" ]; then
    if [ $SKIP_GNUPLOT = "false" ]; then
    # Check if the results files exists
	if ! [ -f "$RL_TOTAL_COMPS" ]; then
	    echo -e "#Total No Of Compiled Files" > $RL_TOTAL_COMPS
	    echo "New copy of $RL_TOTAL_COMPS created"
	else
	    if [ $VERBOSE = "true" ] || [ $DEBUG = "true" ]; then
		echo "File $RL_TOTAL_COMPS exists!"
	    fi
	fi
	if ! [ -f "$RL_TOTAL_COMP_FAILS" ]; then
	    echo -e "#Total No Of Compiled File Failues" > $RL_TOTAL_COMP_FAILS
	    echo "New copy of $RL_TOTAL_COMP_FAILS created"
	else
	    if [ $VERBOSE = "true" ] || [ $DEBUG = "true" ]; then
		echo "File $RL_TOTAL_COMP_FAILS exists!"
	    fi
	fi
	if ! [ -f "$RL_TOTAL_COMP_SUCCS" ]; then
	    echo -e "#Total No Of Compiled File Successess" > $RL_TOTAL_COMP_SUCCS
	    echo "New copy of $RL_TOTAL_COMP_SUCCS created"
	else
	    if [ $VERBOSE = "true" ] || [ $DEBUG = "true" ]; then
		echo "File $RL_TOTAL_COMP_SUCCS exists!"
	    fi
	fi

	TOTAL_COMPILES=`echo $NO_OF_SUCCESSFUL_COMPILES+$NO_OF_UNSUCCESSFUL_COMPILES | bc`
	echo "$REPORT_TIME_FORMAT $NO_OF_SUCCESSFUL_COMPILES" >> $RL_TOTAL_COMP_SUCCS
	echo "$REPORT_TIME_FORMAT $NO_OF_UNSUCCESSFUL_COMPILES" >> $RL_TOTAL_COMP_FAILS
	echo "$REPORT_TIME_FORMAT $TOTAL_COMPILES" >> $RL_TOTAL_COMPS
	gnuplot << EOF
set timefmt "%Y/%m/%d/%H/%M" 
set format x "%m/%Y"
set xdtics 
set xdata time
set xlabel "date"
set ylabel "no of erlang source files"
set key below
set title "Compile Statistics for $SYSTEM_NAME on $STARTTIME"
set terminal png transparent small 
set size 0.75,0.75
set output "$REPORTS_DIR/compile.png"
plot "$REPORTS_LOC/$RL_TOTAL_COMP_SUCCS" using 1:2 title "Successful Compiles" with linespoints, "$REPORTS_LOC/$RL_TOTAL_COMP_FAILS" using 1:2 title "Failed Compiles" with linespoints, "$REPORTS_LOC/$RL_TOTAL_COMPS" using 1:2 title "Total Compiles" with linespoints
EOF
    fi
fi
# Exit here if the compile has failed
if [ $PREPARE_TO_EXIT = "true" ]; then
    exit 2
fi
################################################################################
#                                                                              #
# STEP 9  Run the test suites                                                  #
#                                                                              #
################################################################################
if [ $SKIP_TEST_SUITE = "false" ]; then
    cd "$TEST_SERVER_LOC"
    RETURN=`$ERLANG_LOC/bin/erl -pa ../$TEST_SUITE_SPEC_test -s ts_stub run $TEST_SUITE_SPEC -s init stop -noshell \
    | sed '/Starting test, .*/d' \
    | sed '/.* test case .*/d' \
    | sed '/Updating.*/d'`
    echo "$RETURN"
    # Lets try and strip out the results
    # Firstly the test case success and failure
    NO_OF_TEST_CASES=`echo $RETURN \
                     | grep -o "[0-9]* test cases" \
                     | grep -o "[0-9]*"`
    NO_OF_OK_TCS=`echo $RETURN \
                     | grep -o "[0-9]* ok" \
                     | grep -o "[0-9]*"`
    NO_OF_FAILED_TCS=`echo $RETURN \
                     | grep -o "[0-9]* failed" \
                     | grep -o "[0-9]*"`
    NO_OF_SKIPPED_TCS=`echo $RETURN \
                     | grep -o "[0-9]* skipped" \
                     | grep -o "[0-9]*"`
    # Skipped can be blank, if it is make it zero
    # This is a null test 
    if ! [ $NO_OF_SKIPPED_TCS ]; then
            NO_OF_SKIPPED_TCS="0"
    fi
    # Secondly the code coverage stuff
    echo "Test Server Location is $TEST_SERVER_LOC"

    TEST_SUITE_OUTPUT=`cat $TEST_SERVER_LOC/index.html | grep $TEST_SUITE_SPEC | grep -o "\".*\"" | sed 's/\"//g' | sed 's/\/suite.log.html//'`
    echo "Test suite output is $TEST_SUITE_OUTPUT"
    COVERAGE_OUTPUT="$TEST_SUITE_OUTPUT/cover.html"
    COVERAGE_STATS=`cat $TEST_SERVER_LOC/$COVERAGE_OUTPUT | grep "th align=left>Total"`
    PERC_COVERED=`echo $COVERAGE_STATS \
                  | grep -o "[0-9[:blank:]]*%" \
	          | grep -o "[0-9]*"`
    if ! [ $PERC_COVERED ]; then
            PERC_COVERED="0"
    fi
    LINES_COVERED=`echo $COVERAGE_STATS \
		   | grep -o "[0-9]*</th><th align=right" \
                   | grep -o "[0-9]*"`
    if ! [ $LINES_COVERED ]; then
	    LINES_COVERED="0"
    fi
    LINES_NOT_COVERED=`echo $COVERAGE_STATS \
                       | grep -o "[0-9]*</th></tr>" \
	               | grep -o "[0-9]*"`
    if ! [ $LINES_NOT_COVERED ]; then
            LINES_NOT_COVERED="0"
    fi
    TOTAL_LINES=`echo $LINES_COVERED+$LINES_NOT_COVERED | bc`
    if [ $DEBUG = "true" ] || [ $VERBOSE = "true" ]; then
	    echo "Number of test cases         is $NO_OF_TEST_CASES"
	    echo "Number of ok test cases      is $NO_OF_OK_TCS"
	    echo "Number of failed test cases  is $NO_OF_FAILED_TCS"
	    echo "Number of skipped test cases is $NO_OF_SKIPPED_TCS"
	    echo "Percentage code covered      is $PERC_COVERED"
	    echo "No of lines covered          is $LINES_COVERED"
	    echo "No of lines NOT covered      is $LINES_NOT_COVERED"
	    echo "Total no of lines            is $TOTAL_LINES"
    fi
    if [ $NO_OF_FAILED_TCS = "0" ]; then
	SENDMAIL_SUBJECT="$SENDMAIL_SUBJECT Test Suite Ran"
    else
        HAS_ANY_STEP_FAILED="true"
	SENDMAIL_SUBJECT="$SENDMAIL_SUBJECT Test Suite Failed"
    fi
    SENDMAIL_BODY="$SENDMAIL_BODY\nThe $SYSTEM_NAME Test Suite has run\n\n"
    SENDMAIL_BODY="$SENDMAIL_BODY\n    Test Case Statistics"
    SENDMAIL_BODY="$SENDMAIL_BODY\n        Number of test cases         is $NO_OF_TEST_CASES\n"
    SENDMAIL_BODY="$SENDMAIL_BODY\n        Number of ok test cases      is $NO_OF_OK_TCS\n"
    SENDMAIL_BODY="$SENDMAIL_BODY\n        Number of failed test cases  is $NO_OF_FAILED_TCS\n"
    SENDMAIL_BODY="$SENDMAIL_BODY\n        Number of skipped test cases is $NO_OF_SKIPPED_TCS\n"
    SENDMAIL_BODY="$SENDMAIL_BODY\n    Code Coverage Statistics\n\n"
    SENDMAIL_BODY="$SENDMAIL_BODY\n        Percentage code covered      is $PERC_COVERED\n"
    SENDMAIL_BODY="$SENDMAIL_BODY\n        No of lines covered          is $LINES_COVERED\n"
    SENDMAIL_BODY="$SENDMAIL_BODY\n        No of lines NOT covered      is $LINES_NOT_COVERED\n"
    SENDMAIL_BODY="$SENDMAIL_BODY\n        Total no of lines            is $TOTAL_LINES\n"
fi

################################################################################
#                                                                              #
# STEP 10 Copies the test suite results to the base file for the graphing tool #
#                                                                              #
################################################################################

if [ $SKIP_TEST_SUITE = "false" ]; then
    if [ $SKIP_GNUPLOT = "false" ]; then

# Check if the results files exists
	if ! [ -f $RL_TOTAL_TEST_CASES ]; then
	    echo -e "#Total No Of Test Cases" > $RL_TOTAL_TEST_CASES
	    echo "New copy of $RL_TOTAL_TEST_CASES created"
	else
	    if [ $VERBOSE = "true" ] || [ $DEBUG = "true" ]; then
		echo "File $RL_TOTAL_TEST_CASES exists!"
	    fi
	fi
	if ! [ -f $RL_TOTAL_OK_TESTS ]; then
	    echo -e "#Total No Of Successful Test Cases" > $RL_TOTAL_OK_TESTS
	    echo "New copy of $RL_TOTAL_OK_TESTS created"
	else
	    if [ $VERBOSE = "true" ] || [ $DEBUG = "true" ]; then
		echo "File $RL_TOTAL_OK_TESTS exists!"
	    fi
	fi
	if ! [ -f $RL_TOTAL_FAILED_TESTS ]; then
	    echo -e "#Total No Of Failed Test Cases" > $RL_TOTAL_FAILED_TESTS
	    echo "New copy of $RL_TOTAL_FAILED_TESTS created"
	else
	    if [ $VERBOSE = "true" ] || [ $DEBUG = "true" ]; then
		echo "File $RL_TOTAL_FAILED_TESTS exists!"
	    fi
	fi
	if ! [ -f $RL_TOTAL_SKIPPED_TESTS ]; then
	    echo -e "#Total No Of Skipped Test Cases" > $RL_TOTAL_SKIPPED_TESTS
	    echo "New copy of $RL_TOTAL_SKIPPED_TESTS created"
	else
	    if [ $VERBOSE = "true" ] || [ $DEBUG = "true" ]; then
		echo "File $RL_TOTAL_SKIPPED_TESTS exists!"
	    fi
	fi
	if ! [ -f $RL_TEST_PERC_COV ]; then
	    echo -e "#Percentage Lines Covered In Test Cases" > $RL_TEST_PERC_COV
	    echo "New copy of $RL_TEST_PERC_COV created"
	else
	    if [ $VERBOSE = "true" ] || [ $DEBUG = "true" ]; then
		echo "File $RL_TEST_PERC_COV exists!"
	    fi
	fi
	if ! [ -f $RL_LINES_COV ]; then
	    echo -e "#Total No Of Lines Covered In Test Cases" > $RL_LINES_COV
	    echo "New copy of $RL_LINES_COV created"
	else
	    if [ $VERBOSE = "true" ] || [ $DEBUG = "true" ]; then
		echo "File $RL_LINES_COV exists!"
	    fi
	fi
	if ! [ -f $RL_LINES_NOT_COV ]; then
	    echo -e "#Total No Of Lines Not Covered In Test Cases" > $RL_LINES_NOT_COV
	    echo "New copy of $RL_LINES_NOT_COV created"
	else
	    if [ $VERBOSE = "true" ] || [ $DEBUG = "true" ]; then
		echo "File $RL_LINES_NOT_COV exists!"
	    fi
	fi
	if ! [ -f $RL_TOTAL_LINES ]; then
	    echo -e "#Total No Of Lines Of Code" > $RL_TOTAL_LINES
	    echo "New copy of $RL_TOTAL_LINES created"
	else
	    if [ $VERBOSE = "true" ] || [ $DEBUG = "true" ]; then
		echo "File $RL_TOTAL_LINES exists!"
	    fi
	fi

	echo "$REPORT_TIME_FORMAT $NO_OF_TEST_CASES"  >> $RL_TOTAL_TEST_CASES
	echo "$REPORT_TIME_FORMAT $NO_OF_OK_TCS"      >> $RL_TOTAL_OK_TESTS
	echo "$REPORT_TIME_FORMAT $NO_OF_FAILED_TCS"  >> $RL_TOTAL_FAILED_TESTS
	echo "$REPORT_TIME_FORMAT $NO_OF_SKIPPED_TCS" >> $RL_TOTAL_SKIPPED_TESTS
	echo "$REPORT_TIME_FORMAT $PERC_COVERED"      >> $RL_TEST_PERC_COV
	echo "$REPORT_TIME_FORMAT $LINES_COVERED"     >> $RL_LINES_COV
	echo "$REPORT_TIME_FORMAT $LINES_NOT_COVERED" >> $RL_LINES_NOT_COV
	echo "$REPORT_TIME_FORMAT $TOTAL_LINES"       >> $RL_TOTAL_LINES
	# We are going to plot 3 graphs:
        # * test cases
        # * lines covered
        # * percentage lines covered
	gnuplot << EOF
set timefmt "%Y/%m/%d/%H/%M" 
set format x "%m/%Y"
set xdtics
set xdata time
set xlabel "date"
set ylabel "no of test cases"
set key below
set title "Test Case Statistics for $SYSTEM_NAME on $STARTTIME"
set terminal png transparent small 
set size 0.75,0.75
set output "$REPORTS_DIR/test_cases.png"
plot "$REPORTS_LOC/$RL_TOTAL_TEST_CASES" using 1:2 title "Total Test Cases" with linespoints, "$REPORTS_LOC/$RL_TOTAL_OK_TESTS" using 1:2 title "Total OK Tests" with linespoints, "$REPORTS_LOC/$RL_TOTAL_FAILED_TESTS" using 1:2 title "Total Failed Tests" with linespoints, "$REPORTS_LOC/$RL_TOTAL_SKIPPED_TESTS" using 1:2 title "Total Skipped Tests" with linespoints
EOF
	gnuplot << EOF
set timefmt "%Y/%m/%d/%H/%M" 
set format x "%m/%Y"
set xdtics
set xdata time
set xlabel "date"
set ylabel "no of lines of code"
set key below
set title "Line Coverage Statistics for $SYSTEM_NAME on $STARTTIME"
set terminal png transparent small 
set size 0.75,0.75
set output "$REPORTS_DIR/line_coverage.png"
plot "$REPORTS_LOC/$RL_LINES_COV" using 1:2 title "Total Lines Covered" with linespoints, "$REPORTS_LOC/$RL_LINES_NOT_COV" using 1:2 title "Total Lines Not Covered" with linespoints, "$REPORTS_LOC/$RL_TOTAL_LINES" using 1:2 title "Total Lines" with linespoints
EOF
	gnuplot << EOF
set timefmt "%Y/%m/%d/%H/%M" 
set format x "%m/%Y"
set xdtics
set xdata time
set xlabel "date"
set ylabel "Percentage Lines Covered"
set key off
set yrange [0:100]
set title "% Line Coverage Statistics for $SYSTEM_NAME on $STARTTIME"
set terminal png transparent small 
set size 0.75,0.75
set output "$REPORTS_DIR/percentage_line_coverage.png"
plot "$REPORTS_LOC/$RL_TEST_PERC_COV" using 1:2 with linespoints
EOF
   fi
fi

################################################################################
#                                                                              #
# STEP 11 Run dialyser on the code                                             #
#                                                                              #
################################################################################
if [ $SKIP_DIALYZER = "false" ]; then
    cd "$DIALYZER_LOC"
    RETURN=`./dialyzer -r $LOCAL_CVSROOT -o $REPORT_LOC/$TIMESTAMP/dialyzer.output.txt`
    # We will report the number of lines in the output file (minus 1, of course)
    DIALYZER_ERRORS=`wc -l $REPORT_LOC/$TIMESTAMP/dialyzer.output.txt \
                     | grep -o "[0-9]*"`
    DIALYZER_ERRORS=$(($DIALYZER_ERRORS-$DIALYZER_OFFSET))
    echo "The number of dialyzer errors is $DIALYZER_ERRORS"
    if [ $DIALYZER_ERRORS = 0 ]; then
	SENDMAIL_SUBJECT="$SENDMAIL_SUBJECT Dialyzer Ran"
    else
        HAS_ANY_STEP_FAILED="true"
	SENDMAIL_SUBJECT="$SENDMAIL_SUBJECT Dialyzer Failed"
    fi
    SENDMAIL_BODY="$SENDMAIL_BODY\nDialyzer has run on $SYSTEM_NAME\n"
    SENDMAIL_BODY="$SENDMAIL_BODY    Dialyzer has produced $((DIALYZER_ERRORS)) Messages\n\n"
fi

################################################################################
#                                                                              #
# STEP 12 Output from the dialyzer is crunched and summarised for the          #
#         graphing tool                                                        #
#                                                                              #
################################################################################
if [ $SKIP_DIALYZER = "false" ]; then
    if [ $SKIP_GNUPLOT = "false" ]; then
        # Check if the results files exists
	if ! [ -f $RL_DIALYZER ]; then
	    echo -e "#Total No Of Compiled Files" > $RL_DIALYZER
	    echo "New copy of $RL_DIALYZER created"
	else
	    if [ $VERBOSE = "true" ] || [ $DEBUG = "true" ]; then
		echo "File $RL_DIALYZER exists!"
	    fi
	fi

	echo "$REPORT_TIME_FORMAT $(($DIALYZER_ERRORS))" >> $RL_DIALYZER
	gnuplot << EOF
set timefmt "%Y/%m/%d/%H/%M" 
set format x "%m/%Y"
set xdtics
set xdata time
set xlabel "date"
set ylabel "No of Dialyzer warnings"
set key off
set title "Dialyser Statistics for $SYSTEM_NAME on $STARTTIME"
set terminal png transparent small 
set size 0.75,0.75
set output "$REPORTS_DIR/dialyzer.png"
plot "$REPORTS_LOC/$RL_DIALYZER" using 1:2 with linespoints
EOF
    fi
fi

################################################################################
#                                                                              #
# STEP 13 Run the tsunami load test                                            #
#                                                                              #
################################################################################
if [ $SKIP_TSUNAMI = "false" ]; then
    # First up we hatch a canary
    # When the pre-tsunami script deletes it we will continue
    echo "tweet!" > $WHERES_THE_CANARY/erlang_automated_build.canary
    # Run the pre-tsunami script in a background sub-shell
    ($PRE_TSUNAMI_SCRIPT)&
    NO=1
    while [ -f "$WHERES_THE_CANARY/erlang_automated_build.canary" ]; do
	echo "Data still loading - $NO"
	NO=$((NO+1))
	sleep $SLEEP_TIME
    done

    NO=0
    echo "Running tsunami..."
    cd "/opt/erlang_apps/tsunami/ebin"
    IDX_TSUNAMI_STATUS=""
    idx-tsunami -f $TSUNAMI_XML_FILE start
    # A little more sleep time is a good thing here...
    sleep $SLEEP_TIME
    while [ "$IDX_TSUNAMI_STATUS" != "IDX-Tsunami is not started [ERROR]" ]; do
	IDX_TSUNAMI_STATUS=`idx-tsunami status`
	echo "Load Status is $IDX_TSUNAMI_STATUS: $NO"
	NO=$((NO+1))
	sleep $SLEEP_TIME
    done
    USER=`whoami`
    TSUNAMI_LOG_LOC="/home/$USER/.idx-tsunami/log"
    cd $TSUNAMI_LOG_LOC
    # Tsunami writes to a time/dated log file - we will have do do some
    # finangaling to get which one... @YMMM eek!
    TSU_DATETIME=`ls -c -d -1 -r  2* | tail -n 1`
    cd "$TSUNAMI_LOG_LOC/$TSU_DATETIME"
    RETURN=`$TSU_ANALYSE_LOC/analyse_msg.pl --stats idx-tsunami.log  --html --extra  --plot`
    RETURN=`$POST_TSUNAMI_SCRIPT`
    SENDMAIL_SUBJECT="$SENDMAIL_SUBJECT idx-Tsunami has run"
    SENDMAIL_BODY="$SENDMAIL_BODY\nidx-Tsunami has run on $SYSTEM_NAME\n"
    SENDMAIL_BODY="$SENDMAIL_BODY\nIt used the load file $TSUNAMI_XML_FILE\n"

fi
################################################################################
#                                                                              #
# STEP 14 Copy the data to the remote location                                 #
#                                                                              #
################################################################################
if [ $SKIP_SECURE_COPY = "false" ]; then
    if [ $SKIP_GNUPLOT = "false" ]; then
	if [ $SKIP_COMPILE = "false" ]; then
	    echo "Copying up compile graph"
	    SCP_COMMAND="scp -i $SCP_PRIV_KEY $REPORTS_DIR/compile.png $SCP_LOC"
	    `$SCP_COMMAND`
	fi
	if [ $SKIP_TEST_SUITE = "false" ]; then
	    echo "Copying up test case graph"
	    SCP_COMMAND="scp -i $SCP_PRIV_KEY $REPORTS_DIR/test_cases.png $SCP_LOC"
	    `$SCP_COMMAND`
	    echo "Copying up line coverage graph"
	    SCP_COMMAND="scp -i $SCP_PRIV_KEY $REPORTS_DIR/line_coverage.png $SCP_LOC"
	    `$SCP_COMMAND`
	    echo "Copying up percentage line coverage graph"
	    SCP_COMMAND="scp -i $SCP_PRIV_KEY $REPORTS_DIR/percentage_line_coverage.png $SCP_LOC"
	    `$SCP_COMMAND`
	fi
	if [ $SKIP_DIALYZER = "false" ]; then
	    echo "Copying up dialyzer graph"
	    SCP_COMMAND="scp -i $SCP_PRIV_KEY $REPORTS_DIR/dialyzer.png $SCP_LOC"
	    `$SCP_COMMAND`
	fi
	if [ $SKIP_TSUNAMI = "false" ]; then
	    echo "Copying up Tsunami html"
	    SCP_COMMAND="scp -i $SCP_PRIV_KEY -r $TSUNAMI_LOG_LOC/$TSU_DATETIME/*.html $SCP_LOC"
	    `$SCP_COMMAND`
	    echo "Copying up Tsunami graph images"
	    NEW_SCP_LOC=$SCP_LOC"images"
	    TSU_FILES=$TSUNAMI_LOG_LOC/$TSU_DATETIME"/images/*.png"
	    SCP_COMMAND="scp -i $SCP_PRIV_KEY $TSU_FILES $NEW_SCP_LOC"
	    `$SCP_COMMAND`
	fi
    fi
fi

################################################################################
#                                                                              #
# STEP 15 Send e-mails                                                         #
#                                                                              #
################################################################################

if [ $HAS_ANY_STEP_FAILED = "true" ]; then
    SENDMAIL_TO="$NOTIFY_ON_FAIL"
else
    SENDMAIL_TO="$NOTIFY_ON_SUCCESS"
fi
# This is a null test 
if [ $SENDMAIL_TO ]; then
    send_emails
fi
echo ""
echo "Erlang_automated_build finished"
echo "Started $STARTTIME"
echo "Finished `date`"
