ControlScripts/run_scenario.sh

   1 #!/bin/bash
   2 #
   3 # Copyright: George Milescu 2010 - george.milescu@gmail.com
   4 #
   5 # 2010, Razvan Deaconescu, razvan.deaconescu@cs.pub.ro:
   6 #    * update script to use new configuration file and run scripts
   7 #    * update script for newer configuration file (pre-run and
   8 #      post-run scripts)
   9 #
  10 # Bash script used to run a test scenario
  11 # The script
  12 #  * Reads the scenario setup file
  13 #  * Cleans-up the environment on the target machines
  14 #  * Runs the clients
  15 #  * Retrieves the logs
  16 #  * Cleans-up the environment on the target machines
  17 #  * Parses the log files
  18 #
  19 # Script arguments:
  20 #  * an action that can be setup, start, wait, clean, parse, run
  21 #  ** setup - setup the nodes on which the tests will be run
  22 #  ** start - start the clients on the remote nodes
  23 #  ** wait - wait for all the clients to complete their task
  24 #  ** clean - clean the nodes on whitch the tests were run
  25 #  ** parse - parse the resulted client log files
  26 #  ** run - run a complete campaign (setup, start, wait, clean, parse)
  27 #  * a file describing the scenario. The file must be located in SetupFiles
  28 #  * a campaign name (see run_campaign.sh for details). Example: campaign01.txt-2010.07.14-21.10.45
  29
  30 usage() {
  31         # Print help messages
  32         echo "usage: $0 action scenario-file campaign-name"
  33 }
  34
  35 if [ ! $# -eq 3 ]; then
  36         usage
  37         exit 1
  38 fi
  39
  40 # Read the global configuration file
  41 # Check if the global configuration file exists
  42 if [ ! -e globalconfig ]; then
  43         echo "Warning: The global config file globalconfig does not exist."
  44 else
  45         source globalconfig
  46 fi
  47
  48 # Read arguments
  49 ACTION=$1
  50 SCENARIO_FILE=$2
  51 CAMPAIGN_NAME=$3
  52
  53 SCENARIO_NAME=${SCENARIO_FILE%\.*}
  54 SCENARIO_RESULTS_FOLDER=$RESULTS_FOLDER_REL_PATH/$CAMPAIGN_NAME/$SCENARIO_NAME
  55 ERR_LOG=$SCENARIO_RESULTS_FOLDER/err.log
  56
  57 # Check if the scenario file exists
  58 if [ ! -e $CONFIG_FILES_REL_PATH/$SCENARIO_FILE ]; then
  59         echo "Error: The scenario file $CONFIG_FILES_REL_PATH/$SCENARIO_FILE does not exist."
  60         exit 1
  61 fi
  62
  63 # Check if the scenario file is a regular file
  64 if [ ! -f $CONFIG_FILES_REL_PATH/$SCENARIO_FILE ]; then
  65         echo "Error: The scenario file $CONFIG_FILES_REL_PATH/$SCENARIO_FILE is not a regular file."
  66         exit 1
  67 fi
  68
  69 # Check if the campain results folder exists
  70 if [ ! -d $RESULTS_FOLDER_REL_PATH/$CAMPAIGN_NAME ]; then
  71         echo "Error: The results folder $RESULTS_FOLDER_REL_PATH/$CAMPAIGN_NAME does not exist"
  72         exit 1
  73 fi
  74
  75
  76
  77 # Create a scenario folder inside the campaign results folder
  78 # Check if the scenario results folder already exists
  79 if [ -d $SCENARIO_RESULTS_FOLDER ]; then
  80         echo "Error: The folder $SCENARIO_RESULTS_FOLDER already exist."
  81         exit 1
  82 fi
  83
  84 echo -n " ** Creating scenario results folder $SCENARIO_RESULTS_FOLDER..."
  85 mkdir $SCENARIO_RESULTS_FOLDER
  86 echo "done"
  87
  88
  89
  90 # Parse the scenario file
  91 TMP_FILE=/tmp/proxy.scn.$RANDOM
  92 rm -f $TMP_FILE
  93 cat $CONFIG_FILES_REL_PATH/$SCENARIO_FILE | grep -v ^# | grep -v ^$ | tr -s '\t' > $TMP_FILE
  94 > $ERR_LOG
  95
  96
  97 # Setup the nodes
  98 scenario_setup() {
  99         echo "---------------------- scenario_setup_" >> $ERR_LOG
 100         while IFS=$'\t' read SERVER PORT USER REMOTE_PATH REMOTE_IF DL_BW DL_BURST UL_BW UL_BURST NO_CONNECTIONS PRE_RUN_SCRIPT POST_RUN_SCRIPT CLIENT_TYPE TORRENT_FILE PERIODS; do
 101                 echo " ** Setting-up $SERVER:$PORT"
 102
 103                 # Terminate $USER owned processes (ignore result)
 104                 ssh -n -p ${PORT} root@${SERVER} "pkill -u ${USER}; pkill -KILL -u ${USER}"
 105
 106                 # Clean-up the host by using rsync
 107                 if ! rsync -avP --del --exclude 'Results' --rsh='ssh -p'$PORT $TEST_INFRASTRUCTURE_REL_PATH ${USER}@${SERVER}:${REMOTE_PATH} >> $ERR_LOG 2>&1; then
 108                         echo "Error: rsync error on $SERVER"
 109                         exit 1
 110                 fi
 111
 112                 # Create node config file
 113                 if ! ssh -n -p ${PORT} ${USER}@${SERVER} "
 114 cat > $REMOTE_PATH/P2P-Testing-Infrastructure/ClientWorkingFolders/TmpLogs/node_config << END
 115 SERVER=\"${SERVER}\"
 116 PORT=\"${PORT}\"
 117 USER=\"${USER}\"
 118 REMOTE_PATH=\"${REMOTE_PATH}\"
 119 IFACE=\"${REMOTE_IF}\"
 120 DL_BW=\"${DL_BW}\"
 121 DL_BURST=\"${DL_BURST}\"
 122 UL_BW=\"${UL_BW}\"
 123 UL_BURST=\"${UL_BURST}\"
 124 NO_CONNECTIONS=\"${NO_CONNECTIONS}\"
 125 PRE_RUN_SCRIPT=\"${PRE_RUN_SCRIPT}\"
 126 POST_RUN_SCRIPT=\"${POST_RUN_SCRIPT}\"
 127 CLIENT_TYPE=\"${CLIENT_TYPE}\"
 128 TORRENT_FILE=\"${TORRENT_FILE}\"
 129 PERIODS=\"${PERIODS}\"
 130 END
 131 " >> ${ERR_LOG} 2>&1; then
 132                         echo "Error: cannot create scenario config file"
 133                         exit 1
 134                 fi
 135
 136                 # run pre-run script
 137                 # -n is used to redirect the ssh stdin from /dev/null, in order to keep the stdin in place for the read command
 138                 if ! ssh -n -p $PORT ${USER}@${SERVER} "cd $REMOTE_PATH/P2P-Testing-Infrastructure/ControlScripts; ./${PRE_RUN_SCRIPT}" >> $ERR_LOG 2>&1; then
 139                         echo "Error: tc error setting limitations on $SERVER"
 140                         exit 1
 141                 fi
 142         done < $TMP_FILE
 143         unset IFS
 144         echo "---------------------- _scenario_setup" >> $ERR_LOG
 145 }
 146
 147 # Schedule start/stop
 148 scenario_schedule()
 149 {
 150         echo "---------------------- scenario_schedule" >> $ERR_LOG
 151         while IFS=$'\t' read SERVER PORT USER REMOTE_PATH REMOTE_IF DL_BW DL_BURST UL_BW UL_BURST NO_CONNECTIONS PRE_RUN_SCRIPT POST_RUN_SCRIPT CLIENT_TYPE TORRENT_FILE PERIODS; do
 152                 echo " ** Scheduling $SERVER:$PORT"
 153                 if ! ssh -f -p $PORT ${USER}@${SERVER} "cd $REMOTE_PATH/P2P-Testing-Infrastructure/ControlScripts; ./schedule_client.sh &> $REMOTE_PATH/P2P-Testing-Infrastructure/ClientWorkingFolders/TmpLogs/\$(hostname).log" >> $ERR_LOG 2>&1; then
 154                         echo "Error: error running client_schedule.sh on $SERVER"
 155                         break
 156                 fi
 157         done < $TMP_FILE
 158         unset IFS
 159
 160         # wait for clients to start
 161         sleep 2
 162
 163         echo "---------------------- _scenario_schedule" >> $ERR_LOG
 164 }
 165
 166
 167 # Wait until all the clients are finished
 168 scenario_wait() {
 169         echo "---------------------- scenario_wait_" >> $ERR_LOG
 170         while IFS=$'\t' read SERVER PORT USER REMOTE_PATH REMOTE_IF DL_BW DL_BURST UL_BW UL_BURST NO_CONNECTIONS PRE_RUN_SCRIPT POST_RUN_SCRIPT CLIENT_TYPE TORRENT_FILE PERIODS; do
 171                 echo " ** Waiting for client on $SERVER:$PORT to finish"
 172
 173                 # Start the scripts on the remote clients
 174                 # -n is used to redirect the ssh stdin from /dev/null, in order to keep the stdin in place for the read command
 175                 # The ssh call is blocking
 176                 if ! ssh -n -p $PORT ${USER}@${SERVER} "cd $REMOTE_PATH/P2P-Testing-Infrastructure/ControlScripts; ./detect_complete_client.sh $REMOTE_PATH/P2P-Testing-Infrastructure/ClientWorkingFolders/TmpLogs/\$(hostname).log" >> $ERR_LOG 2>&1; then
 177                         echo "Error: error while waiting for client on $SERVER to finish"
 178                         break
 179                 fi
 180         done < $TMP_FILE
 181         unset IFS
 182         echo "---------------------- _scenario_wait" >> $ERR_LOG
 183 }
 184
 185
 186 # Clean the nodes and retrieve the log files
 187 scenario_clean() {
 188         echo "---------------------- scenario_clean_" >> $ERR_LOG
 189         while IFS=$'\t' read SERVER PORT USER REMOTE_PATH REMOTE_IF DL_BW DL_BURST UL_BW UL_BURST NO_CONNECTIONS PRE_RUN_SCRIPT POST_RUN_SCRIPT CLIENT_TYPE TORRENT_FILE PERIODS; do
 190                 echo " ** Cleaning-up $SERVER:$PORT"
 191
 192                 # Create node config file
 193                 if ! ssh -n -p ${PORT} ${USER}@${SERVER} "
 194 cat > $REMOTE_PATH/P2P-Testing-Infrastructure/ClientWorkingFolders/TmpLogs/node_config << END
 195 SERVER=\"${SERVER}\"
 196 PORT=\"${PORT}\"
 197 USER=\"${USER}\"
 198 REMOTE_PATH=\"${REMOTE_PATH}\"
 199 IFACE=\"${REMOTE_IF}\"
 200 DL_BW=\"${DL_BW}\"
 201 DL_BURST=\"${DL_BURST}\"
 202 UL_BW=\"${UL_BW}\"
 203 UL_BURST=\"${UL_BURST}\"
 204 NO_CONNECTIONS =\"${NO_CONNECTIONS}\"
 205 PRE_RUN_SCRIPT=\"${PRE_RUN_SCRIPT}\"
 206 POST_RUN_SCRIPT=\"${POST_RUN_SCRIPT}\"
 207 CLIENT_TYPE=\"${CLIENT_TYPE}\"
 208 TORRENT_FILE=\"${TORRENT_FILE}\"
 209 PERIODS=\"${PERIODS}\"
 210 END
 211 " >> ${ERR_LOG} 2>&1; then
 212                         echo "Error: cannot create scenario config file"
 213                         exit 1
 214                 fi
 215
 216                 # stopping the client on the server
 217                 # -n is used to redirect the ssh stdin from /dev/null, in order to keep the stdin in place for the read command
 218                 if ! ssh -n -p $PORT ${USER}@${SERVER} "cd $REMOTE_PATH/P2P-Testing-Infrastructure/ControlScripts; ./stop_client.sh" >> $ERR_LOG 2>&1; then
 219                         echo "Error: error stopping client on $SERVER"
 220                 fi
 221                 # wait for the remote client to finish completly
 222                 sleep 2
 223
 224                 # run post-run script
 225                 # -n is used to redirect the ssh stdin from /dev/null, in order to keep the stdin in place for the read command
 226                 if ! ssh -n -p $PORT ${USER}@${SERVER} "cd $REMOTE_PATH/P2P-Testing-Infrastructure/ControlScripts; ./${POST_RUN_SCRIPT}" >> $ERR_LOG 2>&1; then
 227                         echo "Error: tc error deleting limitations on $SERVER"
 228                 fi
 229
 230                 # Remove node config file
 231                 if ! ssh -n -p ${PORT} ${USER}@${SERVER} "rm $REMOTE_PATH/P2P-Testing-Infrastructure/ClientWorkingFolders/TmpLogs/node_config" >> ${ERR_LOG} 2>&1; then
 232                         echo "Error: cannot remove node_config file"
 233                 fi
 234
 235                 # Retrieve log files
 236                 scp -P $PORT ${USER}@${SERVER}:${REMOTE_PATH}/P2P-Testing-Infrastructure/ClientWorkingFolders/TmpLogs/* $SCENARIO_RESULTS_FOLDER >> $ERR_LOG 2>&1
 237
 238                 # Clean-up the host by using rsync
 239                 if ! rsync -avP --del --exclude 'Results' --rsh='ssh -p'$PORT $TEST_INFRASTRUCTURE_REL_PATH ${USER}@${SERVER}:${REMOTE_PATH} >> $ERR_LOG 2>&1; then
 240                         echo "Error: rsync error on $SERVER"
 241                 fi
 242         done < $TMP_FILE
 243         unset IFS
 244         echo "---------------------- _scenario_clean" >> $ERR_LOG
 245 }
 246
 247
 248 # Parse the log files
 249 scenario_parse() {
 250         echo "---------------------- scenario_parse_" >> $ERR_LOG
 251         while IFS=$'\t' read SERVER PORT USER REMOTE_PATH REMOTE_IF DL_BW DL_BURST UL_BW UL_BURST NO_CONNECTIONS PRE_RUN_SCRIPT POST_RUN_SCRIPT CLIENT_TYPE TORRENT_FILE PERIODS; do
 252                 echo -n " ** Parsing log files from $SERVER:$PORT"
 253
 254                 REMOTE_HOSTNAME=$(ssh -n -p $PORT ${USER}@${SERVER} "hostname")
 255
 256                 echo " with hostname $REMOTE_HOSTNAME"
 257
 258                 if ! ./parse_client_log.sh $CLIENT_TYPE $SCENARIO_RESULTS_FOLDER ${REMOTE_HOSTNAME}.log; then
 259                         echo "Error parsing file $SCENARIO_RESULTS_FOLDER/${REMOTE_HOSTNAME}.log"
 260                         exit 1
 261                 fi
 262         done < $TMP_FILE
 263         unset IFS
 264
 265         if ls -lR $SCENARIO_RESULTS_FOLDER | grep "\.updown" &> /dev/null; then
 266                 # There are .updown log files
 267                 echo " *** Parsing peer-conection up/down files"
 268                 if ! ./parse_up_down_log.sh $SCENARIO_RESULTS_FOLDER; then
 269                         echo "Error parsing updown files in $SCENARIO_RESULTS_FOLDER"
 270                         exit 1
 271                 fi
 272         fi
 273         if ls -lR $SCENARIO_RESULTS_FOLDER | grep "\.mon" &> /dev/null; then
 274                 # There are .mon log files
 275                 echo " *** Parsing peer resource monitoring files"
 276                 rm -rf ${SCENARIO_RESULTS_FOLDER}/mon
 277                 mkdir ${SCENARIO_RESULTS_FOLDER}/mon
 278                 if ! ./parse_mon_log.sh $SCENARIO_RESULTS_FOLDER &> /dev/null; then
 279                         echo "Error parsing mon files in $SCENARIO_RESULTS_FOLDER"
 280                         exit 1
 281                 fi
 282         fi
 283         echo "---------------------- _scenario_parse" >> $ERR_LOG
 284 }
 285
 286
 287 #setup, start, wait, clean, parse
 288 case "$ACTION" in
 289         "setup")
 290                 scenario_setup
 291                 ;;
 292
 293         "start")
 294                 scenario_schedule
 295                 ;;
 296
 297         "wait")
 298                 scenario_wait
 299                 ;;
 300
 301         "clean")
 302                 scenario_clean
 303                 ;;
 304
 305         "parse")
 306                 scenario_parse
 307                 ;;
 308
 309         "run")
 310                 scenario_setup
 311                 scenario_schedule
 312                 scenario_wait
 313                 scenario_clean
 314                 scenario_parse
 315                 ;;
 316
 317         *)
 318                 usage
 319                 exit 1
 320                 ;;
 321 esac
 322
 323 rm -f $TMP_FILE
 324 exit 0