# loop through all local S5R1 cache files
for cache in /local/distributed/clone/spray/EatH/S5R1/cache_* ; do
  files=`/home/bema/bin/cache_length "$cache"`
  # loop through all files in that cache, starting with 0
  file=0
  while [ $file -lt $files ]; do
    # that's the get_file call 
    if path=`/home/bema/bin/get_file -m "EatH/S5R1" -c "$cache" -i "$file"`; then
      md5sum "$path" >>/local/bema/S5R1_md5.txt
    else
      echo "get_file failed on path '$path' for cache '$cache' file '$file'" >&2
    fi
    file=`expr $file + 1`
  done 
done

git-clone git+ssh://n0.aei.uni-hannover.de/srv/git/public/bema/atlas.git

Distributing Data (for Admins)

• find a distribution pattern that fits the access pattern of the application
• generate "cache files", i.e. groups of data files that will be accessed in sequence. The syntax is one line for each datafile in the cache:

  data_file_name;/mount/path/to/first/instance;/mount/path/to/second/instance;/mount/path/to/fallback

  • for the EatH/S5R3 distribution scheme, pipe a list of filenames (even with full paths) into the sort_caches program (git repository). For each input line it writes out a line filename;firstnode;secondnode;fallbackdir which could easily extended to a cachefile line e.g. by piping the output through

    awk -F\; '{print $1 ";/atlas/node/" $2 "/distributed/spray/data/EatH/S5R3n;/atlas/node/" $3 "/distributed/spray/data/EatH/S5R3n;/atlas/data/d04/distributed/fallback/EatH/S5R3/" $4}'

  • for a new distibution it's probably easiest to use the nodename of the first instance of the file for the fallback directory (for S5R3 the directoryname is calculated in sort_caches only because the fallback directories were already there from previous obsolete distributions)
• create the fallback directories on the data server and (hard-) link the files that belong there according to the entries in the cache files
• make sure the target directories for files and caches exist on all nodes. This would be /local/distributed/spray/data/EatH/S5R1 and /local/distributed/clone/spray/EatH/S5R1 respectively, if EatH/S5R1 is the module.
• let the nodes fetch the data files from the fallback directories
  • if you followed the advice above for new distributions, nodes < n0672 can simply fetch them from the fallback directories corresponding to their hostname, other nodes should try to fetch them from their partners in a second step, and must substract 671 from their number to get the fallback directory name on the data server to fetch files from if the partners aren't available)
• distribute the cache files to all nodes
• some inspiration for command line to do this can be found in Bernds Sandbox

Useful scripts for maintenance

check_complete_S5R1.sh

#!/bin/sh

cache=/local/distributed/clone/spray/EatH/S5R1
files=/local/distributed/spray/data/EatH/S5R1n
log_s=/root/log/check_compleateS5R1_s.log
log_h=/root/log/check_compleateS5R1_h.log

dsh -g nodes -c -M "cd $files; ls | wc -l" | tee $log_h
dsh -g  nodes -c -M "cd $cache; grep -h \"node/\`hostname\`\" *| cut -d \";\" -f 1| sort| uniq|wc -l"    | tee $log_s

sort $log_s > /tmp/foo
mv /tmp/foo $log_s
sort $log_h > /tmp/foo
mv /tmp/foo $log_h

check_complete_S5R3.sh

#!/bin/sh

cache=/local/distributed/clone/spray/EatH/S5R3
files=/local/distributed/spray/data/EatH/S5R3
log=/root/log/check_compleateS5R3.log

dsh -g nodes -c -M "if [ \"\`hostname| sed \\\"s/n//\\\"| bc\`\" -gt 671 ]; then sn=\`echo \"\$HOSTNAME-671\"| sed \"s/n/1/\"| bc| sed \"s/^1/n/\"\` ;else sn=\$HOSTNAME ;fi;  cd $cache ;j=0;for i in \`head -n 1 *| grep -B1 \$sn|grep cach| cut -d \" \" -f 2\`;do let j=\$j+\`wc -l \$i| cut -d \" \" -f 1\`;done; r=\`ls $files | wc -l | cut -d \" \" -f 1\`; echo \$j-\$r"| tee $log

validate_S5R1.sh

#!/bin/sh

files=/local/distributed/spray/data/EatH/S5R1n
log=/root/log/validate_S5R1.log

dsh -g nodes -c -M  "cd $files;for i in \`ls \`;do /usr/local/bin/validator_test \$i || echo \$i;done 2>/dev/null"| tee $log

get_missing_files_S5R1.sh

#!/bin/sh

nodes=list
cache=/local/distributed/clone/spray/EatH/S5R1
files=/local/distributed/spray/data/EatH/S5R1n
log=/root/log/missingfiles_S5R1.log

dsh -g $nodes -c -M "cd $cache; grep -h \"node/\`hostname\`\" *| cut
-d \";\" -f 1| sort| uniq> /tmp/list1"
dsh -g $nodes -c -M "cd $files; ls > /tmp/list2"
dsh -g $nodes -c -M "cd /tmp; diff list1 list2| grep \"<\"| cut -d \"
\" -f 2"| tee $log