generate_corpus.sh 3.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122
  1. #!/usr/bin/env bash
  2. set -e
  3. DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
  4. BASE_DIR="$( cd "$DIR/../../" && pwd )"
  5. BUILD_DIR_FUZZ_MODE="$DIR/../../build_fuzz"
  6. if [ ! -d "$BUILD_DIR_FUZZ_MODE" ]; then
  7. mkdir $BUILD_DIR_FUZZ_MODE
  8. fi
  9. BUILD_DIR_FUZZ_MODE="$( cd "$DIR/../../build_fuzz" && pwd )"
  10. BUILD_DIR_CORPUS="$DIR/../../build_corpus"
  11. if [ ! -d "$BUILD_DIR_CORPUS" ]; then
  12. mkdir $BUILD_DIR_CORPUS
  13. fi
  14. BUILD_DIR_CORPUS="$( cd "$DIR/../../build_corpus" && pwd )"
  15. cd $BUILD_DIR_CORPUS
  16. if [ -d "$BUILD_DIR_CORPUS/corpus" ]; then
  17. rm -rf "$BUILD_DIR_CORPUS/corpus"
  18. fi
  19. if [ -z ${TRAVIS+x} ]; then
  20. export CC=clang
  21. export CXX=clang++
  22. else
  23. # Travis needs a specific
  24. export CC=clang-6.0
  25. export CXX=clang++-6.0
  26. fi
  27. # First build and run the unit tests without any specific fuzz settings
  28. cmake -DUA_BUILD_FUZZING_CORPUS=ON -DUA_BUILD_UNIT_TESTS=ON ..
  29. make -j && make test ARGS="-V"
  30. if [ $? -ne 0 ] ; then exit 1 ; fi
  31. # Run our special generator
  32. $BUILD_DIR_CORPUS/bin/corpus_generator
  33. if [ $? -ne 0 ] ; then exit 1 ; fi
  34. # Now build the fuzzer executables
  35. cd $BUILD_DIR_FUZZ_MODE
  36. cmake -DUA_BUILD_FUZZING=ON ..
  37. make -j
  38. if [ $? -ne 0 ] ; then exit 1 ; fi
  39. merge_corpus() {
  40. local fuzzer="$1"
  41. local corpus_existing="$2"
  42. local corpus_new="$3"
  43. if [ -d "$corpus_existing" ]; then
  44. echo "Merging ${corpus_new} into ${corpus_existing}"
  45. "$fuzzer" -merge=1 "$corpus_existing" "${corpus_new}"
  46. else
  47. echo "Copying ${corpus_new} into ${corpus_existing}"
  48. cp -r ${corpus_new} ${corpus_existing}
  49. fi
  50. }
  51. # Iterate over all files and combine single message files to a full interaction, i.e.,
  52. # After running the corpus generator, the output directory contains single files for each
  53. # message (HEL, OPN, MSG..., CLO). Fuzzer needs these files to be combined into one single file
  54. CORPUS_SINGLE=$BUILD_DIR_CORPUS/corpus
  55. CORPUS_COMBINED=$BUILD_DIR_CORPUS/corpus_combined
  56. if [ -d $CORPUS_COMBINED ]; then
  57. rm -r $CORPUS_COMBINED
  58. fi
  59. mkdir $CORPUS_COMBINED
  60. # iterate over all the subdirectories
  61. subDirs=$(find $CORPUS_SINGLE -maxdepth 1 -mindepth 1 -type d)
  62. for dirPath in $subDirs; do
  63. # if empty, skip
  64. if ! [ -n "$(ls -A $dirPath)" ]; then
  65. #echo "Skipping empty $dirPath"
  66. continue
  67. fi
  68. dir=$(basename $dirPath)
  69. dirPathTmp=$CORPUS_COMBINED/${dir}
  70. if [ -d $dirPathTmp ]; then
  71. rm -r $dirPathTmp
  72. fi
  73. mkdir $dirPathTmp
  74. # The files are ordered by interaction. So we start with the first file
  75. # and combine all of them until we get the CLO file.
  76. # Then we start a new file and combine them again.
  77. currCount=1
  78. for binFile in `ls $dirPath/*.bin | sort -V`; do
  79. #echo "Combining $binFile to $dirPathTmp/msg_${currCount}.bin"
  80. cat $binFile >> $dirPathTmp/${dir}_msg_${currCount}.bin
  81. # if it is a close message, start new message
  82. if [[ "$binFile" == *clo.bin ]]; then
  83. currCount=$((currCount+1))
  84. fi
  85. done
  86. done
  87. merge_corpus $BUILD_DIR_FUZZ_MODE/bin/fuzz_binary_message $BASE_DIR/tests/fuzz/fuzz_binary_message_corpus/generated $CORPUS_COMBINED
  88. if [ $? -ne 0 ] ; then exit 1 ; fi