path: root/Tools/portbuild/scripts/dopackages

#!/bin/sh

# configurable variables
pb=/a/asami/portbuild
user=asami

# packages for dependencies only
dummyports="x11/XFree86 x11-toolkits/Motif-dummy"

usage () {
  echo "usage: [-nobuild] [-noindex] [-noduds] [-nocvsup] branch"
  exit 1
}

# usage: makeindex pb scripts branch user
makeindex () {
  pb=$1
  scripts=$2
  branch=$3
  user=$4

  cd ${pb}/usr/ports
  echo "================================================"
  echo "generating index"
  echo "================================================"
  echo "index generation started at $(date)"
  ${scripts}/makeindex ${branch} || exit 1
  echo "index generation ended at $(date)"
  if grep -q non-existent INDEX; then
    echo "errors in INDEX:"
    grep non-existent INDEX
    exit 1
#    grep -C non-existent INDEX
#    grep -v non-existent INDEX > INDEX.tmp
#    mv -f INDEX.tmp INDEX
  fi
  echo $(wc -l <INDEX) "lines in INDEX"
  chown ${user} INDEX
  if ! awk -F '|' '{if (NF != 10) { error=1; printf("line %d: %s\n", NR, $0)}} END {if (error == 1) exit(1)}' INDEX; then
    echo "error in INDEX"
    exit 1
  fi
}

# usage: makeduds pb scripts branch
makeduds () {
  pb=$1
  scripts=$2
  branch=$3

  cd ${pb}/usr/ports
  echo "================================================"
  echo "generating duds"
  echo "================================================"
  echo "duds generation started at $(date)"
  if ! ${scripts}/makeduds ${branch}; then
    echo "error(s) detected, exiting script at $(date)"
    exit 1
  fi
  echo "duds generation ended at $(date)"
  echo $(wc -l < ${pb}/${branch}/duds) "items in duds"
  echo "duds diff:"
  diff ${pb}/${branch}/duds.old ${pb}/${branch}/duds
  cp -p ${pb}/${branch}/duds ${pb}/${branch}/duds.old
}

# usage: setupnode pb scripts branch me node md5
setupnode () {
  pb=$1
  scripts=$2
  branch=$3
  me=$4
  node=$5
  md5=$6

  echo "setting up of $node started at $(date)"
  scp -p -a ${scripts}/setupnode ${node}:${scripts}
  ssh -n ${node} ${scripts}/setupnode ${me} ${pb} ${branch} ${md5}
  echo "setting up of $node ended at $(date)"
}

# usage: restrictedlist pb scripts branch
restrictedlist () {
  pb=$1
  scripts=$2
  branch=$3

  cd ${pb}/usr/ports
  echo "================================================"
  echo "creating restricted list"
  echo "================================================"
  echo "restricted list generation started at $(date)"
  make ECHO_MSG=/usr/bin/true clean-restricted-list > ${pb}/${branch}/restricted.sh
  echo "restricted list generation ended at $(date)"
  echo $(grep -c '^#' ${pb}/${branch}/restricted.sh) "ports in ${pb}/${branch}/restricted.sh"
}
  
# usage: cdromlist pb scripts branch
cdromlist () {
  pb=$1
  scripts=$2
  branch=$3

  cd ${pb}/usr/ports
  echo "================================================"
  echo "creating cdrom list"
  echo "================================================"
  echo "cdrom list generation started at $(date)"
  make ECHO_MSG=/usr/bin/true clean-for-cdrom-list > ${pb}/${branch}/cdrom.sh
  echo "cdrom list generation ended at $(date)"
  echo $(grep -c '^#' ${pb}/${branch}/cdrom.sh) "ports in ${pb}/${branch}/cdrom.sh"
}

scripts=${pb}/scripts

umask 002
export PORTSDIR=${pb}/usr/ports

me=$(hostname -s)

echo "Subject: $me package building logs"
echo
echo "Called with arguments: "${1+"$@"}
echo "Started at $(date)"

PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/X11R6/bin

if [ $# = 0 ]; then
  usage
fi

nobuild=0
noindex=0
noduds=0
nocvsup=0

# optional arguments
while [ $# -gt 1 ]; do
  case "x$1" in
    x-nobuild)
      nobuild=1
      ;;
    x-noindex)
      noindex=1
      ;;
    x-noduds)
      noduds=1
      ;;
    x-nocvsup)
      nocvsup=1
      ;;
    *)
      usage
      ;;
  esac
  shift
done

# mandatory argument
branch=$1
if [ "x$branch" != x3 -a "x$branch" != x4 ]; then
  usage
fi

cd ${pb}/usr/ports
if [ "$nocvsup" = 0 ]; then
  echo "================================================"
  echo "running cvsup"
  echo "================================================"
  su ${user} -c 'cvsup -g -L 0 /etc/supfile.cvsup'
  echo "================================================"
  echo "running cvs update on /usr/ports"
  echo "================================================"
  su ${user} -c 'cvs -q update -d -P'
  echo "================================================"
  echo "running make checksubdirs"
  echo "================================================"
  make checksubdirs 
  echo "================================================"
  echo "running cvs update on /usr/opt/doc"
  echo "================================================"
  cd ${pb}/usr/opt/doc
  su ${user} -c 'cvs -q update -d -P'
fi

# these two not in background to check return status

if [ "$noduds" = 0 ]; then
  makeduds ${pb} ${scripts} ${branch}
fi

if [ "$noindex" = 0 ]; then
  makeindex ${pb} ${scripts} ${branch} ${user}
fi

md5=$(/sbin/md5 ${pb}/${branch}/tarballs/bindist.tar | awk '{print $4}')
echo "================================================"
echo "setting up nodes"
echo "================================================"
for node in $(awk '{print $1}' ${pb}/mlist); do
  setupnode ${pb} ${scripts} ${branch} ${me} ${node} ${md5} &
  sleep 2
done

restrictedlist ${pb} ${scripts} ${branch} &
sleep 2

cdromlist ${pb} ${scripts} ${branch} &

wait

echo "all preparation ended at $(date)"

cd ${pb}/usr/ports
make parallel > ../../${branch}/Makefile

cd ${pb}
if [ "$nobuild" = 0 ]; then
  rm -rf bak/distfiles
  mv -f distfiles bak
  mkdir distfiles
  chown -R ${user} distfiles
  cd ${pb}/${branch}/bak
  rm -rf errors logs packages old-errors
  cd ${pb}/${branch}
  mv -f errors logs packages old-errors make.* bak
  mkdir -p errors logs packages/All
  chown -R ${user} errors logs packages 
  for dir in ${dummyports}; do
    cp -p tarballs/$(cd ${pb}/usr/ports/$dir; make package-name).tgz packages/All
  done
  count=$(awk '{sum+=$2}END{print sum+NR/2}' ${pb}/mlist)
  cd ${pb}/${branch}/packages/All
  echo "================================================"
  echo "building packages (phase 1)"
  echo "================================================"
  echo "started at $(date)"
  make -k -j$count -f ../../Makefile > ../../make.0 2>&1 </dev/null
  echo "ended at $(date)"
  echo $(echo $(ls -1 ${pb}/${branch}/packages/All | wc -l) - 2 | bc) "packages built"
  echo $(echo $(du -sk ${pb}/${branch}/packages | awk '{print $1}') / 1024 | bc) "MB of packages"
  echo $(echo $(du -sk ${pb}/distfiles | awk '{print $1}') / 1024 | bc) "MB of distfiles"
  cd ${pb}/${branch}
  if grep -q 'ptimeout: killing' make.0; then
    echo "The following port(s) timed out:"
    grep 'ptimeout: killing' make.0 | sed -e 's/^.*ptimeout:/ptimeout:/'
  fi
  cp -rp errors old-errors
  cd ${pb}/${branch}/old-errors
  ${pb}/scripts/processlogs
  cd ${pb}/${branch}/packages/All
  echo "================================================"
  echo "building packages (phase 2)"
  echo "================================================"
  echo "started at $(date)"
  make -k -j$count -f ../../Makefile > ../../make.1 2>&1 </dev/null
  echo "ended at $(date)"
  rm XFree86-3.*.tgz Motif-*.tgz
  ${pb}/scripts/chopindex ${pb}/usr/ports/INDEX ${pb}/${branch}/packages > \
    ${pb}/${branch}/packages/INDEX
  echo $(ls -1 ${pb}/${branch}/packages/All | wc -l) "packages built"
  echo $(cat ${pb}/${branch}/packages/INDEX | wc -l) "lines in INDEX"
  md5 *.tgz > CHECKSUM.MD5
  echo $(echo $(du -sk ${pb}/${branch}/packages | awk '{print $1}') / 1024 | bc) "MB of packages"
  echo $(echo $(du -sk ${pb}/distfiles | awk '{print $1}') / 1024 | bc) "MB of distfiles"
  cd ${pb}/${branch}
  if grep -q 'ptimeout: killing' make.1; then
    echo "The following port(s) timed out:"
    grep 'ptimeout: killing' make.1 | sed -e 's/^.*ptimeout:/ptimeout:/'
  fi
  cd ${pb}/${branch}/old-errors
  new=""
  for i in *.log; do
    if [ ! -f ../errors/$i ]; then
      new="$new $(basename $i .log)"
    fi
  done
  if [ "x$new" != "x" ]; then
    echo "The following port(s) didn't build the first time around: $new"
  fi
  echo "================================================"
  echo "new failures"
  echo "================================================"
  cd ${pb}/${branch}/errors
  for i in *.log; do
    if [ ! -f ../bak/errors/$i ]; then
      echo -n " $(basename $i .log)"
    fi
  done
  echo
  echo "================================================"
  echo "old packages"
  echo "================================================"
  cd ${pb}/${branch}/bak/packages/All
  for i in *.tgz; do
    if [ ! -f ../../../packages/All/$i ]; then
      echo -n " $(basename $i .tgz)"
    fi
  done
  echo
  echo "================================================"
  echo "old failures"
  echo "================================================"
  cd ${pb}/${branch}/bak/errors
  for i in *.log; do
    if [ ! -f ../../errors/$i ]; then
      echo -n " $(basename $i .log)"
    fi
  done
  echo
  echo "================================================"
  echo "new packages"
  echo "================================================"
  cd ${pb}/${branch}/packages/All
  for i in *.tgz; do
    if [ ! -f ../../bak/packages/All/$i ]; then
      echo -n " $(basename $i .tgz)"
    fi
  done
  echo
  echo "================================================"
  echo "archiving /usr/ports"
  echo "================================================"
  echo "started at $(date)"
  cd ${pb}/usr
  tar --exclude CVS -czf ${pb}/${branch}/tarballs/ports.tar.gz ports
fi
echo "================================================"
echo "all done at $(date)"
echo "================================================"