#!/bin/sh # Script to investigate a FreeBSD system and fix some errors # Script to perform various maintance jobs in a FreeBSD # # Minas Dasygenis # 2017-2026 # LATEST VERSION: wget http://arch.ece.uowm.gr/mdasyg/misc/fix_IDS_freebsd_errors.sh # # To silent some messages set the enviromental varialbe SILENT to YES # export SILENT=YES # setenv SILENT YES # or uncoment the following line: # SILENT=YES # # To override detected versions do use the following: # # env OVERREL=14.1-RELEASE $0 chroot_golden # # OVERARCH=amd64 OVERREL=14.1-RELEASE $0 create_golden # OVERARCH=amd64 OVERREL=14.1-RELEASE $0 update_golden # OVERARCH=amd64 OVERREL=14.1-RELEASE $0 chroot_golden # # (NOTE) if pkg fails inside chroot try to: # fetch https://pkg.FreeBSD.org/FreeBSD:14:amd64/quarterly/Latest/pkg.pkg # tar -xvpf pkg.pkg -C / # # (NOTE) You can try also with: # BSDINSTALL_DISTSITE=http://ftp.fr.freebsd.org/pub/FreeBSD/releases/amd64/14.1-RELEASE/ bsdinstall jail /zstorage/.freebsd-golden/jail # # Future Extension: Consult Obsolete Files: https://raw.githubusercontent.com/musec/freebsd/master/ObsoleteFiles.inc # # 2020.07.28 Added |tee /dev/null to freebsd-update in order to remove the blocking prompts, like [END] # 2022.04.28 Added support for env variables arch/relname version=3 note1="Value1 is $OVERARCH" note2="Value is $OVERREL" # # If the flag SILENT is enabled, then some messages are not printed # if [ x"$SILENT" = x"YES" ] ; then silent=1 else silent=0 fi python_check_fix_script="/scripts/system/python_detect_check_fix_update_multiple_FreeBSD_packages.sh" # Resolve absolute path to this script case "$0" in /*) SCRIPT="$0" ;; # already absolute *) SCRIPT="$(pwd)/$0" ;; # convert relative → absolute esac sname=`basename $0` # Always have a /var/cores directory defaut to save kernel core files mkdir -p /var/cores # Lets check if freebsd-update supports no cron freebsd-update --help | grep -q not-running-from-cron if [ $? -eq 0 ] ; then nocron="--not-running-from-cron" else nocron="" fi # Returns MAJOR.MINOR-RELEASE, like 12.2-RELEASE myuname=`uname -r | sed -E 's/(^.*RELEASE).*$/\1/'` # Returns arch, like i386 or amd64 arch=`uname -m` msg="[$sname] Called at [`hostname -s`] with arguments [$@], running on [`uname -a`]" if [ $silent -eq 0 ] ;then echo $msg echo " " fi logger $msg if [ x"$OVERREL" != "x" ] ; then echo "DETECTED Env Override (OVERREL): $OVERREL" myuname=$OVERREL fi if [ x"$OVERARCH" != "x" ] ; then echo "DETECTED Env Override (OVERARCH): $OVERARCH" arch=$OVERARCH fi # Do we have a location to store pkgs? if [ -d /public/.pkgtmp ]; then pkgextra='-o PKG_CACHEDIR=/public/.pkgtmp' if [ $silent -eq 0 ] ;then echo "USING [$pkgextra]" fi elif [ -d /zstorage/.pkgtmp ] ; then pkgextra='-o PKG_CACHEDIR=/zstorage/.pkgtmp' if [ $silent -eq 0 ] ;then echo "USING [$pkgextra]" fi elif [ -d /zstorage/tmp/pkg ]; then pkgextra='-o PKG_CACHEDIR=/zstorage/tmp/pkg' if [ $silent -eq 0 ] ;then echo "USING [$pkgextra]" fi else pkgextra="" fi note="PKGExtra is: $pkgextra" #Used for automatic answer autoyes=0 mybasedir="/home/$myuname" PATH=/rescue:$PATH export PATH # The link that carries the official kernel and base files #basefiles="ftp://ftp.fr.freebsd.org/mirrors/ftp.freebsd.org/releases/" basefiles2="http://ftp.fr.freebsd.org/pub/FreeBSD/releases/" basefiles="http://ftp.gr.freebsd.org/pub/FreeBSD/releases/" # Download the requisite check at: http://arch.ece.uowm.gr/mdasyg/misc/check_requisite_library_files.sh requisite_files_check="/scripts/system/check_requisite_library_files.sh" #The location of our save file configfile=/root/.freebsdGOLDEN if [ -z "$TMP" ] ; then TMP=/home/tmp fi IDS=$TMP/IDS ids=$TMP/IDS_files list_of_packages=$TMP/$sname.list_of_packages.$$ IDS_symlinks=$TMP/IDS_symlinks IDS_regfile=$TMP/IDS_regfile pkgB=$TMP/IDS_pkgB lsfiles_gold=$TMP/IDS_lsfiles_gold lsfiles_normal=$TMP/lsfiles_normal lsfiles_gold_permissions=$TMP/IDS_lsfiles_gold_permissions lsfiles_normal_permissions=$TMP/lsfiles_normal_permissions emptydirs=$TMP/IDS_emptydirs findoptions='( -path boot/GENERIC -o -path boot/kernel.old1 -o -path usr/local -o -path usr/tmp -o -path boot/kernel.old -o -path proc -o -path "*/proc" -o -path "*/proc/*" ) -prune -o' # ONLY for freebsd-ids # ignore these objects in freebsd-update ids creation (symlinks/wrong permissions/...) ids_ignorefiles=" (/var/db/etcupdate)|(/etc/mail/aliases)|(/usr/.cache)|(passwd)|(/group)|(/inetd.conf)|(hosts)|(/syslog.conf)|(ttys)|(/csh.cshrc)|(/motd)|(/profile)|(.db)|(/services)|(/shells)|(/etc/aliases)|(/etc/ssh/)|(/sysctl.conf)|(/var/spool)|(/locate.rc)|(/etc/mail/aliases)|(/var/spool/mqueue)|(/.cshrc)|(.k5login)|(/.login)|(/etc/ntp.conf)|(/csh.login)|(/csh.logout)|(/dumpdates)|(/var/run)|(/root)|(/etc/crontab)|(/pam.d/sshd)|(/login.conf)|(/nsswitch.conf)|(/pam.d/telnetd)|(/security/audit_control)|(/security/audit_user)|(locate.database) " # We ignore these files/directories # for IDS_not_on_local ignorelsfilesnotonlocal="-e usr/lib32 -e var/db/etcupdate -e usr/.cache -e etc/mail/freebsd.submit.cf -e etc/mail/aliases -e etc/mail/freebsd.cf -e etc/hosts.allow -e usr/local/bin/scripts" # We ignore these files/directories # for IDS_not_on_global # FIX for: "The following files are present on local system but not on golden:" # find list ignorelsfiles="-e boot/modules/if_wg.ko -e usr/ports -e usr/local/www -e boot/modules/if_re.ko -e etc/ntp.conf -e etc/mail/aliases -e etc/mail/aliases -e var/db -e etc/hosts.allow -e etc/mail/freebsd.cf -e boot/modules/vbox -e boot/zfs/zpool.cache -e boot/loader.conf -e boot/entropy -e usr/bin/ldd32 -e usr/lib32 -e usr/compat -e libexec/ld-elf32 -e boot/modules/linker.hints -e boot/linker.hints -e usr/extra -e boot/firmware -e usr/home -e usr/opt -e usr/.cache -e .sujournal -e usr/local/bin/scripts" # Ignore these folders that are empty on local file system ignoreemptydirs="-e usr/local/www/ -e usr/local/bin/scripts -e usr/local/etc -e usr/.snap -e usr/.cache -e usr/compat/linux/sys -e usr/compat/linux/proc -e usr/compat/linux/dev" # # List of folders to verify that no excess files exist, compared to the Golden Image # folders_to_check="boot/ bin/ usr/ lib/ libexec/ rescue/ sbin/ etc/" # # We do require a TMP file # if [ ! -d $TMP ] ; then echo "Please set TMP variable. Currently: [$TMP]" echo " " exit 9 fi # # Synchronize master/passwd database just to make sure everything is in order # /usr/sbin/pwd_mkdb -p /etc/master.passwd > /dev/null 2>&1 # # Clean created files, tidy up # clean() { for i in $ids $pkgB $IDS $IDS_regfile $IDS_symlinks $lsfiles_gold $TMP/IDS_not_on_global $TMP/IDS_not_on_local $emptydirs do if [ -f $i ] ; then rm $i ; if [ $silent -eq 0 ] ; then echo "(clean function) Removing created files by this script: $i" ; fi fi done } FUNCTIONS_FILE="/scripts/backup/functions.sh" LOCAL_DOWNLOADED_FUNCTIONS_FILE="/root/functions.sh" REMOTE_URL="http://arch.ece.uowm.gr/mdasyg/misc/functions.sh" # (A) Try to load a local functions file load_local_file() { if [ -f "$FUNCTIONS_FILE" ]; then if [ $silent -eq 0 ] ; then echo "Using local functions file: $FUNCTIONS_FILE" fi . "$FUNCTIONS_FILE" return 0 fi return 1 } # (B) Try to download AND source the remote file download_file() { echo "Local file missing — attempting download from: $REMOTE_URL" # Try curl if command -v curl >/dev/null 2>&1; then if curl -fsSL "$REMOTE_URL" -o "$LOCAL_DOWNLOADED_FUNCTIONS_FILE"; then echo "Downloaded functions file using curl" . "$LOCAL_DOWNLOADED_FUNCTIONS_FILE" return 0 fi fi # Try wget if command -v wget >/dev/null 2>&1; then if wget -q "$REMOTE_URL" -O "$LOCAL_DOWNLOADED_FUNCTIONS_FILE"; then echo "Downloaded functions file using wget" . "$LOCAL_DOWNLOADED_FUNCTIONS_FILE" return 0 fi fi return 1 } # (C) Empty fallback functions fallback_functions() { echo "WARNING: Using fallback skeleton functions" notify_central_archstatus() { :; } } if load_local_file; then : elif download_file; then : else fallback_functions fi machine=`hostname -s` note="Locating config file /scripts/config/$machine.conf" if [ -r /scripts/config/$machine.conf ]; then note="Loading config file /scripts/config/$machine.conf" . /scripts/config/$machine.conf fi notify_status_message() { msg="Calling notify_status_message. update_status_uuid=$update_status_uuid, update_status_key=$update_status_key" status="$1" message="$2" if [ ! -z "$update_status_uuid" -a ! -z "$update_status_key" -a ! -z "$status" -a ! -z "$message" ] ; then notify_central_archstatus "$update_status_uuid" "$update_status_key" "$status" "$message" fi } # # Tasks to update freebsd unattended (base + packages) # fbsd_update_base() { unlock freebsd-update $nocron fetch | tee /dev/null freebsd-update $nocron install | tee /dev/null rc=$? if [ $rc -eq 0 ] ; then echo "We executed freebsd-update on the LOCAL system, so we have to clear old IDS files" "$SCRIPT" clean #Also, lets recreate the motd to be in sync.. /etc/rc.d/motd onerestart > /dev/null 2>&1 fi return $rc } fbsd_update_pkg() { echo "[$sname] Updating Packages" env ASSUME_ALWAYS_YES=YES pkg $pkgextra update rc=$? if [ $rc -ne 0 ] ;then return $rc ; fi # Lets perform a bit of cleanup rm -f $TMP/$sname.list_of_packages* pkg query '%o' > $list_of_packages env ASSUME_ALWAYS_YES=YES pkg $pkgextra autoremove rc=$? ; if [ $rc -ne 0 ] ;then return $rc ; fi env ASSUME_ALWAYS_YES=YES pkg $pkgextra clean -a rc=$? ; if [ $rc -ne 0 ] ;then return $rc ; fi env ASSUME_ALWAYS_YES=YES pkg $pkgextra upgrade rc=$? if [ $rc -eq 3 ] ;then note="NOT enough space. Lets try work arround. And try again" if [ -d /zstorage/tmp ] ; then mkdir -p /zstorage/tmp/pkg rm -rf /var/cache/pkg/ ln -s /zstorage/tmp/pkg /var/cache/ fi env ASSUME_ALWAYS_YES=YES pkg $pkgextra upgrade rc=$? if [ $rc -ne 0 ] ;then return $rc ; fi fi echo "List of Original Packages was saved at [$list_of_packages]" return 0 } # # Update packages and base system using the silent flag... # update_local() { notify_status_message 20 "[$sname] Starting update procedure for kernel [`uname -r`]." if [ $silent -eq 0 ] ; then fbsd_update_base rc1=$? fbsd_update_pkg rc2=$? else fbsd_update_base >/dev/null 2>&1 rc1=$? fbsd_update_pkg >/dev/null 2>&1 rc2=$? fi if [ -x "$python_check_fix_script" ]; then $python_check_fix_script else echo "[$sname] ====> File [$python_check_fix_script] not found" fi # Ensure local lists are fresh after an update lsfiles_normal if [ $rc1 -ne 0 -o $rc2 -ne 0 ] ; then msg="[$sname] Error in updating system with rc1=$rc1 (update base) and rc2=$rc2 (update packages). Please investigate." notify_status_message 1 "$msg" rc=43 reschedule_action_if_fail $rc 4 "$msg" $@ return 1 else notify_status_message 0 "[$sname] Update Completted succesfully." return 0; fi } # # Bring the files to create the directory of full golden image of the given release # fetch_golden_image() { if [ "$arch" = "amd64" ] ; then packs="base.txz kernel.txz lib32.txz" else packs="base.txz kernel.txz" fi which wget > /dev/null 2>&1 if [ $? -ne 0 ] ; then wget=0 else wget=1 urldownload=wget fi which fetch > /dev/null 2>&1 if [ $wget -eq 0 -a $? -ne 0 ] ; then fetch=0 else fetch=1 urldownload=fetch fi if [ $fetch -eq 0 -a $wget -eq 0 ] ; then echo "[ERROR] Mandatory tool wget/fetch not found" exit 4 fi basedir=$mybasedir if [ -x /scripts/system/mm.sh ] ; then ( /scripts/system/mm.sh $basedir ) > /dev/null 2>&1 fi if [ ! -d "$basedir" ] ; then ok=0 while [ $ok -eq 0 ] ; do echo "Do you want me to create the directory [$basedir]? y/n" read ans if [ x$ans = xy ] ; then mkdir -vp "$basedir" mkdir -vp "$basedir/var/db/freebsd-update/" ok=1 elif [ x$ans = xn ] ; then echo "I will not create this as you requested, and thus I exit." exit 4 else echo "Please only y or n" echo " " fi done fi for file in $packs do if [ -f $TMP/$file ] ; then echo "Deleting [$file] for a fresh copy" rm $TMP/$file fi sleep 1 done sleep 1 for ifile in $packs do cd $TMP && $urldownload \ ${basefiles}/$arch/$arch/${myuname}/$ifile if [ $? -ne 0 ] ; then echo "Error in downloading files from main link. Changing basefile server to [$basefiles2}" cd $TMP && $urldownload \ ${basefiles2}/$arch/$arch/${myuname}/$ifile if [ $? -eq 0 ] ; then basefiles=${basefiles2} fi fi note="On tar we preserve hard links and sparce files" if [ $? -eq 0 ] ; then tar -C $basedir/ -xvSpf $TMP/$ifile && rm $TMP/$ifile if [ $? -ne 0 ] ; then echo "Error in untarring [$TMP/$ifile] to [$basedir]" exit 9 fi else echo "Error in downloading [$ifile] to [$TMP]" exit 4 fi done } # # Delete the golden image # delete_golden_image() { basedir=$1 check_basedir $basedir if [ -d "$basedir" ] ; then echo "I will remove [$basedir]. Are you sure? (y/n)" read yn if [ "$yn" = "y" ]; then chflags -R noschg $basedir rm -rfv $basedir else echo "Sorry this is not a valid answer" fi fi } #Chroot to the golden image and use it as a normal filesystem chroot_golden_image() { basedir=$1 check_basedir $basedir mount -t devfs devfs $basedir/dev mount -t procfs proc $basedir/proc #mount -t tmpfs tmpfs $basedir/tmp mount -t nullfs /tmp $basedir/tmp if [ -d $basedir/zstorage/archive ] ; then mount -t nullfs /zstorage/archive $basedir/zstorage/archive mount -t nullfs /zstorage/archive2 $basedir/zstorage/archive2 fi echo "===> ENTERING/mounting chroot in $basedir <===" chroot $basedir echo "===> EXITING/umounting chroot in $basedir ===>" #after we exit umount $basedir/zstorage/archive umount $basedir/zstorage/archive2 umount $basedir/proc umount $basedir/tmp umount $basedir/dev } # Locate previous version of golden image and nuke them out clean_golden_image() { basedir=$1 check_basedir $basedir myunameRUN=`uname -r | sed -E 's/(^.*RELEASE).*$/\1/'` extraflags="" cd $basedir/../.. listofdirs=`ls` for idir in $listofdirs do echo $idir | grep -q RELEASE if [ $? -ne 0 ] ; then echo "Skipping [$idir] does not seem to be a RELEASE" continue fi localdir=`basename $idir` if [ "$localdir" = "$myunameRUN" ] ; then echo "Skipping [$localdir] because it is our running [$myunameRUN]" continue fi if [ -d "$idir" ] ; then echo "We should clean: [$idir]" echo "Do you want to clean it? Y/y" read akey if [ "$akey" = "Y" -o "$akey" = "y" ]; then echo "You agree to clean it" echo "We will rm-rf [`pwd`]/[$idir] in 5" echo -n 5 ; sleep 1 echo -n 4 ; sleep 1 echo -n 3 ; sleep 1 echo -n 2 ; sleep 1 echo -n 1 ; sleep 1 echo 0 ; sleep 1 chflags -Rv noschg $idir rm -rfv "$idir" else echo "You said NO...nothing to clean" fi fi done } # # Update the golden image using the latest binary patches # update_golden_image() { basedir=$1 check_basedir $basedir myunameRUN=`uname -r | sed -E 's/(^.*RELEASE).*$/\1/'` extraflags="" if [ $myunameRUN != $myuname ] ; then # echo "=======>>>>>> UPDATING Golden is not currently possible" echo "=======>>>>>> Different versions of running kernel [$myunameRUN] and specified kernel [$myuname]" extraflags="--currently-running $myuname" fi if [ -d "$basedir" ] ; then echo "Updating [$basedir]" echo -n "Starting: " echo -n 3 sleep 1 echo -n 2 sleep 1 echo -n 1 sleep 1 echo 0 freebsd-update $extraflags -b $basedir \ $nocron -d $basedir/var/db/freebsd-update/ fetch | tee /dev/null freebsd-update $extraflags -b $basedir \ $nocron -d $basedir/var/db/freebsd-update/ install | tee /dev/null #We update lsflags only on same golden/running kernel situation if [ $? -eq 0 -a -z "$extraflags" ] ; then echo "We executed freebsd-update on the GOLDEN image, so we are re-executing lsfiles_gold creation on golden. Please wait." lsfiles_golden $basedir fi else echo "Basedir [$basedir] absent" exit 5 fi #No need to update lsfiles_gold here, it has already been performed (if we have updated golden) } # # Create a list of Golden files using lsfiles_gold, in order to compare # our list of files and the list of golden files # lsfiles_golden() { basedir=$1 check_basedir $basedir #Lets create the golden list of lsfiles_gold echo "Creating list of GOLDEN files located at [$basedir]..." ( cd "$basedir" && find $folders_to_check $findoptions -type f -print > "$lsfiles_gold" ) rc1=$? ( cd "$basedir" && find $folders_to_check $findoptions -type f -exec stat -f "%OLp %Su:%Sg %N" {} \; > "$lsfiles_gold_permissions" ) rc2=$? if [ $rc1 -ne 0 ] || [ $rc2 -ne 0 ]; then echo "*** ERROR: Failed to generate Golden list at $basedir ***" rc=1 else rc=0 fi return $rc } # # Create a list of Golden files using lsfiles_gold, in order to compare # our list of files and the list of golden files # lsfiles_normal() { #Lets create the list of lsfiles_normal echo "Creating list of normal files.." cd / && find $folders_to_check $findoptions -type f -print > $lsfiles_normal cd / && find $folders_to_check $findoptions -type f -exec stat -f "%OLp %Su:%Sg %N" {} \; > $lsfiles_normal_permissions rc=$? if [ $rc -ne 0 ] ; then echo "*** ERROR at creating lsfiles_normal at lsfiles_normal_permissions ***" fi return $rc } # # Perform an IDS check on the golden image. It should not be # necessary, but just to be sure, why not? # ids_golden() { basedir=$1 check_basedir $basedir echo "Checking IDS of golden image [$basedir]" # IDS not needed usually, everything is in order freebsd-update -b $basedir \ $nocron -d $basedir/var/db/freebsd-update/ IDS | tee /dev/null lsfiles_golden $basedir } # # Usually the golden image directory is located on an NFS disk # in order to be used by many FreeBSD servers # so The structure is usually /mnt/nfs/XXX/XXX/TTT # so we grab the first structure /mnt/nfs and try to mount it (using entry in /etc/fstab) # check_basedir() { basedir=$1 if [ -x /scripts/system/mm.sh ] ; then ( /scripts/system/mm.sh $basedir ) > /dev/null 2>&1 fi if [ ! -d "$basedir" ] ; then #Lets try to mount it in case that does not exist basemount=`echo $basedir | cut -f 1-3 -d "/"` timeout 5 mount $basemount > /dev/null 2>&1 rc=$? #checking again whether basedir exists if [ ! -d "$basedir" ] ; then echo " " echo "Golden Image Folder [$basedir] seems not to exist. Tried to mount the base mountpoint [$basemount] and got [$rc]. Mount it manually or set it as 2nd parameter." echo "Please Create it: mkdir -p $basedir" echo " " exit 7 fi fi } # # We will create/examine the IDS files of our FreeBSD System # create_IDS() { if [ -z "$1" ] ; then basedir=$mybasedir else basedir=$1 fi check_basedir $basedir #First lets check if the IDS files that exist on our $TMP are older than 30 days. If they are older DO delete it /usr/bin/find $IDS -maxdepth 1 -type f -mtime +30 -delete > /dev/null 2>&1 # ONLY run heavy IO if the report is missing if [ ! -f $IDS ] ; then if [ $silent -eq 0 ] ; then echo "IDS log file [$IDS] not found. I will create one. Running: freebsd-update IDS" fi freebsd-update IDS | grep -E -v '(/root)|(hosts)' > $IDS else if [ $silent -eq 0 ] ; then echo "IDS log file [$IDS] is found. I will use this." fi fi # ALWAYS refresh these comparisons if called, # as they are fast CPU/Memory operations lsfiles_normal if [ ! -s "$lsfiles_gold" ] ; then if [ $silent -eq 0 ] ; then echo "DELETING [$lsfiles_gold] because it has 0 bytes" fi rm "$lsfiles_gold" > /dev/null 2>&1 rm "$lsfiles_gold_permissions" > /dev/null 2>&1 lsfiles_golden $basedir fi if [ ! -f "$lsfiles_gold" ]; then lsfiles_golden $basedir fi # Verify lists are not empty before proceeding if [ ! -s "$lsfiles_gold" ] || [ ! -s "$lsfiles_normal" ]; then echo "[ERROR] One or both file lists are empty." echo "Golden list: $lsfiles_gold" echo "Normal list: $lsfiles_normal" exit 1 fi create_emptydirs cat $IDS | grep 'but should have SHA' | grep -v -E "$ids_ignorefiles" | grep -v /libexec/ | grep -v /etc/mail/ | cut -f 1 -d" "> $ids if [ ! -f $lsfiles_gold ]; then lsfiles_golden $basedir fi # 3. Perform Comparisons # Missing/Extra File Comparisons # -F: Fixed strings, -v: Invert match, -x: Match whole line, -f: Read patterns from file # Fix for: Files on Golden but NOT on Local # We want lines in GOLDEN that are NOT in NORMAL # FIX: Removed quotes from variable to allow shell expansion of multiple -e flags grep -Fvxf "$lsfiles_normal" "$lsfiles_gold" | grep -vE $ignorelsfilesnotonlocal > "$TMP/IDS_not_on_local" # Fix for: Files on Local but NOT on Golden # We want lines in NORMAL that are NOT in GOLDEN # Προσθήκη του | grep -v "^etc/" για να μην πειράζει τα τοπικά configs # FIX: Removed quotes from variable to allow shell expansion of multiple -e flags grep -Fvxf "$lsfiles_gold" "$lsfiles_normal" | grep -vE $ignorelsfiles | grep -v "^etc/" > "$TMP/IDS_not_on_global" # Check for Permission/Ownership Mismatches against Golden # FIX: We compare NORMAL against GOLD to extract the GOLDEN attributes # ---> without quotes on ignorelsfiles # Check for Permission/Ownership Mismatches against Golden # FIX: We compare NORMAL against GOLD to extract the GOLDEN attributes grep -Fvxf "$lsfiles_normal_permissions" "$lsfiles_gold_permissions" | \ grep -vE $ignorelsfiles > "$TMP/IDS_perm_mismatch" # ============================================================ # NEW FILTER: Remove entries that are Symbolic Links locally # ============================================================ if [ -s "$TMP/IDS_perm_mismatch" ]; then # Δημιουργία καθαρού προσωρινού αρχείου cp /dev/null "$TMP/IDS_perm_mismatch_clean" while read -r perm_line; do # Extract filename (field 3 onwards) file_path=$(echo "$perm_line" | cut -d " " -f 3-) # Check 1: Does the file exist locally? if [ ! -e "/$file_path" ] && [ ! -L "/$file_path" ]; then # Το αρχείο δεν υπάρχει (είναι missing file που μάλλον αγνοήθηκε), # άρα ΔΕΝ είναι permission mismatch. Το προσπερνάμε. : # Check 2: Is it a Symlink on local system? elif [ -L "/$file_path" ]; then # Είναι symlink, άρα το αγνοούμε (δεν ελέγχουμε permissions σε links) : else # Υπάρχει ΚΑΙ είναι κανονικό αρχείο -> πραγματικό σφάλμα permissions echo "$perm_line" >> "$TMP/IDS_perm_mismatch_clean" fi done < "$TMP/IDS_perm_mismatch" # Αντικατάσταση του αρχικού αρχείου με το καθαρισμένο mv "$TMP/IDS_perm_mismatch_clean" "$TMP/IDS_perm_mismatch" fi # ============================================================ if [ -s "$TMP/IDS_not_on_global" ] ; then # Προσωρινή αποθήκευση των φιλτραρισμένων αποτελεσμάτων grep -v $ignorelsfiles "$TMP/IDS_not_on_global" > "$TMP/IDS_not_on_global_filtered" # Έλεγχος αν το φιλτραρισμένο αρχείο έχει περιεχόμενο if [ -s "$TMP/IDS_not_on_global_filtered" ]; then echo "==========> The following files are present on LOCAL system but not on golden: " sleep 1 cat "$TMP/IDS_not_on_global_filtered" | less echo "==========> The previous files are present on local system but not on golden." sleep 3 fi # Καθαρισμός προσωρινού αρχείου rm "$TMP/IDS_not_on_global_filtered" fi if [ -s "$TMP/IDS_not_on_local" ] ; then # Προσωρινό φιλτράρισμα για να μην ανοίγει κενό less grep -v $ignorelsfiles "$TMP/IDS_not_on_local" > "$TMP/IDS_not_on_local_filtered" if [ -s "$TMP/IDS_not_on_local_filtered" ]; then echo "==========> The following files are present on GOLDEN but not on local system: " sleep 1 cat "$TMP/IDS_not_on_local_filtered" | less echo "==========> The previous files are present on GOLDEN but not on local system." sleep 3 fi rm "$TMP/IDS_not_on_local_filtered" fi if [ -s "$TMP/IDS_perm_mismatch" ] ; then echo "==========> The following files have different PERMISSIONS/OWNERSHIP (Showing EXPECTED/Golden attributes): " sleep 1 # Εμφανίζει: Mode Owner:Group Path (π.χ. 644 root:wheel ./etc/rc.conf) cat "$TMP/IDS_perm_mismatch" | less echo "==========> The previous files have permission mismatches." sleep 3 fi if [ -s "$emptydirs" ] ; then echo "==========> The following directories are reported to be empty on the local file system: " sleep 1 # Εμφάνιση λίστας μόνο cat "$emptydirs" | less echo "==========> The previous directories are empty (Run 'fix' or 'fix_emptydirs' to remove them)." sleep 3 fi if [ -x $requisite_files_check ] ; then if [ $silent -eq 0 ] ; then echo "Examing dynamic libraries consistency (check_requisite_library_files.sh): STARTING" fi $requisite_files_check if [ $silent -eq 0 ] ; then echo "Examing dynamic libraries consistency (check_requisite_library_files.sh): COMPLETED" fi fi } #end create_IDS() create_emptydirs(){ if [ ! -f "$emptydirs" ] ; then #we jave to create the empty directories list if [ $silent -eq 0 ] ; then echo "CREATING [$emptydirs]" fi cd / && find $folders_to_check -type d -empty | grep -v $ignoreemptydirs > $emptydirs fi } # # copy files that are present on the Golden image but # not on our base system # fix_missing_files() { basedir=$mybasedir check_basedir $basedir files_were_changed=0 if [ -s $TMP/IDS_not_on_local ]; then echo "The following files are present on golden but not on local: " sleep 3 less $TMP/IDS_not_on_local echo -n "I will copy them on local in: " echo -n "3" ; sleep 1 echo -n "2" ; sleep 1 echo -n "1" ; sleep 1 echo "0" nr=`cat $TMP/IDS_not_on_local | wc -l` counter=1 cat $TMP/IDS_not_on_local | while read line ; do if [ ! -f "/$line" ] ; then echo -n "Fix missing file [$line] - [$counter / $nr] " mydir=`dirname "$line"` if [ ! -d "/$mydir" ] ; then mkdir -vp "/$mydir" ; fi if [ -L "/$line" ] ; then rm "/$line" ; fi # Perform the copy cp -vPa "$basedir/$line" "/$line" # Set Sane Defaults immediately chown -v root:wheel "/$line" chmod -v 755 "/$line" # Temporary safe default files_were_changed=1 sleep 1 else echo "file [/$line] exists, I do not overwrite it..." fi #this also may count files alreary present, but at least the counter will be accurate counter=`expr $counter + 1` done # CRITICAL: After copying new files, refresh the local permission list # so the next fix_perms call knows about them. lsfiles_normal echo "Moving to phase Present At local file system NOT on golden file system." echo -n "." ; sleep 1 ; echo -n "."; sleep 1 ; echo -n "." ; sleep 1 ; echo . fi if [ -s $TMP/IDS_not_on_global ] ; then echo "The following files are present on local system but not on golden: " sleep 3 cat $TMP/IDS_not_on_global | grep -v $ignorelsfiles | less echo -n "I will move them out of the way in: " echo -n "3" ; sleep 1 echo -n "2" ; sleep 1 echo -n "1" ; sleep 1 echo "0" nr=`cat $TMP/IDS_not_on_global | grep -v $ignorelsfiles | wc -l` counter=1 cat $TMP/IDS_not_on_global | grep -v $ignorelsfiles | while read line ; do echo -n "[$counter / $nr] $line" mkdir -p /usr/local/backup/ if [ -f "/$line" ] ; then echo "$line" | grep 'backup' if [ $? -ne 0 ] ; then chflags noschg "/$line" > /dev/null 2>&1 mv -v "/$line" /usr/local/backup/ #Lets save the date of the last modification date > /usr/local/backup/DASYGENIS_freebsd_ids_BACKUPDIR 2>/dev/null echo "These files here have different IDS from the base and have been moved here. You can safely delete this directory" >> /usr/local/backup/DASYGENIS_freebsd_ids_BACKUPDIR sleep 1 else msg="[$sname] Warning: Line [/$line] cannot be moved to /usr/local/backup/ because it carries the word backup" logger $msg echo $msg fi else echo "$line : file abducted by UFO..." fi counter=`expr $counter + 1` done fi # Phase 3: Conditional Refresh if [ $files_were_changed -eq 1 ]; then echo "System state changed. Re-evaluating integrity..." lsfiles_normal # Refresh local snapshot create_IDS # Re-run create_IDS specifically for the permission diff fix_perms # Apply exact Golden metadata fi echo "Examining packages that have wrong dependencies (pkg check)" pkg check -d pkgB_list } pkgB_list() { if [ ! -f $pkgB ]; then echo "Examing dynamic libraries using pkg (long process...)" ( pkg check -B | grep -v Checking ) > $pkgB 2>&1 fi } fix_pkgB() { pkgB_list if [ -s $pkgB ] ; then listfiles=`cat $pkgB | grep fstat | sed 's@^.*failed for(\(.*\)).*$@\1@'` dependencies=`for line in $listfiles do pkg which $line | awk '{print $NF}' done | uniq` for i in $dependencies do pkg $pkgextra install -y -f $i done fi } fix_python() { if [ -x "$python_check_fix_script" ]; then $python_check_fix_script -f else echo "[$sname] ====> File [$python_check_fix_script] not found" fi } # # According to the IDS results of freebsd-update # try to fix permissions on our base system # fix_perms() { if [ ! -f "$IDS" ] || [ ! -f "$TMP/IDS_perm_mismatch" ]; then create_IDS fi if [ ! -f $IDS ] ; then if [ $silent -eq 0 ] ; then echo "It is required to create the check files....." fi "$SCRIPT" check fi # Fix permissions based on the Golden Permissions list if [ -s "$TMP/IDS_perm_mismatch" ]; then cd / echo "Fixing Permissions and Ownership based on Golden Image..." # Μέτρηση συνολικών γραμμών για τον counter total_lines=$(wc -l < "$TMP/IDS_perm_mismatch" | tr -d ' ') current_line=1 while read -r perm_line; do # New Format from stat: 644 root:wheel ./path/to/file mode=$(echo "$perm_line" | awk '{print $1}') owner=$(echo "$perm_line" | awk '{print $2}') # Cut from field 3 onwards to handle spaces in filenames correctly file_path=$(echo "$perm_line" | cut -d " " -f 3-) # Έλεγχος αν είναι Symbolic Link - Αν ναι, το αγνοούμε if [ -L "/$file_path" ]; then echo "[$current_line / $total_lines] Skipping Symlink: [/$file_path]" elif [ -f "/$file_path" ]; then echo "[$current_line / $total_lines] Fixing line [$perm_line] at file_path [$file_path]" # Apply Octal Mode (e.g., 644) directly chmod -v "${mode:-755}" "/$file_path" # Apply Owner:Group (e.g., root:wheel) chown -v "${owner:-root:wheel}" "/$file_path" else echo "[$current_line / $total_lines] --------FILE DOES NOT EXIST: [/$file_path]" fi # Αύξηση του μετρητή current_line=$((current_line + 1)) done < "$TMP/IDS_perm_mismatch" fi # to enable lastlogin information for users, just in case chmod go+r /var/log/utx* > /dev/null 2>&1 #Lets fix permissions echo "Starting Fixing Permissions:" eval `grep permission $IDS | grep / | awk '{print "chmod -v "$8,$1";"}' 2>/dev/null` echo "OK Done Fixing Permissions. Going to Fixing Ownership:" eval `grep 'should be owned' $IDS | tr -d '.' | awk '{print "chown -v "$NF,$1";"}' 2>/dev/null` } fix_perms_mtree() { # # Restore Basic Permissions using mtree # if [ ! -d /etc/mtree ] ; then echo "Missing template directory /etc/mtree" exit 5 fi # -U Modify the owner, group, permissions, and flags, create dirs # -f read the specifications from file cd / # why??? umount -f /mnt/* chmod 777 /tmp /var/tmp/vi.recover/ /home/tmp/ #chmod -Rv go-w /var #WARNING /var/run/fd/3 points to / to whole disk # FIX: Use find -x to prevent crossing mount points (like /var/run/fd -> tank) find -x /var -type d -exec chmod -v go-w {} + find -x /var -type f -exec chmod -v go-w {} + # chmod 777 /tmp /var/tmp/vi.recover/ chmod -R 777 /var/tmp mtree -U -f /etc/mtree/BSD.root.dist mtree -U -f /etc/mtree/BSD.sendmail.dist cd /var mtree -U -f /etc/mtree/BSD.var.dist cd /usr mtree -U -f /etc/mtree/BSD.usr.dist cd /usr/include mtree -U -f /etc/mtree/BSD.include.dist chmod -Rv go-w /bin /usr /etc/ /scripts /sbin /root /rescue /libexec /lib /boot chmod -R go-rw /home/mdasyg /home/bigbrother chmod go-rwx /etc/ssh/ssh_host_* chmod go-rwx /.snap # Set UID Files chmod u+s `which crontab` chmod u+s `which sudo` chmod u+s `which screen` chmod u+s `which at` chmod go+x /home/bigbrother/ chmod go+r /home/bigbrother/.muttrcTemplate chmod go+r /home/bigbrother/.smime.rc chmod go+r /home/bigbrother/.gpg.rc chmod go+r /home/bigbrother/.mailcap chmod go+rx /dev/ chmod 777 /dev/null chmod -R 700 /tmp/screens chmod 755 /tmp/screens/ chmod 1777 /var/named/tmp chmod 0777 /var/spam chmod -R 0700 /var/samba chmod -R 0755 /var/db/samba4 chmod -R 700 /home/bigbrother/.ssh/ chmod -R 700 /home/mdasyg/.ssh/ chmod go-rwx /scripts/backup/.rsyncd.secrets chmod o-w /var/authpf chmod go-rwx ~bigbrother/.gnupg chmod go-rwx ~mdasyg/.gnupg chmod g-w /var/games rmdir /var/spool/dma/ rmdir /var/rwho rmdir /var/games rmdir /var/authpf # Live USB chmod -R ugo-x /scripts/system/live/etc/ #for wpad.dat chmod go+x /scripts/system/live/etc/squid/ chmod go+x /scripts/system/live/etc/ chmod go+x /scripts/system/live/ } # # According to the IDS results, try to create the messages that # either something is a regfile and should be link or vise versa # create_regfilelist() { if [ $silent -eq 0 ] ; then echo "Calling: create_regfilelist(): STARTING" fi if [ ! -f $IDS_regfile ] ; then cat $IDS | grep -v -E "$ids_ignorefiles" | grep 'is a regular file, but should be a symlink' |\ cut -f 1 -d " " > $IDS_regfile cat $IDS | grep -v -E "$ids_ignorefiles" | grep ' is a symlink, but should be a regular file' |\ cut -f 1 -d " " >> $IDS_regfile fi if [ $silent -eq 0 ] ; then echo "Calling: create_regfilelist(): COMPLETED" fi } # # Fix regular files accoring to the list of IDS regfile # that is produced by freebsd-update IDS # fix_regular_file() { create_regfilelist while read line do #Lets check if original exist if [ -f "$mybasedir/$line" ] ; then if [ -f "$line" ] ; then #echo "Deleting: [$line]" rm "$line" fi echo "Replacing: [$line] with golden file" cp -av "$mybasedir/$line" "$line" fi done < $IDS_regfile } # # Some people lock / and /usr for security purposes. Lets remount RW # unlock() { /sbin/mount -u -w / > /dev/null 2>&1 /sbin/mount -u -w /usr > /dev/null 2>&1 /sbin/mount -u -w /var > /dev/null 2>&1 } # # Create the list of symlinks in order to be fed # to the proper function for fixing # create_symlinks_list() { if [ $silent -eq 0 ] ; then echo "Calling create_symlinks_list(): STARTING" fi if [ ! -f $IDS_symlinks ] ; then cat $IDS | grep 'is a symlink' |\ grep 'but should be a symlink' |\ grep -v -E "$ids_ignorefiles" |\ sed -E 's@^([^ ]*).* ([^ ]*)\.$@\1 - \2@g' > $IDS_symlinks fi if [ $silent -eq 0 ] ; then echo "Calling create_symlinks_list(): COMPLETED" fi } # # Clean empty dirs # fix_emptydirs() { #create the list $emptydirs if not present create_emptydirs cwd=`pwd` nothingelsetoremove=0 while [ $nothingelsetoremove -eq 0 ] ; do I_did_remove_a_directory=0 #originally we have not removed any directory counter=1 nr=`cat $emptydirs | wc -l` while read line do dir=/"$line" # Directories matching an expression are skipped.. echo $dir | grep -q $ignoreemptydirs if [ $? -eq 0 ] ; then continue fi if [ -d "$dir" ] ; then #If this is a mounted dir (even empty) just skip it mount | grep -q " $dir " && continue echo -n "[$counter / $nr] Removing Empty directory [$dir]: " sleep 1 rmdir "$dir" if [ $? -eq 0 ] ; then echo "OK" I_did_remove_a_directory=1 else echo "FAIL" fi else note="-----> DASYGENIS WARNING: Empty Directory [$dir] migrated to another system and does not exist anymore here." fi counter=`expr $counter + 1` done < $emptydirs #If we have removed some directories then re-run creation if [ $I_did_remove_a_directory -eq 1 ] ; then rm $emptydirs create_emptydirs else #Nothing removed, so we can escape the loop nothingelsetoremove=1 fi done } # # Create the symbolink links on the base system # fix_symlinks() { create_symlinks_list cwd=`pwd` while read line do orig=`echo $line | cut -f 1 -d " "` target=`echo $line | cut -f 3 -d " "` newcd=`dirname $orig` if [ -d $newcd ] ; then cd $newcd if [ -f "$target" ] ; then echo "[$orig]--->[$target]" sleep 1 rm $orig ln -v -s $target $orig else echo "-----> WARNING: Target [$target] does not exist at cd [$newcd]" fi else echo "-----> WARNING: Directory [$newcd] does not exist" fi done < $IDS_symlinks } # # If a file fails on IDS, then just copy it from the golden image # fix_ids() { # If the ids list is missing, we must create it if [ ! -s "$ids" ]; then create_IDS fi if [ ! -s $ids ] ; then echo "Filesize of [$ids] is not greater than 0 so nothing to do at fix_ids()" return fi check_basedir $mybasedir sleep 2 clear for file in `cat $ids` ; do if [ -f ${basedir}/$file ] ; then if [ ! -f $file ] ; then echo "File [$file] at original filesystem does not exist originally" else diff -q $file $basedir/$file > /dev/null 2>&1 if [ $? -eq 0 ] ; then echo "File: [$file] is the same with GOLDEN file, skipping."; sleep 1 ; continue ; fi echo "File: [$file] OLD<->NEW diffs" sleep 1 # diff $file $basedir/$file if [ $autoyes -eq 0 ] ; then sdiff $file $basedir/$file | less -F else sdiff $file $basedir/$file | less -F -e fi ok=1 while [ $ok -eq 1 ] ; do if [ $autoyes -eq 0 ] ; then echo -n "======================> Replace y/n/a <======================> " read mychar else echo " " echo " " echo "----------------------> AUTOMATIC REPLY YES <----------------------" mychar=y fi if [ x"$mychar" = x"y" -o x"$mychar" = x"a" ] ; then cp -fpv $basedir/$file $file && echo "COPIED" || echo "COPY FAILED" ok=0 sleep 1 if [ x"$mychar" = x"a" ] ; then echo '****SETTING AUTOMATIC REPLY FROM NOW ON****' autoyes=1 fi elif [ x"$mychar" = x"n" ] ; then echo "Skipping... [$file]" sleep 1 ok=0 else echo "Please retry" sleep 1 continue fi done fi else echo "File [$basedir]/[$file] which has wrong IDS according to file, does not exist in base image. Maybe base image is incomplete....skipping" continue fi done } # ---------------------- # MAIN script starts here # ---------------------- # # Define mybasedir location # # If the user gave 2 parameters, then the second is the # golden image location if [ $# -eq 2 ] ; then if [ -d $2 ]; then mybasedir=$2/$arch else echo "You gave [$2] as the golden image location but I cannot seem to access this. Please check and re-execute." exit 5 fi else if [ -f $configfile ]; then if [ $silent -eq 0 ] ; then echo "Reading $configfile" fi . $configfile #Check if the golden dir carries the #MAJOR.MINOR name. If not then we will #use a subfolder echo $GOLDEN | grep -q '[0-9][0-9]\.[0-9]' if [ $? -ne 0 ]; then mybasedir=$GOLDEN/$myuname/$arch else mybasedir=$GOLDEN/$arch fi fi fi if [ $silent -eq 0 ] ; then echo "GOLDEN image set at [$mybasedir]" fi mybasedir=$(echo "$mybasedir" | sed 's/\/\//\//g') #OLD--> goldendir=`echo $mybasedir | sed "s@\(^.*\)/$myuname/$arch@\1@g"` goldendir=$(echo "$mybasedir" | sed "s@\(^.*\)/$myuname/$arch@\1@g" | sed 's/\/\//\//g') #If there is a config file then check if it needs updating if [ -f $configfile ] ; then grep -q "$goldendir" "$configfile" if [ $? -ne 0 ] ; then echo "GOLDEN=$goldendir" > $configfile if [ $silent -eq 0 ] ; then echo "Updating config file [$configfile]." fi fi else #no config file, save our config if [ $silent -eq 0 ] ; then echo "****[INFO] Creating config file [$configfile]." fi echo "GOLDEN=$goldendir" > $configfile fi rc=0 # # Execution states of this script # case $1 in check) clean create_IDS $mybasedir create_regfilelist create_symlinks_list cat $ids cat $IDS_symlinks cat $IDS_regfile #if size greater than zero if [ -s "$ids" -o -s "$IDS_symlinks" -o -s "$IDS_regfile" -o -s "$TMP/IDS_not_on_global" -o -s "$TMP/IDS_not_on_local" -o -s "$TMP/IDS_perm_mismatch" ] ; then if [ $silent -eq 0 ] ; then echo "*** Problems detected as indicated ***" fi exit 5 else if [ $silent -eq 0 ] ; then echo "*** NO Problems detected ***" fi exit 0 fi ;; ids_golden) ids_golden $mybasedir ;; update_golden|update_remote|update_global) update_golden_image $mybasedir ;; chroot_golden_image|chroot_golden|chroot_remote|chroot_global) chroot_golden_image $mybasedir $@ ;; flush_golden|delete_golden) delete_golden_image $mybasedir ;; update|update_local) unlock update_local rc=$? ;; update_pkg) unlock fbsd_update_pkg rc=$? ;; create|create_golden|create_global) fetch_golden_image "$SCRIPT" update_golden ;; clean_golden|clean_golden_image) unlock clean_golden_image $mybasedir ;; fix_perms) unlock create_IDS fix_perms ;; fix_perms_mtree) unlock fix_perms_mtree ;; fix_ids) unlock create_IDS fix_ids ;; fix_regular_files) unlock create_IDS fix_regular_file ;; fix_symlinks) unlock create_IDS fix_symlinks ;; fix_missing_files) unlock clean fix_missing_files ;; fix_emptydirs) unlock clean fix_emptydirs ;; fix_pkgB) unlock fix_pkgB ;; fix_python) unlock fix_python ;; fix) unlock # First, fix structural/content issues fix_perms fix_regular_file fix_symlinks fix_ids fix_python # Synchronization (Copies missing / Backs up extra) fix_missing_files # This copies files + updates local list # Final cleanup and package repairs fix_emptydirs fix_pkgB if [ -x $requisite_files_check ] ; then $requisite_files_check fix fi #Also, lets recreate the motd to be in sync.. /etc/rc.d/motd onerestart > /dev/null 2>&1 ;; clean) clean ;; *) if [ $# -eq 1 ] ; then echo "You gave: [$1] which is not valid" fi echo "fix_IDS_freebsd_errors.sh: A script by Minas Dasygenis to keep FreeBSD in a good shape. Run it regularly to ensure 100% correctness. Version $version" echo "$sname check/update_golden/delete_golden/clean_golden/update_local/update_pkg/ids_golden/create_golden/fix/fix_perms/fix_perms_mtree/fix_ids/fix_regular_files/fix_missing_files/fix_emptydirs/fix_pkgB/clean" echo "--------------------------------------------------------------------------------------------------------------------" echo "check: Check the base system for typical incosistencies. Full range of tests" echo "create_golden: Fetch and create the golden files" echo "update_golden: binary patch the directory carrying the golden image of our release" echo "chroot_golden: chroot to the golden image and use it normally" echo "delete_golden: delete the golden basedirectory" echo "clean_golden: remove PREVIOUS versions of golden directory if present" echo "update_local: Perform binary base+package update to our system" echo "update_pkg: Update only local packages" echo "ids_golden: Perform an IDS check using freebsd-update on the golden image" echo "fix: Fix possible errors on the base system" echo "fix_perms: Fix possible permissions errors on the base system" echo "fix_perms_mtree: Fix permissions according to mtree (in case of chmod -R 777)" echo "fix_ids: Fix possible ids errors on the base system" echo "fix_regular_file: Fix possible errors on the regular files" echo "fix_missing_files: Fix any missing files on the base system compared to golden. Copy missing files to base. Remove obsolete files from base." echo "fix_emptydirs: Delete empty directories from base system" echo "fix_pkgB: Fix problems that are reported from pkg -B" echo "fix_python: Fix old python libraries and replace with new versions" echo "clean: Perform cleanup of temporary IDS files" echo "--------------------------------------------------------------------------------------------------------------------" echo "STEP1: $SCRIPT update_local" echo "STEP2: $SCRIPT check" echo "STEP3: $SCRIPT fix" exit 7 ;; esac # The /var/spool/mqueue should carry always +x otherwise the create_per_user script misbehaves if [ -d /var/spool/mqueue ] ; then chmod go+x /var/spool/mqueue > /dev/null 2>&1 fi exit $rc