This script lists all the directories that are older than [days]
#!/bin/bash
usage="Usage: dir_diff.sh [days]"
if [ ! "$1" ]
then
echo $usage
exit 1
fi
now=$(date +%s)
hadoop fs -lsr | grep "^d" | while read f; do
dir_date=`echo $f | awk '{print $6}'`
difference=$(( ( $now - $(date -d "$dir_date" +%s) ) / (24 * 60 * 60 ) ))
if [ $difference -gt $1 ]; then
echo $f;
fi
done
#The following command will delete the files before certain date
#hadoop fs -ls shakespeare | tail -n+2 | xargs -n 8 |
#awk '{
#if ( ($(date)-$(date -d $6)) == 0)
#cmd="hadoop fs -rm " $8;
#system(cmd)
#}
#if ( ($(date)-$(date -d $6)) == 0) ---change the if condition to difference which you want..
#currently it will delete todays files...