#!/bin/sh
# /home/jhs/bin/.sh/dups
# http://www.berklix.com/~jhs/bin/.sh/dups
# Bourne shell script by Julian H. Stacey 2021-09-17
# DESCRIPTION:
# Find all duplicate files in current directory & under sub directories,
# & output a suggested script of commands to strip duplicates.
# Do Not run the output script blind, but first review it, eg:
#	# bracketc.c balancer: <
#	cd ~ ; dups > tmp/dups2edit ; vi tmp/dups2edit ; sh tmp/dups2edit
# as tmp/dups2edit will need half the filenames re-ordered,
# so it does not strip the wrong duplicate file,
# eg tmp/dups2edit might suggest:
#	cmpd -d \
#	sources/master/valuable_source.c \
#	sources/scratch_temp/experiment.c \
#
# which if run would delete your original source, not the junk temp.

# NEEDS: http://www.berklix.com/~jhs/src/bsd/jhs/bin/public/cmpd
# BUGS: Not tested on files with spaces or other punctuaton in file names, see
#	http://www.berklix.com/~jhs/bin/.sh/nospace

# Example:
#	cd ~ ; make clean ; dups
#	cd ~/xtra/LN ; make clean ; dups
#	cd ~/mail ; make clean ; dups

# History:
# I have run this on my large home directory & my pictures archive,
# without problem, to seek out duplicates, that I have then Manually
# decided which to strip against which with cmpd.

# Notes:
# I seem to recall hearing it is possible for md5 to not be unique
# between 2 different files ? Whether that is true or not, No problem,
# even if md5 thinks they are the same, just in case they are not,
# or have changed since, cmpd will only delete a file if the content
# is exactly the same as a reference file.

# Improvements done:
# - md5 only gets run once per file to save CPU
# - The list of filename is stored in a file, not an environment variable,
# to avoid exhaustion of max variable size,
# if eg scanning a whole big home directory.

# bracketc.c balancer: <
echo -n "If running $0 from ~/._ first make clean,"	> /dev/stderr
# bracketc.c balancer: <
echo "else there"					> /dev/stderr
# bracketc.c balancer: <
echo -n "will be temporary files,"			> /dev/stderr
# bracketc.c balancer: <
echo "& generated stuff installed into ~/public_html"	> /dev/stderr
# sleep 5
cp /dev/null ~/tmp/.dups.$$.just_md5
cp /dev/null ~/tmp/.dups.$$.namePlusMd5
cp /dev/null ~/tmp/.dups.$$.intercept

# I reduced	 0 1 2 3 15	to 1 2 3 15
# as 0 is normal clean exit. & sometimes when I am debugging.
# I want to more explicity hash out the rm below,

trap "rm -f ~/tmp/.dups.$$.just_md5 ~/tmp/.dups.$$.namePlusMd5 \
	.dups.$$.noIgnoreDirs ~/tmp/.dups.$$.intercept ; exit" 1 2 3 15

# Ignore files I will not want to take manual action on, eg transients
# & those I have moved to pending later directories.
# Files beginning with comma eg ,[0-9]\* are from mailer exmh
# before it moves them from one directory to another.
for i in `find -s . -type f \
	 \! -name ,[0-9]\* \
	 \! -name .just_com \
	 \! -name .mh_sequences \
	 \! -name 20\[0-9\]\[0-9\]\*-1-0\.html \
	 \! -name Makefile \
	 \! -name Makefile_subdirs \
	 \! -name \*-thumb.jpg \
	 \! -name \*/\*-thumb.jpg \
	 \! -name \*/\.backup/\* \
	 \! -name \*/\.xvpics/\* \
	 \! -name \*\/passwd\/\* \
	 \! -name \*\/Inbox\/Pend\/\* \
	 \! -name \*/neat-round.theme/\* \
	 \! -name \.xmhcache \
	 \! -name adr.fax \
	 \! -name adr.ltr \
	 \! -name adr1 \
	 \! -name adr2 \
	 \! -name back.png \
	 \! -name bot.png \
	 \! -name cmt \
	 \! -name co \
	 \! -name dept \
	 \! -name dummy.html \
	 \! -name dummy.pdf \
	 \! -name dummy.txt \
	 \! -name editdata.html \
	 \! -name email \
	 \! -name fax \
	 \! -name filelist.html \
	 \! -name index.html \
	 \! -name index.shtml \
	 \! -name left.png \
	 \! -name liner.txt \
	 \! -name makefile \
	 \! -name msg\.\* \
	 \! -name name1 \
	 \! -name name2 \
	 \! -name next.png \
	 \! -name page.html \
	 \! -name pict0\[0-9\]\[0-9\]\[0-9\]-1-0\.html \
	 \! -name prev.png \
	 \! -name ref.fax \
	 \! -name ref.ltr \
	 \! -name right.png \
	 \! -name stacey_e.txt \
	 \! -name stacey_g.txt \
	 \! -name tel \
	 \! -name top.png \
	 \! -name wpa_supplicant.conf.master \
	 \! -name \*passwd\* \
	 ` ; do
	#	The above patterns fail to exclude files:
	#	Pattern			File
	#	\*/\.backup/\*		./mail/.backup/original/_6pF.b-SlqB.dell.no.berklix.net
	#	\*\/Inbox\/Pend\/\*	./mail/Inbox/Pend/1
	#	\*\/passwd\/\*		./txt/pri/passwd/1and1.co.uk/geoffharries.com/.NOTES
	#	\*\/passwd\/\*		./txt/pri/passwd/1and1.co.uk/geoffharries.com/224378683:
	YY=`echo $i \
		| grep -v /.backup/ \
		| grep -v /.xvpics/ \
		| grep -v /Inbox/Pend \
		| grep -v /passwd/ \
		`
	if [ "X${YY}" = "X" ]; then # {
		# echo $0: Debug: Skipping $i
	else	# }{
		# echo $0: Debug: Doing $i
		XX=`md5 -q $i`
		# Zero size files are often special case dummies
		# not to be removed, they have an md5 of ${ZERO}
		ZERO=d41d8cd98f00b204e9800998ecf8427e
		# bracketc.c balancer: <<
		echo $XX | grep -v ${ZERO}	\
			>> ~/tmp/.dups.$$.just_md5
		# bracketc.c balancer: <<
		echo "$i $XX" | grep -v ${ZERO}	\
			>> ~/tmp/.dups.$$.namePlusMd5
	fi	# }
	done

	# bracketc.c balancer: <
	grep -v Inbox/Pend ~/tmp/.dups.$$.namePlusMd5 | \
	grep -v /.xvpics/ > ~/tmp/.dups.$$.noIgnoreDirs

for i in `sort ~/tmp/.dups.$$.just_md5 | uniq -d` ; do
	echo "cmpd -d \\"
	grep $i ~/tmp/.dups.$$.namePlusMd5 | sed -e s/$i// | sed -e s/\$/\\\\/
	echo
	done

# for i in `sort ~/tmp/.dups.$$.just_md5 | uniq -d` ; do
#	echo "cmpd -d \\"
#	grep $i ~/tmp/.dups.$$.noIgnoreDirs | sed -e s/$i// | sed -e s/\$/\\\\/
#	echo
#	done

rm -f ~/tmp/.dups.$$.just_md5 ~/tmp/.dups.$$.namePlusMd5 ~/tmp/.dups.$$.noIgnoreDirs ~/tmp/.dups.$$.intercept

# Excludes Inbox/Pend when dups is run in ~/mail/ for when I move a
# mail from Inbox, but I still have not dealt with it. Only after I
# have dealt with it, then it can be deleted from mail/Inbox/Pend
# and the only copy remaining just in ref. in mail/archive/