#!/bin/bash -e

# Helper script to develop/debug mini-buildd.
#
# Quickstart:
#  Enter your dev chroot (preferably sid). sudo should be configured.
#  $ cd mini-buildd
#  $ ./devel installdeps
#  $ ./devel update

declare -g -a _ON_EXIT=()
on_exit_run()
{
	local line
	for line in "${_ON_EXIT[@]}"; do
		${line}
	done
}
on_exit()
{
	_ON_EXIT+=("${*}")
}
trap "on_exit_run" EXIT

_check_prog()
{
	local path
	for path in $(printf "${PATH}" | tr ":" " "); do
		local prog="${path}/${1}"
		if [ -x "${prog}" ]; then
			printf "I: Found: ${prog}.\n"
			return 0
		fi
	done
	printf "E: '${1}' not found in path; please install.\n" >&2
	printf "I: You may use './devel installdeps' to install all deps needed.\n" >&2
	exit 1
}

MBD_HTTPD="${MBD_HTTPD:-$(hostname -f):8066}"  # Use 'MBD_HTTPD=some.where.com:123 ./devel' to use something else than this host
MBD_PJPATH="$(readlink -f $(dirname $0))"
MBD_PYPATH="${MBD_PJPATH}/src"
MBD_SETUP_CFG="${MBD_PJPATH}/setup.cfg"
MBD_LINTIANRC="${MBD_PJPATH}/.lintianrc"
MBD_HTTPD_PROTO="http"
MBD_TOOL="${MBD_PYPATH}/mini-buildd-tool --protocol=${MBD_HTTPD_PROTO} admin@${MBD_HTTPD}"  # m-b-tool shortcut for testing calls
MBD_LAST_CHANGES="${MBD_PJPATH}/.last_changes"

mbd_installdeps()
{
	sudo apt-get update
	sudo apt-get --no-install-recommends install devscripts equivs

	# Debian package build dependencies; using target-release=*
	# here to always allow highest versions of any sources
	# configured (for example, for backports).
	mk-build-deps --install --root-cmd=sudo --remove --tool="apt-get --no-install-recommends --target-release='*'"

	# Extra tools needed for checks
	sudo apt-get install --no-install-recommends pycodestyle pylint3 python3-apt python3-pip python3-wheel devscripts faketime python3-bs4 python3-keyrings.alt tidy codespell wget apache2-utils ssl-cert
	# Extra tools needed vc and package building
	sudo apt-get install --no-install-recommends git git-buildpackage
	# binary package dependencies so we can just dpkg -i for testing
	sudo apt-get install --no-install-recommends --target-release='*' sbuild schroot reprepro debootstrap lintian
}

# Find files with python code
declare -a MBD_PYFINDPARAMS=(-not -wholename './debian/*' -not -wholename './.git/*' -not -wholename './build/*' -not -wholename './.pybuild/*' -type f)
mbd_pyscripts()
{
	local f
	for f in $(find \( "${MBD_PYFINDPARAMS[@]}" \) -a -executable); do
		if head -1 "${f}" | grep --quiet "bin/python"; then
			printf "%s\n" "${f}"
		fi
	done
}

mbd_pymodules()
{
	local -a exceptions=(-true)
	[ -z "${*}" ] || exceptions=($(printf " -not -wholename %s" "${@}"))
	find -name "*.py" -a \( "${MBD_PYFINDPARAMS[@]}" \) -a \( "${exceptions[@]}" \)
}

mbd_pysources()
{
	mbd_pyscripts
	mbd_pymodules
}

mbd_pyenv()
{
	printf "export PYTHONPATH=\"${MBD_PYPATH}\"\n"
}

pyenv()
{
	eval "$(mbd_pyenv)"
	python3 ./setup.py build_py
}

mbd_installdjango()
{
	dpkg -s python3-django | grep "^Version" || true
	sudo dpkg --install ../django-versions/python3-django*${1}*.deb
	dpkg -s python3-django | grep "^Version"
}

# python API has nicer support for this.
# Example call: MBD__PACKAGE="mbd-test-cpp" MBD__DIST="squeeze-test-unstable" [MBD__VERSION=1.2.3] ./devel wait4package
mbd_wait4package()
{
	local sleep=30
	printf "\nWaiting for ${MBD__PACKAGE}-${MBD__VERSION} to appear in ${MBD__DIST}:\n"
	while true; do
		# ${MBD_TOOL} show ${MBD__PACKAGE}
		if ${MBD_TOOL} show ${MBD__PACKAGE} 2>/dev/null | grep "^${MBD__DIST}\b.*${MBD__VERSION}"; then
			printf "\nOK, '${MBD__PACKAGE}' found in '${MBD__DIST}'.\n"
			break
		else
			printf "*"
			sleep ${sleep}
		fi
	done
}

mbd_service()
{
	if sudo ischroot && [ -d /run/systemd/system ]; then
		# Seems we are in a chroot, and host is running systemd
		# The service will not be started in that case
		# For now, we really want to start the service anyway, so this is still usable in "traditional" chroots
		# (Rather use a container-based environment to test)
		sudo mv /lib/lsb/init-functions.d/40-systemd /lib/lsb/init-functions.d/40-systemd.DISABLED || true
		sudo /etc/init.d/mini-buildd ${1}
		sudo mv /lib/lsb/init-functions.d/40-systemd.DISABLED /lib/lsb/init-functions.d/40-systemd || true
	else
		sudo service mini-buildd "${1}"
	fi
}

# Create "big files" for HTTPD tests and benchamrking.
# POST: MBD_HTTPD_TESTFILES (fileItem: sha1sum) will be available globally.
mbd_gen_httpd_testfiles()
{
	declare -g -A MBD_HTTPD_TESTFILES=()
	local fileSize relPath="repositories/test/pool/testfiles"
	local absPath="/var/lib/mini-buildd/${relPath}"
	sudo mkdir -p "${absPath}"

	for fileSize in 5 50 100; do
		local id="file${fileSize}M"
		local of="/var/lib/mini-buildd/${relPath}/${id}.bin"
		[ -e "${of}" ] || sudo dd if="/dev/urandom" of="${of}" count="${fileSize}" bs="1M" >/dev/null 2>&1
		MBD_HTTPD_TESTFILES["${id}:${MBD_HTTPD_PROTO}://${MBD_HTTPD}/${relPath}/${id}.bin"]="$(sha1sum < "${of}")"
	done
}

mbd_changes() # [<arch>]
{
	local arch=${1-$(dpkg-architecture --query=DEB_BUILD_ARCH)}
	printf "../mini-buildd_$(dpkg-parsechangelog --show-field=version)_${arch}.changes"
}

#
# Runner functions: mbd_run:<sequence>:<level>:<name>
#
# <sequence>: Two-digit sequence number (0-9) -- order in which to run.
# <level>: Higher second digit, higher "cost".
#  00-09: deploy: Actions needed to test-deploy only.
#  10-19: check : Static tests, fully automatic.
#  20-29: test  : Live tests, system tests.

mbd_run:00:99:prepare-system()
{
	# Hack to be able to do authorized non-interactive API calls
	mbd_pythonkeyringtestconfig()
	{
		sudo apt-get install python3-keyrings.alt || true   # for the PlainTextKeyring (newer versions only)
		local configDir="$(python3 -c "import keyring.util.platform_; print(keyring.util.platform_.config_root())")" || true
		[ -n "${configDir}" ] || configDir="${HOME}/.local/share/python_keyring"
		mkdir -p "${configDir}"

		local configFile="${configDir}/keyringrc.cfg"
		local pkVersion=$(dpkg-query --show --showformat='${Version}' python3-keyring)
		if dpkg --compare-versions ${pkVersion} gt 7; then
			cat <<EOF >"${configFile}"
[backend]
# stretch (p-k > 8)
default-keyring=keyrings.alt.file.PlaintextKeyring
EOF
		else
			cat <<EOF >"${configFile}"
[backend]
# jessie (p-k 4.0)
default-keyring=keyring.backends.file.PlaintextKeyring
EOF
		fi
		cat "${configFile}"
	}

	# Add Debian's default self-signed cert (from ssl-cert) to ca-certificates (make API calls work via https).
	# Note: Don't use the same base filename (ssl-cert-snakeoil.pem -> ssl-cert-snakeoil.crt), else update-ca-certificates will remove/symlink snakeoil the two
	mbd_snakeoil2cacerts()
	{
		sudo make-ssl-cert generate-default-snakeoil
		local snakeoilfile="/etc/ssl/certs/ssl-cert-snakeoil.pem"
		local cacertfile="/usr/local/share/ca-certificates/ssl-cert-mini-buildd.crt"
		[ -e "${cacertfile}" ] || sudo cp -v -a /etc/ssl/certs/ssl-cert-snakeoil.pem "${cacertfile}"
		sudo update-ca-certificates --fresh
	}

	# Add Debian's default self-signed cert (from ssl-cert) to local chromium certstore
	mbd_snakeoil2browser()
	{
		local pkiDir="${HOME}/.pki" trust="P,," f="/etc/ssl/certs/ssl-cert-snakeoil.pem"
		local certDB=$(find  ${HOME}/.mozilla/ -name "cert*.db")
		# chromium
		[ ! -e "${pkiDir}/nssdb" ] || certutil -d sql:${pkiDir}/nssdb -A -t "${trust}" -n"$(basename ${f} .pem)" -i "${f}"
		 # firefox
		[ ! -e "${certDB}" ] || certutil -d $(dirname ${certDB}) -A -t "${trust}" -n"$(basename ${f} .pem)" -i "${f}"
	}

	mbd_pipclear()
	{
		local installed=$(pip3 freeze --user)
		[ -z "${installed}" ] || pip3 uninstall --yes ${installed}
	}

	mbd_pipupdateprospector()
	{
		pip3 install --upgrade --ignore-installed prospector
	}

	mbd_pythonkeyringtestconfig
	mbd_snakeoil2cacerts
	mbd_snakeoil2browser
	mbd_pipclear
	mbd_pipupdateprospector
}

# See .prospector.yaml for config
mbd_run:01:10:prospector()
{
	_check_prog prospector
	(
		pyenv
		prospector $(mbd_pysources)
	)
}

mbd_run:01:12:codespell()
{
	local ups=$(codespell --ignore-words-list=referer,fpr,stati --quiet-level=2 $(mbd_pysources))
	if [ -n "${ups}" ]; then
		printf "${ups}\n" >&2
		return 1
	fi
}

mbd_run:01:12:htmlmisc()
{
	local class errors=0 ignoreRegex="mbd-action-.*"

	local templateTags=$(grep "^def [[:alnum:]]" src/mini_buildd/templatetags/mini_buildd_tags.py | cut -d" " -f2 | cut -d "(" -f1 | tr "\n" " ")
	printf "I: Found django template tags: %s\n" "${templateTags}" >&2
	local idSelectors=$(grep -h -o "#mbd[[:alnum:]\-]\+" src/mini_buildd/static/css/*.css | sort -r | uniq | cut -c 2- | tr "\n" " ")
	printf "I: Found CSS ID selectors: %s\n" "${idSelectors}" >&2

	for token in ${templateTags} ${idSelectors}; do
		if [[ ${token} =~ ${ignoreRegex} ]]; then
			printf "Skipping %s.\n" "${token}"
		else
			if ! grep -r -q "${token}" src/mini_buildd/templates/; then
				printf "\nE: Token unused: %s:\n" "${token}" >&2
				grep -r "${token}" src/
				printf "\n"
				errors+=1
			fi
		fi
	done
	printf "HTML: %s unused tokens found.\n" "${errors}"
	return ${errors}
}

mbd_run:02:12:pydoctests()
{
	(
		pyenv
		for m in $(mbd_pymodules ./setup.py ./doc/conf.py); do  # ./setup.py && ./doc/conf.py can't be used for doctests
			local module="$(basename $(tr '/' '.' <<< ${m:4}) '.py' | cut -d. -f2-)"
			printf "=> Doctest on %s (%s)\n" "${m}" "${module}"
			( cd ./src/ && ./run-doctest "${module}" )
		done
		python3 -m doctest -v src/mini-buildd src/mini-buildd-tool
	)
}

mbd_run:03:00:changelog()
{
	# Checking changelog (must be unchanged)...
	git diff-index --exit-code HEAD debian/changelog
	on_exit git checkout debian/changelog
	gbp dch --snapshot --auto
	mbd_changes >"${MBD_LAST_CHANGES}"
}

mbd_run:04:00:build()
{
	DEB_BUILD_OPTIONS+="nocheck" debuild --no-lintian -us -uc
}

mbd_run:05:11:lintian()
{
	local changes=$(cat "${MBD_LAST_CHANGES}" || mbd_changes)
	printf "I: Lintian-checking: %s\n" "${changes}" >&2
	local result=$(lintian --cfg="${MBD_LINTIANRC}" "${@}" "${changes}")
	printf "%s\n---\n" "${result}"
	if grep "^[EW]:" <<< ${result}; then
		read -p "Lintian FAILED (RETURN to ignore)" DUMMY
	fi
}

# This also checks "full" package building (with doc and check)
mbd_run:06:21:debrepro()
{
	debrepro
}

mbd_run:10:23:remove()
{
	mbd_service stop || true
	sudo dpkg --${1:-remove} mini-buildd mini-buildd-utils mini-buildd-doc python3-mini-buildd python-mini-buildd mini-buildd-common
}

mbd_run:10:23:purge()
{
	mbd_run:10:23:remove purge
}

mbd_run:11:00:install()
{
	cat ./devel.debconf.selections | sudo debconf-set-selections --verbose -
	# sudo debi --with-depends  # 2019 July: Not usable. See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=932896
	sudo apt --yes --allow-downgrades install $(mbd_changes)
}

mbd_run:12:00:restart()
{
	mbd_service restart
	until ${MBD_TOOL} status; do
		sleep 1
	done
}

mbd_run:13:20:apicalls()
{
	${MBD_TOOL} status
	${MBD_TOOL} getkey
	${MBD_TOOL} getdputconf
	${MBD_TOOL} getsourceslist wheezy
}

mbd_run:13:20:tidy()
{
	local url
	for url in \
		${MBD_HTTPD_PROTO}://${MBD_HTTPD}/mini_buildd \
		${MBD_HTTPD_PROTO}://${MBD_HTTPD}/mini_buildd/api \
		${MBD_HTTPD_PROTO}://${MBD_HTTPD}/mini_buildd/api?command=getdputconf \
		${MBD_HTTPD_PROTO}://${MBD_HTTPD}/accounts/login/ \
		; do
		printf "\n=> Testing HTML: ${url}\n"
		wget --output-document=- "${url}" | tidy -output /dev/null
	done
}

mbd_run:14:23:auto-setup()
{
	${MBD_TOOL} autosetup --confirm="autosetup"
	${MBD_TOOL} keyringpackages --confirm="keyringpackages"
	${MBD_TOOL} testpackages --confirm="testpackages"
}

mbd_run:15:23:build-and-migrate()
{
	${MBD_TOOL} testpackages --packages="mbd-test-cpp" --distributions="stretch-test-unstable" --confirm="testpackages"
	MBD__PACKAGE="mbd-test-cpp" MBD__DIST="stretch-test-unstable" mbd_wait4package
	${MBD_TOOL} migrate mbd-test-cpp stretch-test-unstable --confirm="migrate"
	MBD__PACKAGE="mbd-test-cpp" MBD__DIST="stretch-test-testing" mbd_wait4package
	${MBD_TOOL} migrate mbd-test-cpp stretch-test-testing --confirm="migrate"
	MBD__PACKAGE="mbd-test-cpp" MBD__DIST="stretch-test-stable" mbd_wait4package
}

_test_install()
{
	on_exit sudo dpkg --purge "${1}"
	sudo apt "${@:2}" install "${1}"
}

mbd_run:16:23:apt-tofu-bootstrap()
{
	# Bootstrap keyring package (tofu style)
	local sources_list="/etc/apt/sources.list.d/mbd-testsuite.list"
	on_exit sudo rm -v -f "${sources_list}"
	on_exit sudo apt update
	${MBD_TOOL} getsourceslist stretch --suite=stable | sudo tee "${sources_list}"
	sudo apt --allow-insecure-repositories update
	_test_install $(hostname)-archive-keyring --allow-unauthenticated
	sudo apt update
}

mbd_run:17:23:apt-install()
{
	# install previously built test package
	_test_install mbd-test-cpp
}

mbd_run:18:23:extra-packages()
{
	# Extra test packages when available
	[ ! -d ../test-packages/ ] || dput --force --unchecked --no-upload-log mini-buildd-$(hostname) ../test-packages/*.changes
}

mbd_run:19:23:httpd-testfiles()
{
	mbd_gen_httpd_testfiles
	local item
	for item in ${!MBD_HTTPD_TESTFILES[@]}; do
		local id=$(cut -d: -f1 <<<${item}) url=$(cut -d: -f2- <<<${item})
		local of="$(mktemp)"
		wget --output-document="${of}" "${url}"
		local sha1sum="$(sha1sum < "${of}")"
		if [ "${sha1sum}" = "${MBD_HTTPD_TESTFILES[${item}]}" ]; then
			printf "HTTP download OK: %s=%s %s" "${id}" "${of}" "${sha1sum}"
		else
			printf "HTTP download FAILED: %s=%s %s != %s" "${id}" "${of}" "${sha1sum}" "${MBD_HTTPD_TESTFILES[${item}]}"
			return 1
		fi
	done
}

mbd_run:19:23:httpd-benchmark()
{
	mbd_gen_httpd_testfiles

	_ab_val() { cut -d: -f2- | cut -d[ -f1 | tr -d '[:space:]'; }

	local abRequests=50 item

	for item in django:${MBD_HTTPD_PROTO}://${MBD_HTTPD}/mini_buildd/repositories/test/ index:${MBD_HTTPD_PROTO}://${MBD_HTTPD}/repositories/test/pool/main/m/mbd-test-cpp/ ${!MBD_HTTPD_TESTFILES[@]}; do
		local id=$(cut -d: -f1 <<<${item})
		local url=$(cut -d: -f2- <<<${item})
		local c
		for c in 1 4; do
			local abResult=$(ab -n "${abRequests}" -c "${c}" "${url}")
			local server=$(grep "^Server Software:.*" <<<${abResult} | _ab_val)
			if (( c > 1 )); then
				local tpr=$(grep "^Time per request:.*(mean, across all concurrent requests)" <<<${abResult} | _ab_val)
			else
				local tpr=$(grep "^Time per request:.*(mean)" <<<${abResult} | _ab_val)
			fi
			{
				if grep "^Non-2xx responses:" <<<${abResult}; then
					printf "%s" "${abResult}"
					printf "\n\n%s\n" "E: Above ab call has non-200 responses!"
					tpr="-1.0"
				fi
			} >&2
			# Note: decimal separator in "ab" is ".", so e need to set LANG for printf
			LANG="C.UTF-8" printf "[%20s] %-12s: % 8.3f\n" "${server}" "${id} c=${c}" "${tpr}"
		done
	done
}

mbd_sequence()  # [<levelRegex>=00] [<name>] <hr>
{
	local levelRegex="${1:-00}" name="${2}" hr="${3}"
	for func in $(declare -F | cut -d" " -f3- | grep "^mbd_run:[[:alnum:]][[:alnum:]]:${levelRegex}:${name}" | sort || true); do
		if [ -n "${hr}" ]; then
			printf "%s " "$(cut -d: -f4 <<<${func})"
		else
			printf "%s " "${func}"
		fi
	done
}

mbd_run()  # [<levelRegex>=00] [<name>] [<customArgs>...]
{
	local -a info=()
	local func totalStartStamp=$(date +%s)
	local -i count=0
	for func in $(mbd_sequence "${1}" "${2}"); do
		printf "I: Running %s...\n" "${func}"
		local startStamp=$(date +%s)
		${func} "${@:3}"
		count+=1
		info+=("$(printf "OK (%03d seconds): %s" "$(($(date +%s) - startStamp))" "${func}")")
	done
	if ((count <= 0)); then
		printf "E: No runs for this sequence filter: ${1} ${2}"
		return 1
	fi
	printf "\nSequence results (%03d seconds):\n" "$(($(date +%s) - totalStartStamp))"
	printf "%s\n" "${info[@]}"
	printf "\nOK, %s runs succeeded ($(date)).\n" "${count}"
}

# Shortcuts
declare -A MBD_RUN_SHORTCUTS=(
	["check"]="10"
	["update"]="00"
	["updatecheck"]="[0-1][0-1]"
	["updatetest"]="[0-9][0-2]"
	["updatetestall"]="[0-9][0-9]"
)
# We can't iterate through the associative array in the given order later, so we at least want a sorted key list as helper
MBD_RUN_SHORTCUTS_SORTED="$(printf '%s\n' "${!MBD_RUN_SHORTCUTS[@]}" | sort -n)"

mbd_daemon.log()
{
	less --follow-name +F /var/lib/mini-buildd/var/log/daemon.log
}

mbd_access.log()
{
	less --follow-name +F /var/lib/mini-buildd/var/log/access.log
}

main()
{
	if [ -z "${1}" ]; then
		local p="./$(basename "${0}")" b=$(tput bold) i=$(tput sitm) r=$(tput sgr0)
		cat <<EOF
Usage: ${i}${p} <shortcut-or-runner-or-special> | run <groupRegex><levelRegex>${r} [<customArgs>...]

mini-buildd development helper.

${b}Sequence filter shortcuts${r}:
$(for s in ${MBD_RUN_SHORTCUTS_SORTED}; do printf "  ${i}${p} %-15s${r}: (%-10s) %s\n" "${s}" "${MBD_RUN_SHORTCUTS[${s}]}" "$(mbd_sequence "${MBD_RUN_SHORTCUTS[${s}]}" "" "hr")"; done)

${b}Runners${r}:
 ${i}${p} $(mbd_sequence ".." "" "hr" | tr " " "|")

${b}Special (non-runner) targets${r}:
 ${i}${p} logcat${r}: Follow all logs (daemon and access).
 ...-> Check source for other possible esoteric calls.
EOF
	else
		local shortcut="${MBD_RUN_SHORTCUTS[${1}]}"
		if [ -n "${shortcut}" ]; then
			mbd_run "${shortcut}" "" "${@:2}"
		else
			local func="mbd_${1}"  # direct function
			if [ "$(type -t "${func}")" = "function" ]; then
				${func} "${@:2}"
			else
				mbd_run ".." "${1}" "${@:2}"
			fi
		fi
	fi
}

main "${@}"
