#!/bin/sh

dir=../ngram-count-gt

if [ -f $dir/swbd.3grams.gz ]; then
	gz=.gz
else
	gz=
fi

smooth="-cdiscount 0 -gt3min 1"

# create LM from counts
ngram-count -debug 1 \
	$smooth \
	-read $dir/swbd.3grams$gz \
	-vocab $dir/eval2001.vocab \
	-lm swbd.3bo$gz

ngram -debug 0 \
	-lm swbd.3bo$gz \
	-ppl $dir/eval97.text

ngram -lm swbd.3bo$gz \
	-renorm \
	-ppl $dir/eval97.text

ngram -lm swbd.3bo$gz \
	-renorm -minbackoff 1e-5 \
	-ppl $dir/eval97.text \
	2> ngram.stderr

grep -c 'insufficient backoff mass' ngram.stderr

rm -f swbd.3bo$gz ngram.stderr

