#!/usr/local/perl
#>perl -w wordfreq.pl -h < movies.list > simple-count.html
#
# Computes Frequency of Words in Movie Titles.
#
# Uses slightly munched form of IMDb movies.list file.
#
# USAGE: reads file from stdin, outputs to stdout
#		 -h option outputs HTML, limits output to first 1000.
#			ommiting this outputs all words

my %count;			# hash of words and count
my $total = 0;		# total words read
my $movies = 0;		# total movies counted (lines)

my $html = shift;	# -h command line parameter

while (<>) {		# read from stdin
	$movies ++;		# read a line

	chomp;			# remove cr/lf

		# this is specific to the format of the IMDb text file
		# something like: >movie title (year)		year
		#			or    >movie title (year)	(code)		year
	s/\([\d\?]{4}\)[\W]+[\d\?]{4}/ /g;					# get rid of (YYYY)   YYYY
	s/\([\d\?]{4}\)[\W]+\([\w]+\)[\W]+[\d\?]{4}/ /g;	# get rid of (YYYY) (C) YYYY

		# now get word.  word is considered to be any string of alpha numeric characters
		# and a few international ones and a hyphen and apostrophe
	while ( /(\b[-'אבהזטיךמףצרשa-zA-Z0-9]+)/gx ) {
		$count{lc $1}++;		# count word (in lower case)
		$total ++;				# increment total words seen
	}
}

# get a list of sorted keys in the count hash (sort by count, then name)
my @keys = sort {
                $count{$b} <=> $count{$a}
                          ||
                      $a cmp $b
    } keys %count;

my $rank = 0;	# rank of current word being output
my $word;		# word being output

# outputting to HTML?
if( $html eq '-h')
{
# print the page details around the stats
print "<html>
<head>
	<title>Simple Frequency Count of Words in Movie Titles</title>
</head>
<body>
<h1>Simple Frequency Count of Words in Movie Titles</h1>
<p>The following list of word frequencies was generated from the list of all movie titles stored in the
<a href=\"http://www.imdb.com/\">Internet Movie Database</a> (IMDb) by a simple <a href=\"cgi-bin/echo.pl?wordfreq.pl\">perl script</a> on 25 August 1999.</p>
<p>It makes no attempt to remove articles, match words that are parts of other words (hyphenated, simply joined
or word stems) and can't tell the difference between words in different languages :) (eg &quot;die&quot; in German and English)</p>
<p>Only the top 1000 ranking words are shown (out of $#keys words and $movies movies).</p>
<p>Back to <a href=\"index.html\">Tom Paton's Homepage</a>.</p>
<h2>Summary</h2>
<p>The Following words are the first real words (English only, no articles, prepositions etc) in the top 100 (edited by hand, don't take it for gospel):</p>
<blockquote>
";
# print a summary of the top 100, this is then edited (in the html) to remove words that are not too useful.
	$rank = 0;
	foreach $word (@keys) {
		$rank++;
		print "$word (<i>$count{$word}</i>), \n";
		last if $rank >= 100;
	}


print "</blockquote>
<h2>Frequencies</h2>
<center>
<table border=0 width=\"100%\"><tr><td width=\"25%\"><center>
<table border=1>
<tr><th>Rank</th><th>Count</th><th>Word</th></tr>
";
	$rank = 0;
	foreach $word (@keys) {
		$rank++;
		print "<tr><td><i>$rank</i></td><td>$count{$word}</td><td><b>$word</b></td></tr>\n";
# split the output into 4 columns of 250 (each a table in a cell of a 4 column table)
		if ($rank == 250 or $rank == 500 or $rank == 750 )
		{
print "</table></center>
</td><td width=\"25%\"><center>
<table border=1>
<tr><th>Rank</th><th>Count</th><th>Word</th></tr>
";
		}
		last if $rank >= 1000;	# only do the top 1000
	}

print '</table></center>
</td></tr></table></center>
<p>Written by <a href="mailto:madmick@yoyo.cc.monash.edu.au">Tom Paton</a>, 1999.</p>
</body>
</html>
';
} else
# output all stats in text format
{
	$rank = 0;
	foreach $word (@keys) {
		$rank++;
		print "$rank\t$count{$word}\t(%";					# count
		printf("%5.2f",($count{$word}/$total)*100.0);		# frequency
		print ")\t$word\n";									# word
	}
}

# DONE.