#!/usr/bin/perl

# James: 8/25, 1:17 P
# Bruce: 8/14, 9:33 P


# This is a basic starter template script for processing your xml. There are some
# very basic checks at the beginning that will apply to most xml but you may need
# to alter these checks if your xml deviates too much from the norm.


use utf8;
use strict;
binmode(STDIN, ":utf8");
binmode(STDOUT, ":utf8");


# GLOBALS that may be useful

my $BOOK_ID = "writershelp2e";
my $PUBLISHER = "bsm"; # set this to either "bsm" or "fw"
my $CHAPTER;
my $CHAPTER_TITLE;

my %IDS;

my %SUPPS;
if (!open(SUPP, "<supp_filenames.txt")) {
    print STDERR "Can't read supp_filenames.txt: $!\n";
    exit();
}
binmode(SUPP, ":utf8");
my $supp_line;
while ($supp_line = <SUPP>) {
    chomp($supp_line);
    if ($supp_line =~ /(.*?)::(.*)/) {
	$SUPPS{$1} = $2;
    }
}
close(SUPP);

my %SECS;
if (!open(SEC, "<sec_filenames.txt")) {
    print STDERR "Can't read sec_filenames.txt: $!\n";
    exit();
}
binmode(SEC, ":utf8");
my $sec_line;
while ($sec_line = <SEC>) {
    chomp($sec_line);
    if ($sec_line =~ /(.*?)::(.*)/) {
	$SECS{$1} = $2;
    }
}
close(SEC);

my %TITLES;
if (!open(TITLE, "<title_filenames.txt")) {
    print STDERR "Can't read title_filenames.txt: $!\n";
    exit();
}
binmode(TITLE, ":utf8");
my $title_line;
while ($title_line = <TITLE>) {
    chomp($title_line);
    if ($title_line =~ /(.*?)::(.*)/) {
	$TITLES{$1} = $2;
    }
}
close(TITLE);

my %BOXES;
if (!open(BOX, "<box_filenames.txt")) {
    print STDERR "Can't read box_filenames.txt: $!\n";
    exit();
}
binmode(BOX, ":utf8");
my $box_line;
while ($box_line = <BOX>) {
    chomp($box_line);
    if ($box_line =~ /(.*?)::(.*)/) {
	$BOXES{$1} = $2;
    }
}
close(TITLE);


# Use '_NL_' to specify newlines (such as for question formatting)
my $NEWLINE = "_NL_";

if ($BOOK_ID eq "") {
    print STDERR "Please set the BOOK_ID variable in this script\n";
    exit();
}
if ($PUBLISHER ne "bsm" && $PUBLISHER ne "fw") {
    print STDERR "Please set the PUBLISHER variable in this script to either 'bsm' or 'fw'\n";
    exit();
}

print STDERR "BOOK_ID: $BOOK_ID\n";
print STDERR "PUBLISHER: $PUBLISHER\n";

my $query = 0;

my $contents;
my $temp_line;
while ($temp_line = <STDIN>) {
    chomp($temp_line);

    if ($temp_line =~ /<query /) {
	$query = 1;
    }
    if ($temp_line =~ /<\/query>/) {
	$query = 0;
    }

    if ($query) {
	if ($temp_line =~ /MC:\s+$/) {
	    $temp_line =~ s`MC:\s+`MC: _BLANK_`;
	}
	if ($temp_line =~ /\S/) {
	    $temp_line .= "_NL_";
	}
    }

    $temp_line =~ s`<\?xml [^>]+>``g;
    $temp_line =~ s`<(\/)?manuscript( [^>]+)?>``g;

    $temp_line =~ s/[\t\r\n]//g; #remove tabs, line feeds, returns
    $temp_line =~ s/[ ]{2,}/ /g; # remove multiple spaces
    $temp_line =~ s/^\s+//; # remove leading spaces
    $contents .= $temp_line;
}


# If you want to "remove" content from your xml without actually deleting it from
# the file, wrap it in <rm>...</rm>
# WARNING: do not embed <rm> tags inside of <rm> tags!
$contents =~ s`<rm>.*?</rm>``g;

$_ = $contents;


# At this point, the xml from the file is now one big string with no newlines

# Get chapter info/check for start-numbering-at
s`<chapter ([^>]+)>`"<chapter " . get_chapter_info($1) . ">"`e;

s`<supp>(.*?)</supp>`process_supp($1)`ge;


sub process_supp {
    my $data = shift;

#    print STDERR "process_supp: $data\n";

    my $unique_num = 0;

    # James, we don't use <title num="N"> any more.  This was for something
    # I thought we were going to have to do but we don't.
    ### Don't use
    if ($data =~ /^<title num="(\d+)">(.*?)<\/title>/) {
	$unique_num = $1;
	my $supp_file = $2;
	if (!open(FILE, "<supps/${supp_file}")) {
	    print STDERR "Can't read supp/${supp_file}: $!\n";
	    exit();
	}
	$data = "";
	my $line;
	while ($line = <FILE>) {
	    chomp($line);
	    $data .= $line;
	}
	close(FILE);
    }
    ### Don't use


    if ($data =~ /^<box title="([^"]+)" numbered="false" id="([^"]+)">/) {
	$data =~ s`^<box title="([^"]+)" numbered="false" id="([^"]+)">`<section id="${2}" title="${1}" block_type="supp" level="3">`;
	$data =~ s`</box>$`</section>`;
    }

    if ($data =~ /^<title( id="([^"]+)")?>(.*?)<\/title>/) {
	my $id = $2;
	my $title = $3;

	if ($id !~ /\S/) {
	    if ($unique_num <= 0) {
		print STDERR "Can't find id or unique num for $data\n";
	    }
	    $id = "chart";
	}
	
	$title =~ s`<`{`g;
	$title =~ s`>`}`g;
	$data =~ s`^<title( [^>]+)?>.*?</title>`<section id="${id}" title="${title}" block_type="supp_old" level="3">`;
	
	
	$data =~ s`</P>`</p>`g;
	$data =~ s`<a name="[^"]+"></a>``g;
	
	$data =~ s`<div class="subhead">(.*?)</div>`<p block_type="subhead">$1</p>`g;
	
	$data =~ s`<p [^>]+><img type="display" src="pics/([^"]+)" alt="([^"]*)"/></p>`<p block_type="img"><image src="asset/supp_img/$1" alt="$2"/></p>`g;
	$data =~ s`<p [^>]+><img type="display" src="pics/([^"]+)"\s*/></p>`<p block_type="img"><image src="asset/supp_img/$1"/></p>`g;
	
	
	$data =~ s`<ol class="([^"]+)">`<list type="ordered" block_type="$1">`g;
	$data =~ s`<ol style="[^"]+" class="([^"]+)">`<list type="ordered" block_type="$1">`g;
	$data =~ s`</ol>`</list>`g;
	
	$data =~ s`<ul class="([^"]+)">`<list type="unordered" block_type="$1">`g;
	$data =~ s`</ul>`</list>`g;
	
	$data =~ s`<li>\s+`<li>`g;
	
	$data =~ s`<div [^>]+>``g;
	$data =~ s`</div>``g;
	$data =~ s`</DIV>``g;
	
	$data =~ s`<b>`<strong>`g;
	$data =~ s`</b>`</strong>`g;
	$data =~ s`<i>`<em>`g;
	$data =~ s`</i>`</em>`g;
	
	$data =~ s`<a class='term' href="#" [^>]+>(.*?)</a>`<termref>$1</termref>`g;
	
	$data =~ s`<a name=""></a>``g;
	
	$data =~ s`<span class="[^"]+">`<phrase block_type="${1}">`g;
	$data =~ s`</span>`</phrase>`g;
	
	$data =~ s`<a [^>]+ href="writershelp_ch(\d+_\d+)\.html">(.*?)</a>`<phrase block_type="link" data-file="${1}">$2</phrase>`g;
	
	$data =~ s`<a [^>]+ href="no_file_[^"]+">(.*?)</a>`<phrase block_type="link" data-file="unknown">$1</phrase>`g;
	
	my $p_count = 0;
	
	$data =~ s`(<p[^>]*>)`fix_supp_p($1, $id, $p_count++)`ge;
	
	$data =~ s`</phrase>([a-z])`</phrase> $1`g;

	$data = $data . "</section>";


    }
    

    if ($data =~ /^<pdf id="([^"]+)" title="([^"]+)" file="([^"]+)">/) {
	my $id = $1;
	my $title = $2;
	my $file = $3;
	$data = qq`<section id="${1}" title="${2}" block_type="supp_pdf" level="3"><p id="${id}-p-1"><link href="asset/pdfs/${file}" target="_pop">${title}</link></p></section>`;
	

    }

    if ($data =~ /<pdf num="(\d+)" file="([^"]+)">(.*?)<\/pdf>/) {
	my $num = $1;
	my $file = $2;
	my $title = $3;

	$title =~ s`<`{`g;
	$title =~ s`>`}`g;

	$data = qq`<section id="pdf-supp-ch${CHAPTER}-${num}" title="${title}" block_type="supp_pdf" level="3"><p id="pdf-supp-ch${CHAPTER}-${num}-p-1"><link href="asset/pdfs/${file}" target="_pop">${title}</link></p></section>`;
    }


    $data =~ s`id="([^"]+)"`id="${1}-supp-ch${CHAPTER}-${unique_num}"`g;

    return $data;

}


sub fix_supp_p {
    my $tag = shift;
    my $id = shift;
    my $count = shift;

    if ($tag =~ /<phrase/) {
	return $tag;
    }

    my ($args) = $tag =~ /<p(.*)>/;

    if ($args !~ /id=/) {
	$args = " id=\"${id}-p-${count}\"" . $args;
    }
    $args =~ s`class=`block_type=`;

#    print STDERR "<p${args}>\n";

    return "<p${args}>";
}


# Get chapter info/check for start-numbering-at
#s`<chapter ([^>]+)>`"<chapter " . get_chapter_info($1) . ">"`e;

sub get_chapter_info {
    my $tag = shift;

    ($CHAPTER_TITLE) = $tag =~ /title="([^"]+)"/;
    ($CHAPTER) = $tag =~ /number="(?:ch)?(?:[0]+)?(\d+)"/;

    if ($CHAPTER_TITLE eq "") {
	print STDERR "Can't find chapter title\n";
	exit();
    }
    if ($CHAPTER eq "") {
	print STDERR "Can't find chapter number\n";
	exit();
    }

    # If chapter does not have start-numbering-at attribute then add it
    if ($tag !~ /start-numbering-at/) {
	$tag .= qq` start-numbering-at="${CHAPTER}"`;
    }
    # Otherwise, make sure start-numbering-at is the same as chapter number
    else {
	$tag =~ s`start-numbering-at="[^"]+"`start-numbering-at="${CHAPTER}"`;
    }

    $tag .= " customnumbering=\"1\"";

    return $tag;
}



### Uncomment the following lines if your vendor supplied IDs need fixing
### (i.e. they are not unique enough)
### The fix_id subroutine my also need fixing depending on the format of the IDs

s`id="([^"]+)"`"id=\"" . fix_id($1) . "\""`ge;
#s`refid="([^"]+)"`"refid=\"" . fix_id($1) . "\""`ge;

sub fix_id {
    my $id = shift;

    if ($id !~ /^$BOOK_ID/) {
	return $BOOK_ID . "-" . $id;
    }

    return $id;
}




# BEGINNING THE PROCESS FUNCTIONS
# This script is designed to cycle through all the major elements in the xml file.
# For each element there is a corresponding "process" function where you can add your
# custom code.

# convert level 4s to boxes
s`</section><section (id="[^"]+") [^>]+ title="(Related[^"]+)" [^>]+ level="4">(.*?)</section>`<box $1 block_type="related" title="$2">$3</box></section>`g;

s`</section><section (id="[^"]+") [^>]+ title="(Other steps[^"]+)" [^>]+ level="4">`<rawhtml><h4 $1 class="steps">$2</h4></rawhtml>`g;

s`</section><section (id="[^"]+") [^>]+ title="(Steps[^"]+)" [^>]+ level="4">`<rawhtml><h4 $1 class="steps">$2</h4></rawhtml>`g;


s`</section><section (id="[^"]+") [^>]+ title="([^"]+)" [^>]+ level="33">`<rawhtml><h4 $1 class="dash_blue">$2</h4></rawhtml>`g;


s`</section><box `<box `g;

# "pre-process" sections
# Anything you need to do to a section before you begin processing the elements
# within that section goes in the pre_process_section function
s`<section ([^>]+)>(.*?)</section>`pre_process_section($1, $2)`ge;


# process boxes
# put all of your box processing code in the process_box function
my $BOX_COUNT = 0;
while (/(<box [^>]+>)/) {
    s`<box ([^>]+)>((?:(?!<box ).)*?)</box>`process_box($1, $2)`ge;
    $BOX_COUNT++;
    if ($BOX_COUNT > 1000) {
	print STDERR "Too many boxes ...\n";
	print STDERR $1, "\n";
	exit(0);
    }
}
# these are important, don't remove them
s`<boxer `<box `g;
s`</boxer>`</box>`g;


# process paras
# put all of your para processing code in the process_para function
s`<p ([^>]+)>(.*?)</p>`process_para($1, $2)`ge;


# process figures
# put all of your figure processing code in the process_figure function
s`<figure ([^>]+)>(.*?)</figure>`process_figure($1, $2)`ge;


# process lists
# put all of your list processing code in the process_list function
my $LIST_COUNT = 0;
while (/(<list [^>]+>)/) {
    s`<list ([^>]+)>((?:(?!<list ).)*?)</list>`process_list($1, $2)`ge;
    $LIST_COUNT++;
    if ($LIST_COUNT > 1000) {
	print "Too many lists ...\n";
	exit(0);
    }
}
# these are important, don't remove them
s`<lister `<list `g;
s`</lister>`</list>`g;


# process tables
# put all of your table processing code in the process_table function
my $TABLE_COUNT = 0;
while (/(<table [^>]+>)/) {
    s`<table ([^>]+)>((?:(?!<table ).)*?)</table>`process_table($1, $2)`ge;
    $TABLE_COUNT++;
    if ($TABLE_COUNT > 1000) {
	print "Too many tables ...\n";
	exit(0);
    }
}
# these are important, don't remove them
s`<tabler `<table `g;
s`</tabler>`</table>`g;


# post process section
# Anything you need to do to a section after you have processed the elements
# within that section goes in the post_process_section function
s`<section ([^>]+)>(.*?)</section>`post_process_section($1, $2)`ge;


# ID sanity check
s` id="([^"]+)?"`" id=\"" . check_ID($1) . "\""`ge;

s`&ldquo;`&#8220;`g;
s`&rdquo;`&#8221;`g;
s`&rsquo;`&#8217;`g;
s`&mdash;`&#8212;`g;
s`&ndash;`&#8211;`g;
s`&eacute;`&#233;`g;
s`&hellip;`&#8230;`g;

s`<rawhtml>(.*?)</rawhtml>`"<rawhtml>" . fix_rawhtml($1) . "</rawhtml>"`ge;

s`title="_NONE_"`title=""`g;

s`_BLANK_`&#160;`g;

# add new lines back in to xml
s`(<(rawhtml|chapter|section|box|layout|table|tbody|tr|list|figure|layout|question|image|asset_source)(/| [^>]+)?>)`\1\n`g;
s`(</(metadata|p|div|table|tr|td|list|li|box|question|query|tbody|th|rawhtml|caption|figure)>)`\1\n`g;
s`</section>`</section>\n\n`g;
s`</rawhtml>`\n</rawhtml>`g;
s`</section-metadata>`</section-metadata>\n`g;

# restore newlines
s`${NEWLINE}`\n`g;


### print out the xml ###

print $_, "\n";


sub check_ID {
    my $id = shift;

    if ($id eq "") {
	print STDERR "Empty ID\n";
	exit();
    }

    if (exists($IDS{$id})) {
	print STDERR "Duplicate ID: $id\n";
	exit();
    }

    if ($id !~ /writershelp2e/) {
	print STDERR "Bad ID: $id\n";
	exit();
    }

    $IDS{$id} = 1;

    return $id;
}


sub fix_rawhtml {
    my $text = shift;
    $text =~ s`<`&lt;`g;
    $text =~ s`>`&gt;`g;
    return $text;
}

sub pre_process_section {
    my $args = shift; # attributes on the <section> tag
    my $content = shift; # content of section

    # of course, now they want to get rid of all the "(pop-up window)"
    $args =~ s`\(pop-up windows?\)``;
	
	# add supp block_type to Activity sections
	if ($args =~ /title="Activity \d+:/) {
		$args = $args . " block_type=\"supp\""
	}

    my ($id) = $args =~ /id="([^"]+)"/;

    if ($content =~ /<html>(.*)<\/html>/) {
	$content = process_html($id, $1);
    }

    # process special <pdf> tags to be used in Resources sections
    if ($content =~ /<pdf ([^>]+)>/) {
	my $file = $1;
	my ($id) = $args =~ /id="([^"]+)"/;
	my ($title) = $args =~ /title="([^"]+)"/;
	$title =~ s`{(/?.*?)}`<\1>`g;
	$content =~ s`<pdf [^>]+>`<p id="${id}-pdf"><link href="asset/pdfs/${file}">${title}</link></p>`;
    }
    
    # process special <extlink> tags to be used in Resources sections
    if ($content =~ /<extlink ([^>]+)>/) {
	my $file = $1;
	my ($id) = $args =~ /id="([^"]+)"/;
	my ($title) = $args =~ /title="([^"]+)"/;
	$content =~ s`<extlink ([^>]+)>`<p id="${id}-link"><link href="$1">${title}</link></p>`;
    }
    
    # convert full url pdf links
    $content =~ s`href="http://www\.writershelp\.bedfordstmartins\.com/ebooks/helphandbook/pdfs/`href="asset/pdfs/`g;

    $args =~ s`chapter="[^"]+"`chapter="${CHAPTER}"`;

    if ($CHAPTER != 3 && $CHAPTER != 4 && $CHAPTER != 5 && $CHAPTER != 10 && $CHAPTER != 11 && $CHAPTER != 12 && $CHAPTER != 13 && $CHAPTER != 14 && $CHAPTER != 15 && $CHAPTER != 16 && $CHAPTER != 17 && $CHAPTER != 18 && $CHAPTER != 19 && $CHAPTER != 20 && $CHAPTER != 21 && $CHAPTER != 22 && $CHAPTER != 23) {
	if ($args =~ /title=""/) {
	    $args =~ s`title=""`title="${CHAPTER_TITLE}"`;
	    $args .= " level=\"1\"";
	}
	else {
	    my ($level) = $args =~ /level="(\d)"/;
	    $level++;
	    $args =~ s`level="\d"`level="${level}"`;
	}
    }

    $args =~ s`#em#`{em}`g;
    $args =~ s`#/em#`{/em}`g;

    if ($args =~ /title="EXERCISE (\d+)&#x2013;(\d+)/) {
	my $ex_num = "${1}_${2}";
	$args .= " data-ex-num='${ex_num}'";
	$args =~ s`title="EXERCISE (\d+)&#x2013;(\d+) `title="`;

	my ($title) = $args =~ /title="([^"]+)"/;

	$args =~ s`id="([^"]+)"`id="$1-ex"`;
	$args =~ s` chapter="\d+"``;
	$args =~ s`level="\d"`level="3"`;
	$content =~ s`id="([^"]+)"`id="$1-ex"`g;

	$content =~ s`<p id="[^"]+" block_type="x_n_t"><strong><phrase block_type="orange_text">.*?</p>``g;

	$content =~ s`<rawhtml>&lt;a id="[^"]+"&gt;&lt;/a&gt;</rawhtml>``g;

	$content =~ s`<rawhtml>.*?showExample.*?</rawhtml>`<p id="writershelp2e_ex_link_${ex_num}" block_type="show_example"><link href="asset/ex/ex_${ex_num}.html" target="_pop">Example</link></p>`;

	$content =~ s`<field key="question_report_text">EXERCISE \d+–\d+ .*? - (\d+ of \d+):.*?</field>`<field key="question_report_text">${title} ($1)</field>`g;


    }

    return qq`<section ${args}>${content}</section>`;
}


sub process_html {
    my $id = shift;
    my $data = shift;

    $data =~ s`</P>`</p>`g;
    $data =~ s`<a name="[^"]+"></a>``g;
    $data =~ s`<title></title>``g;
    
    $data =~ s`<div class="subhead"></div>``g;
    $data =~ s`<div class="subhead">(.*?)</div>`<p block_type="subhead">$1</p>`g;
    $data =~ s`<div class="subhead3">(.*?)</DIV>`<p block_type="subhead">$1</p>`g;
    
    $data =~ s`<p><p [^>]+><img type="display" src="pics/([^"]+)" alt="([^"]*)"/></p>`<p block_type="img"><image src="asset/supp_img/$1" alt="$2"/></p>`g;
    $data =~ s`<p><p ><img type="display" src="pics/([^"]+)" alt="([^"]*)"/></p>`<p block_type="img"><image src="asset/supp_img/$1" alt="$2"/></p>`g;
    $data =~ s`<p><p [^>]+><img type="display" src="pics/([^"]+)"\s*/></p>`<p block_type="img"><image src="asset/supp_img/$1"/></p>`g;
    $data =~ s`<p [^>]+><img type="display" src="pics/([^"]+)" alt="([^"]*)"/></p>`<p block_type="img"><image src="asset/supp_img/$1" alt="$2"/></p>`g;
    $data =~ s`<p><img type="display" src="pics/([^"]+)" alt="([^"]*)"/></p>`<p block_type="img"><image src="asset/supp_img/$1" alt="$2"/></p>`g;
    $data =~ s`<p [^>]+><img type="display" src="pics/([^"]+)"\s*/></p>`<p block_type="img"><image src="asset/supp_img/$1"/></p>`g;
    $data =~ s`(<td [^>]+>)<p></td>`\1</td>`g;
    $data =~ s`<b>(<p [^>]+>)(.*?)</b></p>`\1<strong>\2</strong></p>`g;
    
    $data =~ s`<resource [^>]+> <p [^>]+>(<p [^>]+>.*?</p>)`\1`g;
    
    $data =~ s`<br style="clear:both">`<br/>`g;
    $data =~ s`<br>`<br/>`g;
    
    $data =~ s`<TABLE class="([^"]+)" cellspacing="0">`<table block_type="\1"><tbody>`g;
    $data =~ s`<TABLE class="([^"]+)">`<table block_type="\1"><tbody>`g;
    $data =~ s`<TABLE ([^>]+) class="([^"]+)">`<table \1 block_type="\2"><tbody>`g;
    $data =~ s`<TABLE>`<table><tbody>`g;
    $data =~ s`</table>`</tbody></table>`g;
    $data =~ s`(valign="top") valign=top`\1`g;
    $data =~ s`(<td valign="top") id[^>]+>`\1>`g;
    
    $data =~ s`<ol class="([^"]+)">`<list type="ordered" block_type="$1">`g;
    $data =~ s`<ol style="[^"]+" class="([^"]+)">`<list type="ordered" block_type="$1">`g;
    $data =~ s`</ol>`</list>`g;
    
    $data =~ s`<ul class="([^"]+)">`<list type="unordered" block_type="$1">`g;
    $data =~ s`</ul>`</list>`g;
    
    $data =~ s`<li>\s+`<li>`g;
    
    $data =~ s`<div [^>]+>``g;
    $data =~ s`</div>``g;
    $data =~ s`</DIV>``g;
    
    $data =~ s`<b>`<strong>`g;
    $data =~ s`</b>`</strong>`g;
    $data =~ s`<i>`<em>`g;
    $data =~ s`</i>`</em>`g;
    $data =~ s`</emph>`-`g;
    
    $data =~ s`<a class='term' href="#" [^>]+>(.*?)</a>`<termref>$1</termref>`g;
    
    $data =~ s`<a name=""></a>``g;
    
    $data =~ s`<span class="[^"]+">`<phrase block_type="${1}">`g;
    $data =~ s`</span>`</phrase>`g;
    
    $data =~ s`<a [^>]+ href="writershelp_ch(\d+_\d+)\.html">(.*?)</a>`<phrase block_type="link" data-file="${1}">$2</phrase>`g;
    $data =~ s`<a [^>]+ href="(\d+_\d+)\.html">(.*?)</a>`<phrase block_type="link" data-file="${1}">$2</phrase>`g;
    
    $data =~ s`<a [^>]+ href="no_file_[^"]+">(.*?)</a>`<phrase block_type="link" data-file="unknown">$1</phrase>`g;
    
    $data =~ s`<a class="[^"]+" href="(http://[^"]+)" ([^>]+)>(.*?)</a>`<link href="\1" \2>\3</link>`g;
    $data =~ s`<a class="[^"]+" href="([^"]+)" ([^>]+)>(.*?)</a>`<link href="http://\1" \2>\3</link>`g;
    
    my $p_count = 0;
    
    $data =~ s`(<p[^>]*>)`fix_supp_p($1, $id, $p_count++)`ge;
    
    $data =~ s`</phrase>([a-z])`</phrase> $1`g;

    return $data;
}


sub post_process_section {
    my $args = shift; # attributes on <section> tag
    my $content = shift; # content of section

    if ($args =~ /{/) {
	my ($title) = $args =~ /title="([^"]+)"/;
	$title =~ s`{`<`g;
	$title =~ s`}`>`g;
	$content = "<section-metadata><section-title>${title}</section-title></section-metadata>" . $content;
	$args =~ s`title="[^"]+"`title=""`;
    }


    # move term boxes up one para by default
    $content =~ s`(<p [^>]+>(?:(?!<p ).)*?</p>)(<box [^>]+ block_type="term">.*?</box>)`\2\1`g;

    # get rid of pdf icons
    $content =~ s`<image asset-id="[^"]+" alt="image" src="images/pdficon.jpg"/>``g;


    # try to create as many links as we can
    $content =~ s`(<link [^>]+>)(.*?)</link>`process_link($1, $2) . "</link>"`ge;
    $content =~ s`(<p [^>]+>)<phrase block_type="underline">(.*?)</phrase></p>`$1 . process_link("nolinktag", $2) . "</p>"`ge;
    
    $content =~ s`<nobr>`<phrase block_type="nobr">`g;
    $content =~ s`</nobr>`</phrase>`g;
    
    # add <question> and <query> tags when [[Text box]] occurs 
    if ($content =~ /\[\[[tT]ext box\]\]/) {
		my ($title) = $args =~ /title="([^"]+)"/;
		my $q_count = 1;
		$content =~ s`<p id="([^"]+)" [^>]+>\[\[[Tt]ext box\]\]</p>`"<question id=\"$1-question\" numbered=\"false\" title=\"\" number=\"\"><query id=\"$1-query\">ES: &#160;\r_points: 10</query><metadata><field key=\"question_report_text\">${title} - " . $q_count++ . "</field></metadata></question>"`ge;
    	
    }


    return qq`<section ${args}>${content}</section>`;
}


sub process_link {
    my $args = shift;
    my $text = shift;

    $text =~ s`<phrase [^>]+>``g;
    $text =~ s`</phrase>``g;
    $text =~ s`<(/)?strong>``g;

#    print STDERR "processing link: ", $text, "\n";

    my $lc_text = lc($text);

    $args =~ s`href="[^"]+\.xml(#[^"]+)"`href="$1"`;


    if ($args eq "nolinktag") {
#	print STDERR "nolinktag\n";
	if (exists($TITLES{$lc_text})) {
#	    print STDERR "returning: <link href=\"$TITLES{$lc_text}\">${text}</link>\n";
	    return qq`<link href="$TITLES{$lc_text}">${text}</link>`;
	}
    }
    
    # box files
    elsif ($args =~ /data-class="chart"/ && $args =~ /href="(asset\/box\/|box_)(\d+_\d+)/) {

#	print STDERR "box file\n";
	
	my $box_num = $2;

	if (exists($BOXES{$box_num})) {
	    $args =~ s`href="([^"]+)?"`href="$BOXES{$box_num}"`;
#	    print STDERR "in BOXES: $args\n";
		if ($args !~ /target="_/) {
			$args =~ s`(<link [^>]+)>`\1 target="_pop">`g;
		}
            return $args . $text;
	}

        print STDERR "Can't find box file for $box_num\n";
        return $args . $text;
    }
    
    # supp files
    elsif ($args =~ /data-class="chart"/ && $args !~ /data-href-alias/) {

#	print STDERR "supp file\n";

	if (exists($SUPPS{$lc_text})) {
	    $args =~ s`href="([^"]+)?"`href="$SUPPS{$lc_text}"`;
#	    print STDERR "in SUPPS: $args\n";
            return $args . $text;
	}

        if ($lc_text =~ /listing ideas about (.*)/) {
	    $lc_text = "listing ideas: " . $1;
	}
        elsif ($lc_text =~ /(.*?) on (.*)/) {
	    $lc_text = $1 . ": " . $2;
	}

        if (exists($SUPPS{$lc_text})) {
            $args =~ s`href="([^"]+)?"`href="$SUPPS{$lc_text}"`;
            return $args . $text;
        }

        print STDERR "Can't find supp file for $text\n";
        return $args . $text;
    }
    
    # section IDs
    elsif ($args =~ /href="#([^"]+)"/) {
	my $secid = $1;
	if ($secid !~ /^writershelp2e-/) {
	    $secid = "writershelp2e-" . $secid;
	}
	if (exists($SECS{$secid})) {
	    $args =~ s`href="([^"]+)?"`href="$SECS{$secid}"`;
	}
	else {
	    print STDERR "Can't find secid file for $secid\n";
	    $args =~ s`href="([^"]+)?"`href="#"`;
	}
    }
    # shorcut links (<link #ID>)
    elsif ($args =~ /^<link\s+#(\S+)>$/) {
	my $secid = $1;
	if ($secid !~ /^writershelp2e-/) {
	    $secid = "writershelp2e-" . $secid;
	}
	if (exists($SECS{$secid})) {
	    $args = qq`<link href="$SECS{$secid}">`;
	}
	else {
	    print STDERR "Can't find secid file for $secid\n";
	    $args = qq`<link href="#">`;
	}
    }
    # shorcut for chart pop links (<link pop#ID>)
    # this format should only be used to pop up pages in the Resources folder
    elsif ($args =~ /^<link\s+pop#(\S+)>$/) {
	my $secid = $1;
	if ($secid !~ /^writershelp2e-/) {
	    $secid = "writershelp2e-" . $secid;
	}
	if (exists($SECS{$secid})) {
	    $args = qq`<link data-class="chart" href="$SECS{$secid}" target="_pop">`;
	}
	else {
	    print STDERR "Can't find secid file for $secid\n";
	    $args = qq`<link href="#">`;
	}
    }

    if ($args eq "nolinktag") {
	$args = "";
    }

    return $args . $text;
}


sub process_figure {
    my $args = shift; # attributes on <figure> tag
    my $content = shift; # contents of figure

    $content =~ s`src="images/([^"]+)"`src="asset/img_ch${CHAPTER}/$1"`;

    # any image that is centered should be large
    $content =~ s`align="center" width="medium"`align="center" width="large"`;

    # Create supp html files for figures
    my ($img_file_no_ext) = $content =~ /src="(?:[^"]+\/)?(.*?)\.(?:jpg|gif|png|tiff)"/;
    create_supp_file("figure", $img_file_no_ext, $args, $content);

    return qq`<figure ${args}>${content}</figure>`;
}



sub process_list {
    my $args = shift; # attributes on <list> tag
    my $content = shift; # contents of list




    # yes, we are returning "lister"
    return qq`<lister ${args}>${content}</lister>`;
}


sub process_table {
    my $args = shift; # attributes on <table> tag
    my $content = shift; # contents of table

    # process each <td>
    $content =~ s`(<td[^>]*>.*?</td>)`process_td($1)`ge;


    # Create table supp html files for numbered tables.  You may need to adjust
    # the regex for your particular xml.
    if ($content =~ /<table_caption>(TABLE \d+-\d+)/) {
	my $table_file_no_ext = $1;
	$table_file_no_ext = lc($table_file_no_ext);
	$table_file_no_ext =~ s`[ \-\.]`_`g;
	create_supp_file("table", $table_file_no_ext, $args, $content);
    }

    # yes, we are returning "tabler"
    return qq`<tabler ${args}>${content}</tabler>`;
}


sub process_td {
    my $td = shift;


    return $td;
}


sub process_box {
    my $args = shift; #attributes on <box> tag
    my $content = shift; # contents of box

    if ($args !~ /block_type/) {
	$args .= " block_type=\"links\"";
    }


    # They changed the titles on a bunch of pop-ups, so adjust here
    $content =~ s`popupicon.jpg"/>Listing ideas about ([a-zA-Z])`"popupicon.jpg\"/>Listing ideas: " . uc($1)`ge;
    $content =~ s`popupicon.jpg"/>Freewriting on ([a-zA-Z])`"popupicon.jpg\"/>Freewriting: " . uc($1)`ge;
    $content =~ s`popupicon.jpg"/>Focused freewriting on ([a-zA-Z])`"popupicon.jpg\"/>Focused freewriting: " . uc($1)`ge;


    # create alias links for pop up content
    $content =~ s`<phrase block_type="underline"><image [^>]+ src="images/popupicon.jpg"/>(.*?)</phrase></p>`"<link data-class=\"chart\" href=\"\" data-href-alias=\"pop_" . create_alias_string($1) . "\" target=\"_pop\">$1</link></p>"`ge;
    $content =~ s`<image [^>]+ src="images/popupicon.jpg"/><phrase block_type="green"><phrase block_type="underline">(.*?)</phrase></phrase>`"<link data-class=\"chart\" href=\"\" data-href-alias=\"pop_" . create_alias_string($1) . "\" target=\"_pop\">$1</link>"`ge;
    $content =~ s`<image [^>]+ src="images/popupicon.jpg"/>(.*?)</p>`"<link data-class=\"chart\" href=\"\" data-href-alias=\"pop_" . create_alias_string($1) . "\" target=\"_pop\">$1</link></p>"`ge;


    # yes, we are returning "boxer"
    return qq`<boxer ${args}>${content}</boxer>`;
}

sub create_alias_string {
    my $text = shift;
    $text =~ s`\s``g;
    $text =~ s`&#\d+;``g;
    $text = lc($text);
    $text =~ s`<metadata>.*?</metadata>``g;
    $text =~ s`[^a-z0-9]``g;

    return $text;
}

sub process_para {
    my $args = shift; #attributes on <p> tag
    my $content = shift; # contents of para

    #remove <metadata> from paragraphs
    my $pre_data = "";
    if ($content =~ /(<metadata>.*?<\/metadata>)/) {
	$pre_data .= $1;
	$content =~ s`$1``g;
    }


    # we can get rid of the underline inside of the link
    $content =~ s`(<link [^>]+>)<phrase block_type="underline">(.*?)</phrase></link>`$1$2</link>`g;

    return qq`${pre_data}<p ${args}>$content</p>`;
}


# This function will automatically create subdirectories to store the supp
# html files it produces as follows:

# figures: img_chN/ (where 'N' is the chapter number specified in the xml)
# tables: table/

sub create_supp_file {
    my $type = shift; # 'figure' or 'table'
    my $filename = shift; # the name of the supp file we are creating
    my $args = shift; # the attributes of the tag
    my $content = shift; # the content in the tag

    if ($filename eq "" || $filename =~ /\s/) {
	print STDERR "Bad filename for supp file [args: $args]\n";
	return;
    }

    # determine write file
    my $write_file;

    if ($type eq "figure") {
	# if only the filename is passed then add the default sub directory
	if ($filename !~ /\//) {
	    my $sub_dir = "img_ch${CHAPTER}";
	    if (!-d $sub_dir) {
		mkdir($sub_dir);
	    }
	    $write_file = $sub_dir . "/" . $filename;
	    $write_file =~ s`\.([a-z]+)$`.html`;
	    if ($write_file !~ /\.html$/) {
		$write_file .= ".html";
	    }
	}
	# otherwise use the filename passed in
	else {
	    $write_file = $filename;
	}
    }
    elsif ($type eq "table") {
	if ($filename !~ /\//) {
	    my $sub_dir = "table";
	    if (!-d $sub_dir) {
		mkdir($sub_dir);
	    }
	    $write_file = $sub_dir . "/" . $filename;
	    $write_file =~ s`\.([a-z]+)$`.html`;
	    if ($write_file !~ /\.html$/) {
		$write_file .= ".html";
	    }
	}
	else {
	    $write_file = $filename;
	}
    }
    else {
	print STDERR "Bad type ($type) passed to create_supp_file\n";
	return;
    }

    $content =~ s`<link (href="[^"]+")>(.*?)</link>`<span data-type="link" data-$1>$2</span>`g;

    if (!open(OUT, ">${write_file}")) {
	print STDERR "create_supp_file: Can't write to ${write_file}: $!\n";
	return;
    }

    binmode(OUT, ":utf8");

    my $supp_content;

    if ($type eq "figure") {

	my ($block_type) = $args =~ /block_type="([^"]+)"/;
	my ($id) = $args =~ /id="([^"]+)"/;

	$content =~ s`<figure [^>]+>`<div data-type="figure">\n`;
	$content =~ s`<phrase block_type="head">(.*?)</phrase>`<span data-block_type="head">\1</span>`;
	$content =~ s`<asset_source>(.*?)</asset_source>`<div data-type="asset_source">\1</div>\n`;

	# We assume there is only one <caption> per <figure>
	$content =~ s`<caption>(.*?)</caption>`<div data-type="figure_text"><span data-type="caption">\1</span></div>\n`;

	$content =~ s`</figure>`</div>\n`;
	$content =~ s`<layout [^>]+>``;

	$content =~ s`src="(?:[^"]+\/)?(.*\.(?:jpg|gif|png|tiff))"`src="$1"`;

	$content =~ s`<image `<img `;

	$supp_content = qq`<div data-type="figure" data-figure-id="${id}" data-block_type="${block_type}">${content}</div>`;

    }
    elsif ($type eq "table") {

	my ($block_type) = $args =~ /block_type="([^"]+)"/;
	my ($id) = $args =~ /id="([^"]+)"/;
	my ($layout) = $content =~ /<layout(?:[^>]+)?align="([^"]+)"(?:[^>]+)?\/>/;
	my ($width) = $content =~ /<layout(?:[^>]+)?width="([^"]+)"(?:[^>]+)?\/>/;
	my ($border) = $content =~ /<layout(?:[^>]+)?border="([^"]+)"(?:[^>]+)?\/>/;
	my ($number) = $args =~ /number="([^"]+)"/;

	$content =~ s`<layout[^>]+>``;

	$content =~ s`<table_caption>(.*?)</table_caption>`<div data-type="table_text"><span data-type="number">Table ${number}: </span><span data-type="table_caption">$1</span></div>`;

	$supp_content = qq`<div data-type="table" data-block_type="${block_type}" id="${id}" data-layout-align="${layout}" data-layout-width="${width}" data-layout-border="${border}" data-mmsrc="" data-attr=""><table width="auto" height="auto">${content}</table></div>`;
    }

    $supp_content =~ s`<phrase block_type="([^"]+)">`<span data-block_type="\1">`g;
    $supp_content =~ s`</phrase>`</span>`g;

    my $pub_ext = "";
    if ($PUBLISHER eq "fw" || $PUBLISHER eq "FW") {
	$pub_ext = "_fw";
    }

    print OUT<<ENDL;
<!DOCTYPE html>
<!--[if lt IE 7 ]> <html lang="en" class="no-js ie6"> <![endif]-->
<!--[if IE 7 ]>    <html lang="en" class="no-js ie7"> <![endif]-->
<!--[if IE 8 ]>    <html lang="en" class="no-js ie8"> <![endif]-->
<!--[if IE 9 ]>    <html lang="en" class="no-js ie9"> <![endif]-->
<!--[if (gt IE 9)|!(IE)]><!--> <html lang="en" class="no-js"> <!--<![endif]-->
<head>
    <title></title>
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">


<link href="../../css/boilerplate.css" media="screen" rel="stylesheet" type="text/css" >
<link href="../../css/jquery-ui-custom.css" media="screen" rel="stylesheet" type="text/css" >
<link href="../../css/manuscript.css" media="screen" rel="stylesheet" type="text/css" >
<link href="../../css/digfir_ebook${pub_ext}.css" media="screen" rel="stylesheet" type="text/css" >
<link href="../../css/${BOOK_ID}.css" media="screen" rel="stylesheet" type="text/css" >
</head>

<body id="supp_win">
<div id="manuscript" data-chapter-number="${CHAPTER}">
<div data-type="section">

${supp_content}

</div>
</div><!-- /#manuscript -->
<br class="clear" />
<!-- end of material in original view -->

<script type="text/javascript" src="../../js/utilities.js"></script>
<script type="text/javascript" src="../../js/query_types.js"></script>
<script type="text/javascript" src="../../js/player.js"></script>
<script type="text/javascript" src="../../js/jquery.js"></script>
<script type="text/javascript" src="../../js/jquery-ui-1.8.16.custom.min.js"></script>
<script type="text/javascript" src="../../js/jquery_extensions.js"></script>
<script type="text/javascript" src="../../js/swfobject.js"></script>
<script type="text/javascript" src="http://admin.brightcove.com/js/BrightcoveExperiences.js"></script>
<script type="text/javascript" src="../../js/digfir_ebook${pub_ext}.js"></script>
<script type="text/javascript" src="../../js/${BOOK_ID}.js"></script>
<script type="text/javascript">
    //<!--

\$(window).ready(function () {
	player.initialize('52c6e83c757a2ef70c000000');
});
    //-->
</script>

</body>
</html>
ENDL


close(OUT);
} # end create_figure_supp_file

sub qrt_rm_tags {
	my $list_content = shift; #contents of the <list> tag for questions
	
	$list_content =~ s`</li>`\r`g;
	$list_content =~ s`<[^>]+>``g;
	$list_content =~ s`</[^>]+>``g;
	
	return qq`${list_content}`;
}


__END__

(NOTE: Do not put any perl code below.  All perl code should go above __END__)

CHANGE LOG
==========

Bruce: 8/14 9:33P - Added handling of shortcut links (<link #ID> and <link pop#ID>) 
to process_link().
James: 8/16 10:17A - Updated process_html and alias links for pop up content
James: 8/18 9:55A - Updated process_html: more scenarios for <img> tag
James: 8/18 11:50A - Updated post_process_section: convert [[text box]] to quesitons w/ question_report_text
James: 8/18 7:08P - Updated process_html: more scenarios for <img> tag
James: 8/19 9:34A - Updated process_link: added target="_pop" to supp links
James: 8/19 10:33A - Added special character conversion &ndash; and &eacute;
James: 8/19 11:23A - pre_process_section: add "supp" block_type to sections titled "Activity N: ..."
James: 8/19 1:09P - Updated process_html: tables
James: 8/19 5:23P - Updated process_html: added more scenarios for <a> tags
James: 8/19 6:00P - Updated process_html: remove <title> tags
James: 8/19 6:22P - Added special character conversion &mdash; and &hellip;
James: 8/20 12:00P - Updated process_link: Added box_filenames.txt to convert box_n_n hrefs to urls
James: 8/21 11:39A - Updated create_alias_string: remove <metadata> tags and content from string
James: 8/21 3:03P - Updated pre_process_section: convert full url pdf links (http://www.writershelp.bedfordstmartins.com/ebooks/helphandbook/pdfs/..)
James: 8/22 11:05A - Added process special <extlink> tags to be used in Resources sections for sections that call out an external link
James: 8/22 1:10P - Updated process special <pdf> tags to be used in Resources sections to handle {em} tags in title
James: 8/22 3:10P - Updated process_html: more scenarios for <table> tag
James: 8/22 8:27P - Updated process_html: <br> tags (no slash) / scenarios for <p> tags
James: 8/23 12:04P - Updated process_html: scenarios for <table> tags
James: 8/25 11:06A - Updated process_html: scenarios for <div class="subhead"> tags
James: 8/25 1:17P - Updated process_html: convert </emph> to hyphen and remove id from <td> tag