Chapter 3. Building with a bash script

For those of you not familiar with Apache ant, I've also done a script which works for me, under Fedora Linux. YMMV

See the references for a full list of files, but the basics follow the ant idea. It is called build.sh, uses an external file (build.properties.sh) to hold most of the variables (exepting the website user name and password), also a couple of odd functions I've developed. Each part (action) of the ant script is created as a bash function so they can be tested from the bottom of the main script.

@@TODO. Create some sort of case statement which will call up an appropriate function, e.g. when I want to upload, I can call the appropriate function. Done, first pass. Could be refined further.

  $build.sh upload

Something like that. Then a help screen if you don't pass any parameters to the script.

It is also included here (xIncluded) so you can see the ideas. Since this is the actual file used, it contains some very long lines which are wrapped using the \ line terminator. This works in bash version 3.

There are a couple of statements worthy of note for those unfamiliar with bash. The source command includes or imports another bash script, here build.properties.sh and website.properties.sh. These then become accessible to this and any called scripts (see tmp.sh which is created and executed later on).

There are a few helper functions, not very nicely mixed between this file and build.properties.sh. The functions are (hopefully) self evident, I don't want to document them here in case they change. Of note (and something that trips me up regularly) is the defined function, which tests that a variable both is defined and non-empty. Bash isn't very forgiving on empty variables, so I found this necessary. The intent is to test every variable in this manner. Perhaps I could/should have done it once, rather than at each function call, but that's a mod you can put in if the time constraint bothers you.

Dependencies are managed by having a function which calls up a sequence of other functions necessary for them to run, e.g. to build the main html output, I first expand all xIncludes then validate the result. I've commented out the testing sequences I ran. The file is build.sh xIncluded below

#!/bin/bash
# Rev 1.1 2009-05-21T08:51:36Z
# externalized all variable testing - see testvars.sh
# variable definitions
source build.properties.sh
# properties for my website. See website.properties.dummy
source website.properties.sh

#                         functions
# todays date
#
TODAY(){
echo `date -u '+%Y-%m-%dT%H:%M:%S'`
}

#
# Get filename, no extension
#
namename()
{
   name=${1##*/}
   name0="${name%.*}"
   echo "${name0:-$name}"
}

#
# Test if a variable us set, does not test for an empty value
defined()
   {
   if [ ! "${!1-one}" == "${!1-two}" ]
       then
        echo variable ${1} does not exist! Quitting
exit 2
   fi
   : "${!1:? "Variable $1 is empty, aborting."}"
}

basedir=`pwd`
properties=${basedir}/build.properties.sh
#Main input file
main_infile=setup.xml

# ================================================
# Main input file
# ================================================

infile_basename=`namename ${main_infile}`

# name of zip file
zip_filename=${infile_basename}.zip

# ================================================
# assert that a variable is set and is not empty
#
#source http://stackoverflow.com/questions/874389/
#bash-test-for-a-variable-unset-using-a-function
#  ================================================
assertIsSet2() {
    if [[ ! ${!1} && ${!1-_} ]]
then
        echo "$1 is not set, aborting." >&2
        exit 1
    #else
# echo ${1} is set, value [${1}]
    fi
    : "${!1:? "$1 is empty, aborting."}"
}

# ================================================
#Test exit status of last process, quit if non-zero
# ================================================
exitOK(){
if [ $? -ne 0 ]
   then
   echo  ${1} failed, quitting
   exit 2
fi
}

# ================================================
#  Initialisation
# ================================================
init(){
echo Do initialisational things
# Run the python script to generate the testvars.sh
rm -f testvars.sh
touch testvars.sh
python antvars.sh.py -in build.properties.sh -out testvars.sh
# include the created shell script
source testvars.sh
# Now run the function
testvars
echo building on `TODAY`. All variables tested

}

# ================================================
#Clean out the ${out_dir} directory: If needed.
# ================================================
clean_html(){
   if [ ! -d ${out_html_dir} ]
  then
          echo directory ${out_html_dir} does not exist, quitting
          exit 2
    fi
    echo Clean out ${out_html_dir} directory
    rm -v -f ${out_html_dir}/*.html
}

# ================================================
# Clean out the pdf directory
# ================================================
clean_pdf(){
    if [ ! -d ${out_pdf_dir} ]
  then
          echo directory ${out_pdf_dir} does not exist, quitting
          exit 2
    fi

  echo Clean out the ${out_pdf_dir} directory
  rm -v -f ${out_pdf_dir}/*.fo
  rm -v -f ${out_pdf_dir}/*.pdf
}

# ================================================
# Expand any xincludes in the source file
# ================================================
Xinclude(){
echo Expand Xincludes in $main_infile

xmllint -o ${xincluded_file} --xinclude ${basedir}/$main_infile
if [ $? -ne 0 ]
  then
   echo  $xmllint fail, xincludes incorrect
fi
}

# ================================================
# Validate. Depends on Xinclude
# ================================================
Jing() {
    echo Validate using Jing
    # Check variables -->
    java -cp ${jing_classpath} \
-Dorg.apache.xerces.xni.parser.XMLParserConfiguration=org.apache.xerces.parsers.XIncludeParserConfiguration \
com.thaiopensource.relaxng.util.Driver $my_schema_rng ${xincluded_file}
    exitOK Jing
    echo $main_infile is valid
}



# ================================================
#Generate a script to move all used images
# into either html or PDF directory
# ================================================
listimages(){
echo List all images in ${main_infile}
java -cp ${xslt1_processor_classpath}  \
-Dorg.apache.xerces.xni.parser.XMLParserConfiguration=\
org.apache.xerces.parsers.XIncludeParserConfiguration \
-Djavax.xml.parsers.DocumentBuilderFactory=\
org.apache.xerces.jaxp.DocumentBuilderFactoryImpl \
${xslt1_processor} -l -o ${out_dir}/${copy_cmd} \
-x org.apache.xml.resolver.tools.ResolvingXMLReader \
-y org.apache.xml.resolver.tools.ResolvingXMLReader \
-r org.apache.xml.resolver.tools.CatalogResolver  \
${in_dir}/${xincluded_file} ${listimages_sh_stylesheet} ${param_args_post}

exitOK
echo Finished at `TODAY`. See ${out_dir}/${copy_cmd}
chmod +x ${out_dir}/${copy_cmd}

}

#  ================================================
# Copy the images over to the target
#  ================================================
copyimages (){
${copy_cmd} $1
}


# ================================================
# Generic XSLT-processor call (main docbook transform)
# ================================================
# Depends on init, clean, validate, listimages
docbook() {
#echo ${xslt1_processor_classpath}
java -cp ${xslt1_processor_classpath}  \
-Dorg.apache.xerces.xni.parser.XMLParserConfiguration=\
org.apache.xerces.parsers.XIncludeParserConfiguration \
-Djavax.xml.parsers.DocumentBuilderFactory=\
org.apache.xerces.jaxp.DocumentBuilderFactoryImpl \
${xslt1_processor} -l -o ${out_dir}/${main_outfile} \
-x org.apache.xml.resolver.tools.ResolvingXMLReader \
-y org.apache.xml.resolver.tools.ResolvingXMLReader \
-r org.apache.xml.resolver.tools.CatalogResolver  \
${in_dir}/${xincluded_file} ${html_stylesheet} ${param_args_post}

exitOK
echo Finished at `TODAY`

}

#  ================================================
# Process xIncluded source to xsl-fo
#  ================================================
fo()  {  # depends="init, validate, clean_pdf"
echo Generate the fo output
java -cp ${xslt1_processor_classpath} \
-Dorg.apache.xerces.xni.parser.XMLParserConfiguration=\
org.apache.xerces.parsers.XIncludeParserConfiguration \
-Djavax.xml.parsers.DocumentBuilderFactory=\
org.apache.xerces.jaxp.DocumentBuilderFactoryImpl \
${xslt1_processor} \
-x org.apache.xml.resolver.tools.ResolvingXMLReader \
-y org.apache.xml.resolver.tools.ResolvingXMLReader \
-r org.apache.xml.resolver.tools.CatalogResolver -l \
-o ${out_pdf_dir}/${main_fo_outfile}  \
${in_dir}/${xincluded_file} ${main_fo_stylesheet}
exitOK
echo Finished at `TODAY`
}

#  ================================================
# Generate PDF from fo, using XEP
#  ================================================
pdf()  {  #depends="fo"
echo Generate the pdf in $out_pdf_dir from ${out_pdf_dir}/${main_fo_outfile}
#echo cp is ${fo_processor_classpath}
java -cp ${fo_processor_classpath}  \
-Dcom.renderx.xep.CONFIG=${xephome}/xep.xml \
${fo_processor_class} -quiet \
-fo ${out_pdf_dir}/${main_fo_outfile} \
-pdf ${out_pdf_dir}/${infile_basename}.pdf
echo Finished at `TODAY`

}

# ================================================
#                zip task. Zip up a list of files
# ================================================
zip(){
rm -rf ${zip_src_directory}
mkdir ${zip_src_directory}
echo creating ${zip_filename} from ${infile_basename}

#zip  <zipfile> file file file # zips up a directory into zipfile
# -f freshen existing zip
# -r recurse into directories
# -u update as determined by dates
echo -n zip  ${zip_src_directory}/${zip_filename} > tmp.sh
echo -n " " >> tmp.sh
for f in ${zip_file_list}
  do
   echo -n $f >> tmp.sh
   echo -n " " >> tmp.sh
   done

cat tmp.sh
chmod +x tmp.sh
#Now run the zip command
echo Creating ${zip_src_directory}/${zip_filename}
tmp.sh

rm -f tmp.sh
# List it if needed
#unzip -l ${zip_src_directory}/${zip_filename}

}

#================================================
#               upload    all files to website, ftp
#================================================

uploadhtml ()
{

echo sending from ${out_html_dir} to ${myroot}
ftp -in  ${website_tgt} <<ENDFTP
verbose
user ${user_id} ${user_password}
cd  ${froot}
cd ${ftp_tgt}
lcd ${out_html_dir}
mput *.html
mput *.css
mkdir graphics
mput graphics/*.*
bye
ENDFTP

}

#
# Validate. Depends on Xinclude
#
validate (){
Xinclude
Jing
}

#
#Docbook with all dependencies
#
docbookGroup(){
init
clean_html
validate
listimages
copyimages ${local_graphics_html_tgt}
docbook
}

pdfGroup(){
init
clean_pdf
validate
listimages
copyimages ${local_graphics_pdf_tgt}
fo
pdf
}

#
# Basic Show help function
#
usage(){
echo Usage:

echo "validate     - Validate the source against docbook v5"
echo "docbook      - Produce an html output of the source"
echo "pdf          - Produce a PDF output of the source"
echo "zip          - Zip up the associated working files"
echo "upload html|pdf - Upload to the website"
echo "help         - Produce this help message"
exit 2
}

#
#  Handle command line parameters
#

if [ $# -lt 1 ]
    then
    usage
    exit 2
fi
echo $1 is "[${1}]"
case $1 in
    init)
init
;;
    (validate|valid)
       validate
       ;;

    (docbook|docbookGroup|html)
       docbookGroup
       ;;

    ("pdf|pdfGroup|print")
       pdfGroup
       ;;

    zip)
       zip
       ;;

    upload)
       if [ $# = 2 ]
   then
   if [ $2 = "html" ]
       then
       uploadhtml
   else
       if [ $2 = "pdf" ]
   then
   uploadpdf
       else
   echo "Invalid parameter $2"
   exit 2
       fi
   fi
       else
   echo "Second Parameter needed to upload,"
   echo "Either html or pdf. Quitting"
   exit 2
       fi
       ;;

    *)
help
exit 2

esac

#zip
#upload

#pdfGroup
#bname=`namename ${in_dir}/docbook.fo.xsl`
#echo $bname

#clean_html
#'clean_pdf
#defined xmllint

#docbookGroup
#assertIsSet2 docbookX
#assertIsSet2 docbook

#x=
#defined x
#
#echo \$docbook has value $docbook
#assertIsSet2 docbook
#assertIsSet2 x
#defined docbook
#defined x

@TODO As per the ant script, one action which would tidy this up is to collate the variable checking into a single place. I made this harder by using two forms of variables (using underscore and period to isolate words in variable names).

That's it folks. The included file build.properties.sh (holding the variables) is below

#properties file for generic docbook ant script.
#Avoids all build specific values!

#Version 1.0
#Date    2009-05-17T06:30:00Z
#Author  Dave Pawson
#

#Version 1.1
#Date: 2009-05-28T07:37:24Z
#Author Dave Pawson
# Adopted for use with ant as well, for common defs

TODAY_UK=
fintim=

# docbook location: Everything taken relative to this for docbook stuff
docbook=/sgml/docbook
# Main stylesheets
stylesheets=${docbook}/docbook-xsl-ns
html_stylesheets=${stylesheets}/html
fo_stylesheets=${stylesheets}/fo

# Main Docbook stylesheet is imported via my customization layer
# (docbook.html.xsl)

html_stylesheet=docbook.html.xsl

# saxon extensions which come with docbook
dbsaxonextensions=${stylesheets}/extensions/saxon65.jar

# Docbook v5 Schema
db_schema_rng=$docbook/v5/rng/docbookxi.rng
my_schema_rng=poem.rng

#
# Working files, used for zip and upload, see ${zip_file_list}
#
#
#
docbook_html_xsl=docbook.html.xsl
poem_rng=poem.rng
docbook_fo_xsl=docbook.fo.xsl
listimages_sh_xsl=listimages.sh.xsl
dpawson_css=dpawson.css
build_properties=build.properties
build_properties_sh=build.properties.sh
build_sh=build.sh
build_xml=build.xml
cp_images_sh=cp.images.sh
docbook_html_xsl=docbook.html.xsl
listimages_sh_xsl=listimages.sh.xsl
listimages_stylesheet=listimages.xsl
imagelist_anttask=copyimages.xml
test_docbook_xml=test.docbook.xml
testvars_xsl=testvars.xsl
testprops_xsl=testprops.xsl
antvars_py=antvars.py
antvars_sh_py=antvars.sh.py
example_catalog=db5.catalog.xml
catalog_manager=CatalogManager.properties

# Input properties:

basedir=`pwd`

# all xml files should be in this directry
in_dir=${basedir}


# source file for doLayout target
autolayout_infile=newlayout.xml
# source file for website transform on second pass
website_infile=autolayout.xml

# Output Properties: Output directory
out_dir=${basedir}


out_html_dir=${out_dir}/html
# Main output file used for docbook html transform
main_outfile=${infile_basename}.html

# fo output
out_pdf_dir=${out_dir}/pdf
out_pdf_dir=${out_dir}/pdf
main_fo_stylesheet=${in_dir}/docbook.fo.xsl
main_fo_outfile=test.fo

#pdf output
main_pdf_outfile=${infile_basename}.pdf


# global Styles directory, on disk and on website
global_styles=/styles

#local graphics source
local_graphics_src=${in_dir}/graphics

#local graphics target - html
local_graphics_html_tgt=${out_html_dir}/graphics

#local graphics target - pdf
local_graphics_pdf_tgt=${out_pdf_dir}/graphics

# task to copy images to local target

#XSLT stylesheet used to generate copy command (html and fo)
listimages_sh_stylesheet=listimages.sh.xsl

# copy command - used for both html and fo
copy_cmd=cp.images.sh

#CSS stylesheet
css_stylesheet=dpawson.css

# Java locations

java=/myjava

# zip file build target directory
zip_src_directory=${out_dir}/upload
# zip filename - define in bash.sh or build.xml

# Post XSLT transform parameter. Leave as is for Saxon
param_args_post="saxon.extensions=1"

# XSLT  properties
xslt1_processor=com.icl.saxon.StyleSheet
xslt2_processor=net.sf.saxon.Transform
xslt1_jar=saxon655.jar
xslt2_jar=saxon9.jar

# xsl-fo processing
fo_processor_class=com.renderx.xep.XSLDriver
xephome=$java/xep

# Catalog resolver
resolver_jar=resolver.jar
xmlcatalog=/sgml/catalog.xml

# XML parser
xerces_jar=xerces.jar
xerces_impl_jar=xercesImpl.jar
xml_apis_jar=xml-api
#xerces api
xerces_api_jar=xml-apis.jar

#xinclude processor
xmllint=/usr/bin/xmllint

# Temporary xincluded file
xincluded_file=tmp.xml

#Jing parser
jing_jar=jing.jar

#zip file list of required files
zip_file_list="${docbook_html_xsl}  ${poem_rng}  ${docbook_fo_xsl}  \
${listimages_sh_xsl}  ${dpawson_css}  ${xmlcatalog}   \
${build_properties_sh}  ${build_sh}  ${build_xml}  ${cp_images_sh}  \
${docbook_html_xsl}  ${listimages_sh_xsl}  ${listimages_stylesheet}  \
${test_docbook_xml} ${antvars_py} ${antvars_sh_py} ${testvars_xsl} \
${example_catalog} ${catalog_manager}"

# local directory for uploading
uploaddir=${basedir}/upload

#classpath for Jing, relax NG validator from James Clark
jing_classpath=${java}/jing.jar:${java}/${xslt1_jar}:$java/${xerces_jar}:\
$java/${xerces_api_jar}

# xslt 1 class
xslt1_processor_class=com.icl.saxon.StyleSheet
xslt2_processor=net.sf.saxon.Transform

# path for xslt processor.
#Includes resolver and extensions and catalogManager.properties file.
xslt1_processor_classpath=\
${java}/${xslt1_jar}:\
${java}/${resolver_jar}:\
${java}/${xerces_jar}:\
${java}/${xerces_impl_jar}\
${java}/${xml_apis_jar}\
${dbsaxonextensions}:\
/sgml

# path for xslt 2.processor.
#Includes resolver and extensions and catalogManager.properties file.  -->
xslt2_processor_classpath=$java/saxon9.jar:$java/resolver.jar:\
${dbsaxonextensions}:$java/xercesImpl.jar:/sgml:.

#Location of the XML catalog, see
#http://www.asios-open.org/committees/entity/spec-2001-08-06.html
catalogpath=$xmlcatalog

# Paths for xsl-fo processing
fo_processor_classpath=/usr/java/default/lib/tools.jar:\
$java/xep/lib/xep.jar:$xephome/lib/saxon.jar:$xephome/lib/xt.jar