#!/usr/bin/env python """ Tool that takes python script and runs it. Returns the results embedded in the code in a pdf """ # Copyright: Gael Varoquaux # License: LGPL #TODO: - Output ps, rst, html, moinmoin syntax ? # - Long, long term: use reportlab to avoid the dependencies on # LaTeX DEBUG = False version = "0.1.8" #---------------------------- Imports ---------------------------------------- # To deal with stderr import sys # To deal with regexp import re import os, os.path import popen2 import token, tokenize # to treat StdIn, StdOut as files: import StringIO from docutils import core as docCore from docutils import io as docIO #------------------------------- Subroutines --------------------------------- def DEBUGwrite(variable,filename): """ If DEBUG is enabled, writes variable to the file given by "fillename" """ if DEBUG: f = open(filename,'w') f.write(variable.__repr__()) f.close() def TeX2pdf(fileName, options): """ Compiles a TeX file with pdfLaTeX and cleans up the mess afterwards """ if not options.quiet : print >> sys.stderr, "Compiling document to pdf" fileName=os.path.splitext(fileName)[0] texcmd = "pdflatex --interaction scrollmode %s.tex" % fileName if options.verbose: os.system(texcmd) else: (err_out, stdin) = popen2.popen4(texcmd) while True: line = err_out.readline() if not line: break if not options.quiet : print >> sys.stderr, "Cleaning up" if not DEBUG: os.unlink(fileName+".tex") os.unlink(fileName+".log") os.unlink(fileName+".aux") def epstopdf(figureName): """ Converts eps file generated by the script to a pdf file """ os.environ['GS_OPTIONS'] = "-dUseFlatCompression=true -dPDFSETTINGS=/prepress -sColorImageFilter=FlateEncode -sGrayImageFilter=FlateEncode -dAutoFilterColorImages=false -dAutoFilterGrayImages=false -dEncodeColorImages=false -dEncodeGrayImages=false -dEncodeMonoImages=false" os.system("epstopdf --nocompress " + figureName) os.unlink(figureName) def rmFigures(figureName): """ Removes the pdf files of the figures generated by the script """ os.unlink(figureName[:-4]+".pdf") def rst2latex(restString): """ Calls docutils' engine to convert a rst string to a LaTeX file. """ overrides = {'output_encoding': 'latin1', 'initial_header_level': 0} TeXstring = docCore.publish_string( source=restString, writer_name='latex', settings_overrides=overrides) return TeXstring def py2blocks( fileName, options ): """ Returns the list of blocks to be processed of a given python file, with there starting line number in the original file.""" file=open(fileName) blockList =[] currentBlock = "" startlinenum = 1 linenum = 0 pos = 0 numopenbrakets = 0 numopencurlybrakets = 0 numopenpar = 0 for tokendesc in tokenize.generate_tokens(file.readline): tokentype = token.tok_name[tokendesc[0]] startpos = tokendesc[2][1] tokeninitial = tokendesc[1] tokencontent = tokendesc[1] newline = False if tokendesc[2][0] > linenum: # We just started a new line tokencontent = startpos * " " + tokencontent newline = True elif startpos > pos : tokencontent = (startpos - pos) * " " + tokencontent pos = startpos + len(tokendesc[1]) linenum = tokendesc[2][0] if tokentype == 'OP': if tokeninitial == "{": numopencurlybrakets += 1 elif tokeninitial == "}": numopencurlybrakets += -1 elif tokeninitial == "[": numopenbrakets += 1 elif tokeninitial == "]": numopenbrakets += -1 elif tokeninitial == "(": numopenpar += 1 elif tokeninitial == ")": numopenpar += -1 if numopenbrakets == 0 and numopencurlybrakets == 0 and numopenpar==0 : if (tokentype == 'NEWLINE' or tokentype == 'ENDMARKER'): currentBlock += tokencontent blockList += [[currentBlock,startlinenum],] startlinenum = tokendesc[2][0] + 1 currentBlock = "" pos = 0 elif tokentype == 'COMMENT' : lines = tokencontent.splitlines() if not newline: currentBlock += lines[0] lines = lines[1:] lines = map(lambda z : z + "\n",lines[:]) for line in lines: if line[0:3]=="#!/" and linenum==1: # This is a "#!/foobar" this must be a executable call: currentBlock += tokencontent elif line[0:3]=="#%s " % options.commentchar : blockList+=[ [line[3:], "commentBlock"],] elif line[0:2]=="#%s" % options.commentchar : blockList+=[ [line[2:], "commentBlock"],] elif options.latexliterals and line[0:2]=="#$" : blockList+=[ [line[2:], "latexBlock"],] else: currentBlock += line blockList += [[currentBlock,startlinenum],] startlinenum += 1 currentBlock = "" pos = 0 else: currentBlock += tokencontent else: currentBlock += tokencontent DEBUGwrite(blockList,'pyblock') #blockList = condenseBlocks(blockList) blockList = condenseBlocks(blockList, options) DEBUGwrite(blockList,'pycondensedblock') return blockList def condenseBlocks( blockList, options ): """ Groups the code lines together to form the code blocks""" OutBlockList =[] for block in blockList: if block[0] == '': continue if len(OutBlockList)>0 and not ( block[1] == "commentBlock" or block[1] == "latexBlock" or OutBlockList[-1][1] == "commentBlock" or OutBlockList[-1][1] == "latexBlock" ) and ( block[0][0] == " " or block[0][0:2] == "\n " or block[0][0] == "#" or re.match(r"\n*(elif|else|finally|except)",block[0]) #or OutBlockList[-1][0]=='\n' ): OutBlockList[-1][0] += block[0] else : OutBlockList += [block, ] # Overload sys.argv initText = "\n\nimport sys \nsys.argv = '%s'.split(' ') \n" % options.arguments return [[initText,0],] + OutBlockList class _Show(object): figureList = () def __call__(self): figureName = 'pyreport_%d.eps' % (len(self.figureList) ) from pylab import savefig self.figureList += (figureName, ) print "Here goes figure " + figureName savefig(figureName) try: import pylab pylab.show = _Show() except ImportError: pass def executeBlock( namespace, block, options, figureList): """ Excute a python command block, and returns the locals variables created, the stderr and the stdout generated, and the list of figures generated.""" blockText = "\n\n" + block[0] lineNumber = block[1] outValue = "" ## define a "show" command, to replace pylab's show() #class _Show(object): # def __init__(self, figureList ): # self.figureList = figureList # def __call__(self): # figureName = 'pyreport_%d.eps' % (len(self.figureList) ) # from pylab import savefig # self.figureList += (figureName, ) # print "Here goes figure " + figureName # savefig(figureName) # #show = _Show(figureList) #showname = options.pylabname + "show" #if namespace.has_key(showname): # namespace[showname] = show # create file-like string to capture output codeOut = StringIO.StringIO() codeErr = StringIO.StringIO() try: # capture output and errors sys.stdout = codeOut sys.stderr = codeErr exec blockText in namespace outValue=codeOut.getvalue() errorValue=codeErr.getvalue() # restore stdout and stderr sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ except Exception, inst: print >> sys.stderr, inst sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ errorValue = inst.__str__() print >> sys.stderr, "Error in executing script on line ",lineNumber ,": ", inst if errorValue: print >> sys.stderr, errorValue namespace=globals() namespace.update(locals()) codeOut.close() codeErr.close() if outValue: print outValue if namespace.has_key('pylab'): figureList = pylab.show.figureList else: figureList = () if options.silent: errorValue="" return namespace, outValue, errorValue, figureList def condenseOutputList(outputList): """ Takes the "outputList", made of block of different type and merges successiv blocks of the same type.""" outList=[['commentBlock',''],] for block in outputList: if block[0]==outList[-1][0]: outList[-1][1]+=block[1] if block[0]=='outputBlock': outList[-1][2]+=block[2] outList[-1][1]=re.sub(r"(\n)+",r"\n",outList[-1][1]) else: outList+=[block] return outList def addIndent(lines): """ Add an indentation (4 spaces) to all the lines""" output=re.sub(r"\n","\n ",lines) return output def protect(string): """ Protects all the "\" in a string by adding a second one before """ return re.sub(r"\\",r"\\\\",string) def checkRstBlock(outputList): """ Scans the list of blocks and check if every commentBlock can be compiled as Rest.""" publisher = docCore.Publisher( source_class = docIO.StringInput, destination_class = docIO.StringOutput ) publisher.set_components('standalone', 'restructuredtext', 'pseudoxml') publisher.process_programmatic_settings(None, None, None) for block in outputList: if block[0]=="commentBlock": publisher.set_source(block[1],None) compiledRst = publisher.reader.read(publisher.source, publisher.parser, publisher.settings) if compiledRst.parse_messages: print >> sys.stderr, """Error reading rst on literate comment line *to be added* falling back to plain text""" else: block[0] = "rstBlock" return outputList def RstString(outputList): """ Given a list of blocks being either input blocks, output blocks or error blocks, returns a rst string """ emptyListing=re.compile(r"""\\begin\{lstlisting\}\{\}\s*\\end\{lstlisting\}""",re.DOTALL) inputBlocktpl = r""" .. raw:: LaTeX {\inputBlocksize \lstset{backgroundcolor=\color{lightblue},fillcolor=\color{lightblue},numbers=left,name=pythoncode,firstnumber=%(linenumber)d,xleftmargin=0pt,fillcolor=\color{white},frame=single,fillcolor=\color{lightblue},rulecolor=\color{lightgrey}} \begin{lstlisting}{} %(textBlock)s \end{lstlisting} } """ latexBlocktpl = r""" .. raw:: LaTeX \smallskip\smallskip %s \smallskip\smallskip """ errorBlocktpl = r""" .. raw:: LaTeX {\color{red}{\bfseries Error: } \begin{verbatim}%s\end{verbatim}} """ outputBlocktpl = r""" .. raw:: LaTeX \vskip -1ex \lstset{backgroundcolor=,numbers=none,name=answer,xleftmargin=3ex,frame=none} \begin{lstlisting}{} %s \end{lstlisting} \vskip -1ex """ figuretpl = r''' \end{lstlisting} \\vskip -0.5em \\vskip 0.3cm \\centerline{\includegraphics[scale=0.5]{%s}} \\begin{lstlisting}{}''' outputBlocktpl = r""" .. raw:: LaTeX \lstset{backgroundcolor=,numbers=none,name=answer,xleftmargin=3ex,frame=none} \begin{lstlisting}{} %s \end{lstlisting} """ figuretpl = r''' \end{lstlisting} \\centerline{\includegraphics[scale=0.5]{%s}} \\begin{lstlisting}{}''' commentBlocktpl = r""":: %s""" outString="" for block in outputList: if block[0]=="inputBlock": data = {'linenumber' : block[2], 'textBlock' : addIndent(block[1]), } TeXtext = inputBlocktpl % data TeXtext=re.sub(emptyListing,"",TeXtext) outString += TeXtext elif block[0]=="errorBlock": outString += errorBlocktpl % (addIndent(block[1])) elif block[0]=="latexBlock": outString += latexBlocktpl % (addIndent(block[1])) elif block[0]=="rstBlock": outString += "\n" + block[1] + "\n" elif block[0]=="commentBlock": outString += commentBlocktpl % (addIndent(block[1])) elif block[0]=="outputBlock": if not len(block[2])==0: for file in block[2]: block[1]=re.sub("Here goes figure "+ re.escape(file), (figuretpl % ( os.path.splitext(file)[0] )) , block[1]) TeXtext = outputBlocktpl % (addIndent(block[1])) TeXtext = re.sub(emptyListing,"",TeXtext) outString += TeXtext outString += "\n" return outString #------------------------------- Entry point --------------------------------- def commandlineCall(version): from optparse import OptionParser as OP usage = """usage: %prog [options] pythonfile Processes a python script and pretty prints the results using LateX. If the script uses "show()" commands (from pylab) they are caught by %prog and the resulting graphs are inserted in the output pdf. Comments lines starting with "#!" are interprated as rst lines and pretty printed accordingly in the pdf. By Gael Varoquaux""" parser = OP(usage=usage, version="%prog " + version ) parser.add_option("-o", "--outfile", dest="outfile", help="write report to FILE", metavar="FILE") parser.add_option("-d", "--double", dest="double", action="store_true", default=False, help="compile to two columns per page") parser.add_option("-n", "--nocode", dest="nocode", action="store_true", default=False, help="do not display the source code") parser.add_option("-f", "--figuretype", metavar="TYPE", action="store", type="string", dest="figuretype", default="pdf", help="output figure type TYPE instead of pdf (TYPE can only be ps, for now") parser.add_option("-t", "--type", metavar="TYPE", action="store", type="string", dest="outputtype", default="pdf", help="output to TYPE instead of a pdf, TYPE can only be tex, for now") parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="print all the message, including tex messages") parser.add_option("-q", "--quiet", action="store_true", dest="quiet", default=False, help="don't print status messages to stderr") parser.add_option("-c", "--commentchar", action="store", dest="commentchar", default="!", metavar="CHAR", help='literal comments start with "#CHAR" ') parser.add_option("-e", "--latexescapes", action="store_true", dest="latexescapes", default=False, help='allow LaTeX math mode escape in code wih dollar signs ') parser.add_option("-s", "--silent", dest="silent",action="store_true", default=False, help="""Suppress the display of warning and errors in the report""") #parser.add_option("-p", "--pylabname", # dest="pylabname", # default="", metavar="NAME", # help="""override function NAME.show() in script to catch figures""") parser.add_option("-l", "--latexliterals", action="store_true", dest="latexliterals", default=False, help='allow LaTeX literal comment lines starting with "#$" ') parser.add_option("-a", "--arguments", action="store", dest="arguments", default="", type="string", metavar="ARGS", help='pass the arguments "ARGS" to the script') (options, args) = parser.parse_args() if len(args)==0: parser.print_help() sys.exit(1) if not len(args)==1: print >> sys.stderr, "1 argument: input file" sys.exit(1) if options.arguments: options.arguments = args[0] + ' ' + options.arguments #if not options.pylabname == "": # options.pylabname += "." if options.commentchar=="$" and options.latexliterals: print >> sys.stderr, """The choice of the litteral comment character ($) collides with the caracter used for LaTeX litterals, LaTeX litterals are therefore turned off""" if options.quiet and options.verbose: print >> sys.stderr, """Options quiet and verbose have opposite effect and cancel each other.""" quiet = False verbose = False main(args[0], options) def main(pyFileName,options): if not options.outfile: options.outfile = os.path.splitext(pyFileName)[0]+".pdf" texFileName=os.path.splitext(options.outfile)[0]+".tex" global DEBUG LaTeXpreambule = r""" \usepackage{listings} \usepackage{color} \usepackage{graphicx} \definecolor{darkgreen}{cmyk}{0.7, 0, 1, 0.5} \definecolor{darkblue}{cmyk}{1, 0.8, 0, 0} \definecolor{lightblue}{cmyk}{0.05,0,0,0.05} \definecolor{grey}{cmyk}{0.1,0.1,0.1,1} \definecolor{lightgrey}{cmyk}{0,0,0,0.5} \definecolor{purple}{cmyk}{0.8,1,0,0} \makeatletter \let\@oddfoot\@empty\let\@evenfoot\@empty \def\@evenhead{\thepage\hfil\slshape\leftmark {\rule[-0.11cm]{-\textwidth}{0.03cm} \rule[-0.11cm]{\textwidth}{0.03cm}}} \def\@oddhead{{\slshape\rightmark}\hfil\thepage {\rule[-0.11cm]{-\textwidth}{0.03cm} \rule[-0.11cm]{\textwidth}{0.03cm}}} \let\@mkboth\markboth \markright{{\bf %s }\hskip 3em \today} \def\maketitle{ \centerline{\Large\bfseries\@title} \bigskip } \makeatother \lstset{language=python, extendedchars=true, aboveskip = 0pt, belowskip = 0pt, basicstyle=\ttfamily, keywordstyle=\sffamily\bfseries, identifierstyle=\sffamily, commentstyle=\slshape\color{darkgreen}, stringstyle=\rmfamily\color{blue}, showstringspaces=false, tabsize=4, breaklines=true, numberstyle=\footnotesize\color{grey}, classoffset=1, morekeywords={eyes,zeros,zeros_like,ones,ones_like,array,rand,indentity,mat,vander},keywordstyle=\color{darkblue}, classoffset=2, otherkeywords={[,],=,:},keywordstyle=\color{purple}\bfseries, classoffset=0""" % ( re.sub( "_", r'\\_', pyFileName) ) + options.latexescapes * r""", mathescape=true""" +""" } """ if options.nocode: LaTeXcolumnsep=r""" \setlength\columnseprule{0.4pt} """ else: LaTeXcolumnsep="" LaTeXdoublepage = r""" \usepackage[landscape,left=1.5cm,right=1.1cm,top=1.8cm,bottom=1.2cm]{geometry} \usepackage{multicol} \def\inputBlocksize{\normalsize} \makeatletter \renewcommand\normalsize{% \@setfontsize\normalsize\@ixpt\@xipt% \abovedisplayskip 8\p@ \@plus4\p@ \@minus4\p@ \abovedisplayshortskip \z@ \@plus3\p@ \belowdisplayshortskip 5\p@ \@plus3\p@ \@minus3\p@ \belowdisplayskip \abovedisplayskip \let\@listi\@listI} \normalsize \renewcommand\small{% \@setfontsize\small\@viiipt\@ixpt% \abovedisplayskip 5\p@ \@plus2\p@ \@minus2\p@ \abovedisplayshortskip \z@ \@plus1\p@ \belowdisplayshortskip 3\p@ \@plus\p@ \@minus2\p@ \def\@listi{\leftmargin\leftmargini \topsep 3\p@ \@plus\p@ \@minus\p@ \parsep 2\p@ \@plus\p@ \@minus\p@ \itemsep \parsep}% \belowdisplayskip \abovedisplayskip } \renewcommand\footnotesize{% \@setfontsize\footnotesize\@viipt\@viiipt \abovedisplayskip 4\p@ \@plus2\p@ \@minus2\p@ \abovedisplayshortskip \z@ \@plus1\p@ \belowdisplayshortskip 2.5\p@ \@plus\p@ \@minus\p@ \def\@listi{\leftmargin\leftmargini \topsep 3\p@ \@plus\p@ \@minus\p@ \parsep 2\p@ \@plus\p@ \@minus\p@ \itemsep \parsep}% \belowdisplayskip \abovedisplayskip } \renewcommand\scriptsize{\@setfontsize\scriptsize\@vipt\@viipt} \renewcommand\tiny{\@setfontsize\tiny\@vpt\@vipt} \renewcommand\large{\@setfontsize\large\@xpt\@xiipt} \renewcommand\Large{\@setfontsize\Large\@xipt{13}} \renewcommand\LARGE{\@setfontsize\LARGE\@xiipt{14}} \renewcommand\huge{\@setfontsize\huge\@xivpt{18}} \renewcommand\Huge{\@setfontsize\Huge\@xviipt{22}} \setlength\parindent{14pt} \setlength\smallskipamount{3\p@ \@plus 1\p@ \@minus 1\p@} \setlength\medskipamount{6\p@ \@plus 2\p@ \@minus 2\p@} \setlength\bigskipamount{12\p@ \@plus 4\p@ \@minus 4\p@} \setlength\headheight{12\p@} \setlength\headsep {25\p@} \setlength\topskip {9\p@} \setlength\footskip{30\p@} \setlength\maxdepth{.5\topskip} \makeatother \AtBeginDocument{ \setlength\columnsep{1.1cm} """ + LaTeXcolumnsep + r""" \begin{multicols*}{2} \small} \AtEndDocument{\end{multicols*}} """ if options.double: LaTeXpreambule += LaTeXdoublepage else: LaTeXpreambule += """\usepackage[top=2.1cm,bottom=2.1cm,left=2cm,right=2cm]{geometry} \def\inputBlocksize{\small} """ # Slice the input file into blocks blockList = py2blocks(pyFileName,options) DEBUGwrite( blockList, 'debug1' ) # Process the blocks # Each element of "outputList" is made of a string, describing the # type of the entry, either "inputBlock", or "outputBlock", or # "errorBlock", and the content of the entry. namespace = {} outputList = [] figureList = () if not options.quiet : print >> sys.stderr, "Running python script %s:\n" % pyFileName for block in blockList: if block[1]=="commentBlock": outputList+=[['commentBlock',block[0]],] elif block[1]=="latexBlock": outputList+=[['latexBlock',block[0]],] else: [ namespace, outValue, errorValue, figureList ] = executeBlock(namespace, block, options, figureList) if not options.nocode: outputList+=[['inputBlock', block[0], block[1]], ] if errorValue: outputList+=[['errorBlock', errorValue], ] if outValue: outputList+=[['outputBlock', outValue, figureList], ] DEBUGwrite( outputList, 'debug') # Clean up the output list if not options.nocode: outputList = outputList[1:] outputList=condenseOutputList(outputList) DEBUGwrite( outputList, 'debug2') outputList = checkRstBlock(outputList) DEBUGwrite( outputList, 'debug3') restString = RstString(outputList) DEBUGwrite( restString, texFileName[:-4]+".rst") texString = rst2latex(restString) texString=re.sub(r"\\begin{document}", protect(LaTeXpreambule) + r"\\begin{document}",texString) # Write the tex file texFile = open(texFileName,"w") texFile.write(texString) texFile.close() # Compile it if (not options.figuretype=="ps") or options.outputtype == "pdf": if options.figuretype=="ps": print >> sys.stderr, "Warning: ps figures requested but pdf file output, no figure output" if options.verbose: print >> sys.stderr, "Compiling figures" map(epstopdf,figureList ) if options.outputtype=="pdf": TeX2pdf(texFileName, options) map(rmFigures,figureList) figureList = () if namespace.has_key('pylab'): pylab.show.figureList = () if __name__ == '__main__': commandlineCall(version)