#!/usr/bin/env python import os, difflib, time, gc, codecs, platform, sys from pprint import pprint import textwrap # Setup a logger manually for compatibility with Python 2.3 import logging logging.getLogger('MARKDOWN').addHandler(logging.StreamHandler()) import markdown TEST_DIR = "tests" TMP_DIR = "./tmp/" WRITE_BENCHMARK = True WRITE_BENCHMARK = False ACTUALLY_MEASURE_MEMORY = True ###################################################################### if platform.system().lower() == "darwin": # Darwin _proc_status = '/proc/%d/stat' % os.getpid() else: # Linux _proc_status = '/proc/%d/status' % os.getpid() _scale = {'kB': 1024.0, 'mB': 1024.0*1024.0, 'KB': 1024.0, 'MB': 1024.0*1024.0} def _VmB(VmKey): '''Private. ''' global _proc_status, _scale # get pseudo file /proc/<pid>/status try: t = open(_proc_status) v = t.read() t.close() except: return 0.0 # non-Linux? # get VmKey line e.g. 'VmRSS: 9999 kB\n ...' i = v.index(VmKey) v = v[i:].split(None, 3) # whitespace if len(v) < 3: return 0.0 # invalid format? # convert Vm value to bytes return float(v[1]) * _scale[v[2]] def memory(since=0.0): '''Return memory usage in bytes. ''' if ACTUALLY_MEASURE_MEMORY : return _VmB('VmSize:') - since def resident(since=0.0): '''Return resident memory usage in bytes. ''' return _VmB('VmRSS:') - since def stacksize(since=0.0): '''Return stack size in bytes. ''' return _VmB('VmStk:') - since ############################################################ DIFF_FILE_TEMPLATE = """ <html> <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <style> td { padding-left: 10px; padding-right: 10px; } colgroup { margin: 10px; } .diff_header { color: gray; } .ok { color: green; } .gray { color: gray; } .failed a { color: red; } .failed { color: red; } </style> </head> <body> <h1>Results Summary</h1> <table rules="groups" > <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup> <th> <td></td> <td>Seconds</td> <td></td> <td>Memory</td> </th> <tbody> """ FOOTER = """ </body> </html> """ DIFF_TABLE_TEMPLATE = """ <table class="diff" rules="groups" > <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup> <th> <td></td> <td>Expected</td> <td></td> <td></td> <td>Actual</td> </th> <tbody> %s </tbody> </table> """ def smart_split(text) : result = [] for x in text.splitlines() : for y in textwrap.wrap(textwrap.dedent(x), 40): result.append(y) return result differ = difflib.Differ() try : htmldiff = difflib.HtmlDiff() except: htmldiff = None class TestRunner : def __init__ (self) : self.failedTests = [] if not os.path.exists(TMP_DIR): os.mkdir(TMP_DIR) def test_directory(self, dir, measure_time=False, safe_mode=False, encoding="utf-8", output_format='xhtml1') : self.encoding = encoding benchmark_file_name = os.path.join(dir, "benchmark.dat") self.saved_benchmarks = {} if measure_time : if os.path.exists(benchmark_file_name) : file = open(benchmark_file_name) for line in file.readlines() : test, str_time, str_mem = line.strip().split(":") self.saved_benchmarks[test] = (float(str_time), float(str_mem)) repeat = range(10) else : repeat = (0,) # First, determine from the name of the directory if any extensions # need to be loaded. parts = os.path.split(dir)[-1].split("-x-") if len(parts) > 1 : extensions = parts[1].split("-") print extensions else : extensions = [] mem = memory() start = time.clock() self.md = markdown.Markdown(extensions=extensions, safe_mode = safe_mode, output_format=output_format) construction_time = time.clock() - start construction_mem = memory(mem) self.benchmark_buffer = "construction:%f:%f\n" % (construction_time, construction_mem) html_diff_file_path = os.path.join(TMP_DIR, os.path.split(dir)[-1]) + ".html" self.html_diff_file = codecs.open(html_diff_file_path, "w", encoding=encoding) self.html_diff_file.write(DIFF_FILE_TEMPLATE) self.diffs_buffer = "" tests = [x.replace(".txt", "") for x in os.listdir(dir) if x.endswith(".txt")] tests.sort() for test in tests : self.run_test(dir, test, repeat) self.html_diff_file.write("</table>") if sys.version < "3.0": self.html_diff_file.write(self.diffs_buffer.decode("utf-8")) self.html_diff_file.write(FOOTER) self.html_diff_file.close() print "Diff written to %s" % html_diff_file_path benchmark_output_file_name = benchmark_file_name if not WRITE_BENCHMARK: benchmark_output_file_name += ".tmp" self.benchmark_file = open(benchmark_output_file_name, "w") self.benchmark_file.write(self.benchmark_buffer) self.benchmark_file.close() #################### def run_test(self, dir, test, repeat): print "--- %s ---" % test self.html_diff_file.write("<tr><td>%s</td>" % test) input_file = os.path.join(dir, test + ".txt") output_file = os.path.join(dir, test + ".html") expected_output = codecs.open(output_file, encoding=self.encoding).read() input = codecs.open(input_file, encoding=self.encoding).read() actual_output = "" actual_lines = [] self.md.source = "" gc.collect() mem = memory() start = time.clock() for x in repeat: actual_output = self.md.convert(input) conversion_time = time.clock() - start conversion_mem = memory(mem) self.md.reset() expected_lines = [x.encode("utf-8") for x in smart_split(expected_output)] actual_lines = [x.encode("utf-8") for x in smart_split(actual_output)] #diff = difflib.ndiff(expected_output.split("\n"), # actual_output.split("\n")) diff = [x for x in differ.compare(expected_lines, actual_lines) if not x.startswith(" ")] if not diff: self.html_diff_file.write("<td class='ok'>OK</td>") else : self.failedTests.append(test) self.html_diff_file.write("<td class='failed'>" + "<a href='#diff-%s'>FAILED</a></td>" % test) print "MISMATCH on %s/%s.txt" % (dir, test) print for line in diff : print line if htmldiff!=None : htmlDiff = htmldiff.make_table(expected_lines, actual_lines, context=True) htmlDiff = "\n".join( [x for x in htmlDiff.splitlines() if x.strip().startswith("<tr>")] ) self.diffs_buffer += "<a name='diff-%s'/><h2>%s</h2>" % (test, test) self.diffs_buffer += DIFF_TABLE_TEMPLATE % htmlDiff expected_time, expected_mem = self.saved_benchmarks.get(test, ("na", "na")) self.html_diff_file.write(get_benchmark_html(conversion_time, expected_time)) self.html_diff_file.write(get_benchmark_html(conversion_mem, expected_mem)) self.html_diff_file.write("</tr>\n") self.benchmark_buffer += "%s:%f:%f\n" % (test, conversion_time, conversion_mem) def get_benchmark_html (actual, expected) : buffer = "" if not expected == "na": if actual > expected * 1.5: tdiff = "failed" elif actual * 1.5 < expected : tdiff = "ok" else : tdiff = "same" if ( (actual <= 0 and expected < 0.015) or (expected <= 0 and actual < 0.015)) : tdiff = "same" else : tdiff = "same" buffer += "<td class='%s'>%.2f</td>" % (tdiff, actual) if not expected == "na": buffer += "<td class='gray'>%.2f</td>" % (expected) return buffer def run_tests() : tester = TestRunner() #test.test_directory("tests/basic") tester.test_directory("tests/markdown-test", measure_time=True) tester.test_directory("tests/misc", measure_time=True) tester.test_directory("tests/extensions-x-tables") tester.test_directory("tests/extensions-x-footnotes") #tester.test_directory("tests/extensions-x-ext1-ext2") tester.test_directory("tests/safe_mode", measure_time=True, safe_mode="escape") tester.test_directory("tests/extensions-x-wikilinks") tester.test_directory("tests/extensions-x-toc") tester.test_directory("tests/extensions-x-def_list") tester.test_directory("tests/extensions-x-abbr") tester.test_directory("tests/html4", output_format='html4') try: import pygments except ImportError: # Dependancy not avalable - skip test pass else: tester.test_directory("tests/extensions-x-codehilite") print "\n### Final result ###" if len(tester.failedTests): print "%d failed tests: %s" % (len(tester.failedTests), str(tester.failedTests)) else: print "All tests passed, no errors!" run_tests()