beremiz

Tests: add HTML report generation and a workaround to bad exception handling in sikuli.
wxPython4
2022-04-07, Edouard Tisserant
65c5f66e9298
Parents de8cc85b688a
Children 523f6fcc7a28
Tests: add HTML report generation and a workaround to bad exception handling in sikuli.

In case of exception in python code, and since a thread is running
to observe stdout, sikuli was never terminated after an exception.
Unfortunately sys.exepthook doesn't work in that version of jython/sikuli.
Test are now written inside functions witch are passed to run_test to deal
with exception.
4 files changed, 255 insertions(+), 194 deletions(-)
  • +28 -35
    tests/ide_tests/edit_project.sikuli/edit_project.py
  • +117 -123
    tests/ide_tests/new_project.sikuli/new_project.py
  • +25 -30
    tests/ide_tests/run_python_exemple.sikuli/run_python_exemple.py
  • +85 -6
    tests/ide_tests/sikuliberemiz.py
    • --- a/tests/ide_tests/edit_project.sikuli/edit_project.py Tue Mar 29 08:50:01 2022 +0200
    +++ b/tests/ide_tests/edit_project.sikuli/edit_project.py Thu Apr 07 07:40:32 2022 +0200
    @@ -9,62 +9,55 @@
    addImportPath(os.path.dirname(getBundlePath()))
    # common test definitions module
    -from sikuliberemiz import *
    +from sikuliberemiz import run_test
    -# Start the app
    -app = BeremizApp(exemple="python")
    +def test(app):
    +
    + app.doubleClick("1646062660770.png")
    -app.doubleClick("1646062660770.png")
    -
    -app.WaitIdleUI()
    + app.WaitIdleUI()
    -app.click("example")
    + app.click("example")
    -app.WaitIdleUI()
    + app.WaitIdleUI()
    -app.type(Key.DOWN * 10, Key.CTRL)
    -
    -app.WaitIdleUI()
    + app.type(Key.DOWN * 10, Key.CTRL)
    -app.doubleClick("1646066996620.png")
    + app.WaitIdleUI()
    -app.WaitIdleUI()
    + app.doubleClick("1646066996620.png")
    -app.type(Key.TAB*3) # select text content
    + app.WaitIdleUI()
    -app.type("'sys.stdout.write(\"EDIT TEST OK\\n\")'")
    + app.type(Key.TAB*3) # select text content
    -app.type(Key.ENTER)
    + app.type("'sys.stdout.write(\"EDIT TEST OK\\n\")'")
    -app.WaitIdleUI()
    + app.type(Key.ENTER)
    -app.k.Save()
    + app.WaitIdleUI()
    -app.k.Clean()
    + app.k.Save()
    -app.waitForChangeAndIdleStdout()
    + app.k.Clean()
    -app.k.Build()
    -
    -app.waitForChangeAndIdleStdout()
    + app.waitForChangeAndIdleStdout()
    -app.k.Connect()
    + app.k.Build()
    -app.waitForChangeAndIdleStdout()
    + app.waitForChangeAndIdleStdout()
    -app.k.Transfer()
    + app.k.Connect()
    -app.waitForChangeAndIdleStdout()
    + app.waitForChangeAndIdleStdout()
    -app.k.Run()
    + app.k.Transfer()
    -# wait 10 seconds for 10 patterns
    -found = app.waitPatternInStdout("EDIT TEST OK", 10)
    + app.waitForChangeAndIdleStdout()
    -app.close()
    + app.k.Run()
    -if found:
    - exit(0)
    -else:
    - exit(1)
    + # wait 10 seconds for 10 patterns
    + return app.waitPatternInStdout("EDIT TEST OK", 10)
    +run_test(test, exemple="python")
    --- a/tests/ide_tests/new_project.sikuli/new_project.py Tue Mar 29 08:50:01 2022 +0200
    +++ b/tests/ide_tests/new_project.sikuli/new_project.py Thu Apr 07 07:40:32 2022 +0200
    @@ -11,128 +11,122 @@
    # common test definitions module
    from sikuliberemiz import *
    -# Start the app without any project given
    -app = BeremizApp()
    -
    -new_project_path = os.path.join(os.path.abspath(os.path.curdir), "new_test_project")
    -
    -# New project path must exist (usually created in directory selection dialog)
    -os.mkdir(new_project_path)
    -
    -app.WaitIdleUI()
    -
    -# Create new project (opens new project directory selection dialog)
    -app.k.New()
    -
    -app.WaitIdleUI()
    -
    -# Move to "Home" section of file selecor, otherwise address is
    -# "file ignored" at first run
    -app.type("f", Key.CTRL)
    -app.type(Key.ESC)
    -app.type(Key.TAB)
    -
    -# Enter directory by name
    -app.k.Address()
    -
    -# Fill address bar
    -app.type(new_project_path + Key.ENTER)
    -
    -app.WaitIdleUI()
    -
    -# When prompted for creating first program select type ST
    -app.type(Key.TAB*4) # go to lang dropdown
    -app.type(Key.DOWN*2) # change selected language
    -app.type(Key.ENTER) # validate
    -
    -app.WaitIdleUI()
    -
    -# Name created program
    -app.type("Test program")
    -
    -app.WaitIdleUI()
    -
    -# Focus on Variable grid
    -app.type(Key.TAB*4)
    -
    -# Add 2 variables
    -app.type(Key.ADD*2)
    -
    -# Focus on ST text
    -app.WaitIdleUI()
    -
    -app.type(Key.TAB*8)
    -
    -app.type("""\
    -LocalVar0 := LocalVar1;
    -{printf("Test OK\\n");fflush(stdout);}
    -""")
    -
    -app.k.Save()
    -
    -# Close ST POU
    -app.type("w", Key.CTRL)
    +def test(app):
    +
    + new_project_path = os.path.join(os.path.abspath(os.path.curdir), "new_test_project")
    +
    + # New project path must exist (usually created in directory selection dialog)
    + os.mkdir(new_project_path)
    +
    + app.WaitIdleUI()
    +
    + # Create new project (opens new project directory selection dialog)
    + app.k.New()
    +
    + app.WaitIdleUI()
    +
    + # Move to "Home" section of file selecor, otherwise address is
    + # "file ignored" at first run
    + app.type("f", Key.CTRL)
    + app.type(Key.ESC)
    + app.type(Key.TAB)
    +
    + # Enter directory by name
    + app.k.Address()
    +
    + # Fill address bar
    + app.type(new_project_path + Key.ENTER)
    +
    + app.WaitIdleUI()
    +
    + # When prompted for creating first program select type ST
    + app.type(Key.TAB*4) # go to lang dropdown
    + app.type(Key.DOWN*2) # change selected language
    + app.type(Key.ENTER) # validate
    +
    + app.WaitIdleUI()
    +
    + # Name created program
    + app.type("Test program")
    +
    + app.WaitIdleUI()
    +
    + # Focus on Variable grid
    + app.type(Key.TAB*4)
    +
    + # Add 2 variables
    + app.type(Key.ADD*2)
    +
    + # Focus on ST text
    + app.WaitIdleUI()
    +
    + app.type(Key.TAB*8)
    +
    + app.type("""\
    + LocalVar0 := LocalVar1;
    + {printf("Test OK\\n");fflush(stdout);}
    + """)
    +
    + app.k.Save()
    +
    + # Close ST POU
    + app.type("w", Key.CTRL)
    +
    + app.WaitIdleUI()
    +
    + # Focus project tree and select root item
    + app.type(Key.TAB)
    +
    + app.type(Key.LEFT)
    +
    + app.type(Key.UP)
    +
    + # Edit root item
    + app.type(Key.ENTER)
    +
    + app.WaitIdleUI()
    +
    + # Switch to config tab
    + app.type(Key.RIGHT*2)
    +
    + # Focus on URI
    + app.type(Key.TAB)
    +
    + # Set URI
    + app.type("LOCAL://")
    +
    + # FIXME: Select other field to ensure URI is validated
    + app.type(Key.TAB)
    +
    + app.k.Save()
    +
    + # Close project config editor
    + app.type("w", Key.CTRL)
    +
    + app.WaitIdleUI()
    +
    + # Focus seems undefined at that time (FIXME)
    + # Force focussing on "something" so that next shortcut is taken
    + app.type(Key.TAB)
    +
    + app.waitIdleStdout()
    +
    + app.k.Build()
    +
    + app.waitIdleStdout(5,30)
    +
    + app.k.Connect()
    +
    + app.waitIdleStdout()
    +
    + app.k.Transfer()
    +
    + app.waitIdleStdout()
    +
    + app.k.Run()
    +
    + # wait 10 seconds
    + return app.waitPatternInStdout("Test OK", 10)
    -app.WaitIdleUI()
    -# Focus project tree and select root item
    -app.type(Key.TAB)
    -
    -app.type(Key.LEFT)
    -
    -app.type(Key.UP)
    -
    -# Edit root item
    -app.type(Key.ENTER)
    -
    -app.WaitIdleUI()
    -
    -# Switch to config tab
    -app.type(Key.RIGHT*2)
    -
    -# Focus on URI
    -app.type(Key.TAB)
    -
    -# Set URI
    -app.type("LOCAL://")
    -
    -# FIXME: Select other field to ensure URI is validated
    -app.type(Key.TAB)
    -
    -app.k.Save()
    -
    -# Close project config editor
    -app.type("w", Key.CTRL)
    -
    -app.WaitIdleUI()
    -
    -# Focus seems undefined at that time (FIXME)
    -# Force focussing on "something" so that next shortcut is taken
    -app.type(Key.TAB)
    -
    -app.waitIdleStdout()
    -
    -app.k.Build()
    -
    -app.waitIdleStdout(5,30)
    -
    -app.k.Connect()
    -
    -app.waitIdleStdout()
    -
    -app.k.Transfer()
    -
    -app.waitIdleStdout()
    -
    -app.k.Run()
    -
    -# wait 10 seconds
    -found = app.waitPatternInStdout("Test OK", 10)
    -
    -app.close()
    -
    -if found:
    - exit(0)
    -else:
    - exit(1)
    -
    +run_test(test)
    --- a/tests/ide_tests/run_python_exemple.sikuli/run_python_exemple.py Tue Mar 29 08:50:01 2022 +0200
    +++ b/tests/ide_tests/run_python_exemple.sikuli/run_python_exemple.py Thu Apr 07 07:40:32 2022 +0200
    @@ -11,34 +11,29 @@
    # common test definitions module
    from sikuliberemiz import *
    -# Start the app
    -app = BeremizApp(exemple="python")
    -
    -app.k.Clean()
    -
    -app.waitForChangeAndIdleStdout()
    -
    -app.k.Build()
    -
    -app.waitForChangeAndIdleStdout()
    -
    -app.k.Connect()
    -
    -app.waitForChangeAndIdleStdout()
    +def test(app):
    + # Start the app
    +
    + app.k.Clean()
    +
    + app.waitForChangeAndIdleStdout()
    +
    + app.k.Build()
    +
    + app.waitForChangeAndIdleStdout()
    +
    + app.k.Connect()
    +
    + app.waitForChangeAndIdleStdout()
    +
    + app.k.Transfer()
    +
    + app.waitForChangeAndIdleStdout()
    +
    + app.k.Run()
    +
    + # wait 10 seconds for 10 Grumpfs
    + return app.waitPatternInStdout("Grumpf", 10, 10)
    +
    +run_test(test, exemple="python")
    -app.k.Transfer()
    -
    -app.waitForChangeAndIdleStdout()
    -
    -app.k.Run()
    -
    -# wait 10 seconds for 10 Grumpfs
    -found = app.waitPatternInStdout("Grumpf", 10, 10)
    -
    -app.close()
    -
    -if found:
    - exit(0)
    -else:
    - exit(1)
    -
    --- a/tests/ide_tests/sikuliberemiz.py Tue Mar 29 08:50:01 2022 +0200
    +++ b/tests/ide_tests/sikuliberemiz.py Thu Apr 07 07:40:32 2022 +0200
    @@ -3,6 +3,7 @@
    import os
    import sys
    import subprocess
    +import traceback
    from threading import Thread, Event, Lock
    from time import time as timesec
    @@ -33,17 +34,17 @@
    "Address": ("l",sikuli.Key.CTRL)} # to reach address bar in GTK's file selector
    def __init__(self, app):
    - self.app = app.sikuliapp
    + self.app = app
    def __getattr__(self, name):
    fkey = self.fkeys[name]
    if type(fkey) != tuple:
    fkey = (fkey,)
    - app = self.app
    def PressShortCut():
    - app.focus()
    + self.app.sikuliapp.focus()
    sikuli.type(*fkey)
    + self.app.ReportText("Sending " + name + " shortcut")
    return PressShortCut
    @@ -86,6 +87,8 @@
    break
    c = c - 1
    + self.ReportScreenShot("UI is idle" if c != 0 else "UI is not idle")
    +
    if c == 0:
    raise Exception("Window did not idle before timeout")
    @@ -113,6 +116,7 @@
    if len(a) == 0 or a is None:
    break
    sys.stdout.write(a)
    + self.ReportOutput(a)
    self.event.set()
    if self.pattern is not None and a.find(self.pattern) >= 0:
    sys.stdout.write("found pattern in '" + a +"'")
    @@ -126,7 +130,11 @@
    """
    start_time = timesec()
    - if self.event.wait(timeout):
    + wait_result = self.event.wait(timeout)
    +
    + self.ReportScreenShot("stdout changed" if wait_result else "stdout didn't change")
    +
    + if wait_result:
    self.event.clear()
    else:
    raise Exception("Stdout didn't become active before timeout")
    @@ -148,8 +156,11 @@
    self.event.clear()
    else:
    # timeout -> no event -> idle -> exit
    + self.ReportScreenShot("stdout is idle")
    return True
    + self.ReportScreenShot("stdout did not idle")
    +
    raise Exception("Stdout did not idle before timeout")
    def waitPatternInStdout(self, pattern, timeout, count=1):
    @@ -170,6 +181,7 @@
    if found >= count:
    break
    self.pattern = None
    + self.ReportScreenShot("found pattern" if res else "pattern not found")
    return res
    class BeremizApp(IDEIdleObserver, stdoutIdleObserver):
    @@ -185,6 +197,21 @@
    Sikuli App class instance
    """
    + self.screenshotnum = 0
    + self.starttime = timesec()
    + self.screen = sikuli.Screen()
    +
    + self.report = open("report.html", "w")
    + self.report.write("""<!doctype html>
    +<html>
    + <head>
    + <meta charset="utf-8">
    + <meta name="color-scheme" content="light dark">
    + <title>Test report</title>
    + </head>
    + <body>
    +""")
    +
    command = [python_bin, opj(beremiz_path,"Beremiz.py"), "--log=/dev/stdout"]
    if exemple is not None:
    @@ -198,6 +225,8 @@
    # - use wmctrl to find IDE window details and maximize it
    # - pass exact window title to App class constructor
    + self.ReportText("Launching " + repr(command))
    +
    self.proc = subprocess.Popen(command, stdout=subprocess.PIPE, bufsize=0)
    # Window are macthed against process' PID
    @@ -236,12 +265,21 @@
    stdoutIdleObserver.__init__(self)
    # stubs for common sikuli calls to allow adding hooks later
    - for n in ["click","doubleClick","type"]:
    - setattr(self, n, getattr(sikuli, n))
    + for name in ["click","doubleClick","type"]:
    + def makeMyMeth(n):
    + def myMeth(*args, **kwargs):
    + getattr(sikuli, n)(*args, **kwargs)
    + self.ReportScreenShot(n + "(" + repr(args) + "," + repr(kwargs) + ")")
    + return myMeth
    + setattr(self, name, makeMyMeth(name))
    def close(self):
    self.sikuliapp.close()
    self.sikuliapp = None
    + self.report.write("""
    + </body>
    +</html>""")
    + self.report.close()
    def __del__(self):
    if self.sikuliapp is not None:
    @@ -249,3 +287,44 @@
    IDEIdleObserver.__del__(self)
    stdoutIdleObserver.__del__(self)
    + def ReportScreenShot(self, msg):
    + elapsed = "%.3fs: "%(timesec() - self.starttime)
    + fname = "capture"+str(self.screenshotnum)+".png"
    + cap = self.screen.capture(self.r)
    + cap.save(".", fname)
    + # self.report.write("ReportScreenShot " + msg + " " + fname + "\n")
    + self.screenshotnum = self.screenshotnum + 1
    + self.report.write( "<p>" + elapsed + msg + "<img src=\""+ fname + "\">" + "</p>")
    +
    + def ReportText(self, text):
    + elapsed = "%.3fs: "%(timesec() - self.starttime)
    + self.report.write("<p>" + elapsed + text + "</p>")
    +
    + def ReportOutput(self, text):
    + elapsed = "%.3fs: "%(timesec() - self.starttime)
    + self.report.write("<pre>" + elapsed + text + "</pre>")
    +
    +
    +def run_test(func, *args, **kwargs):
    + app = BeremizApp(*args, **kwargs)
    + try:
    + success = func(app)
    + except:
    + # sadly, sys.excepthook is broken in sikuli/jython
    + # purpose of this run_test function is to work around it.
    + # and catch exception cleanly anyhow
    + e_type, e_value, e_traceback = sys.exc_info()
    + err_msg = "\n".join(traceback.format_exception(e_type, e_value, e_traceback))
    + sys.stdout.write(err_msg)
    + app.ReportOutput(err_msg)
    + success = False
    +
    + app.close()
    +
    + if success:
    + sikuli.exit(0)
    + else:
    + sikuli.exit(1)
    +
    +
    +