beremiz

Tests: added tests/Makefile and a first test in tests/ide_tests. Test requires sikuli and Xvfb or Xnest.
wxPython4
2022-02-13, Edouard Tisserant
7db96e011fe7
Parents 84afcc0ebadd
Children ee3b84d09ccf
Tests: added tests/Makefile and a first test in tests/ide_tests. Test requires sikuli and Xvfb or Xnest.
5 files changed, 438 insertions(+), 0 deletions(-)
  • +2 -0
    .hgignore
  • +1 -0
    BeremizIDE.py
  • +160 -0
    tests/Makefile
  • +57 -0
    tests/ide_tests/run_python_exemple.sikuli/run_python_exemple.py
  • +218 -0
    tests/ide_tests/sikuliberemiz.py
    • --- a/.hgignore Sun Feb 13 20:53:00 2022 +0100
    +++ b/.hgignore Sun Feb 13 20:59:42 2022 +0100
    @@ -24,3 +24,5 @@
    doc/_build
    doc/locale
    +
    +^.*\$py.class$
    --- a/BeremizIDE.py Sun Feb 13 20:53:00 2022 +0100
    +++ b/BeremizIDE.py Sun Feb 13 20:59:42 2022 +0100
    @@ -398,6 +398,7 @@
    self.LogConsole.MarkerDefine(0, wx.stc.STC_MARK_CIRCLE, "BLACK", "RED")
    self.LogConsole.SetModEventMask(wx.stc.STC_MOD_INSERTTEXT)
    + self.LogConsole.SetCaretPeriod(0)
    self.LogConsole.Bind(wx.stc.EVT_STC_MARGINCLICK, self.OnLogConsoleMarginClick)
    self.LogConsole.Bind(wx.stc.EVT_STC_MODIFIED, self.OnLogConsoleModified)
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/tests/Makefile Sun Feb 13 20:59:42 2022 +0100
    @@ -0,0 +1,160 @@
    +#! gmake
    +
    +# beremiz/tests/Makefile :
    +#
    +# Makefile to prepare and run Beremiz tests.
    +#
    +# For developper to:
    +# - quickly run a test (TDD) on current code
    +# - write new tests, debug existing tests
    +#
    +# Use cases :
    +#
    +# run given tests
    +# $ make run_python_exemple.sikuli
    +#
    +# run tests from particular test classes
    +# $ make ide_tests
    +#
    +# run one particular test in a Xnest window
    +# $ make xnest_run_python_exemple.sikuli
    +#
    +# run Xnest window with just xterm
    +# $ make xnest_xterm
    +#
    +# run Xnest window with sikuli IDE and xterm
    +# $ make xnest_xterm
    +#
    +# build minimal beremiz and matiec to run tests
    +# $ make own_apps
    +#
    +# For CI/CD scripts to catch and report all failures. Use cases :
    +#
    +# run all tests
    +# $ make
    +#
    +#
    +# Variable $(src) is directory such that executed
    +# $(src)/beremiz/tests/Makefile is this file.
    +#
    +# Test results, and other test byproducts are in $(test_dir),
    +# $(test_dir) defaults to $(HOME)/test and can be overloaded:
    +# $ make test_dir=${HOME}/other_test_dir
    +#
    +#
    +# Matiec and Beremiz code are expected to be clean as if after hg clean --all.
    +# Any change in Matiec directory triggers rebuild of matiec.
    +# Any change in Matiec and Beremiz directory triggers copy of source code
    +# to $(test_dir)/build.
    +#
    +#
    +# Please note:
    +# In order to run asside a freshly build Matiec, tested beremiz instance
    +# needs to run on code from $(test_dir)/build/beremiz, a fresh copy
    +# of the Beremiz directory $(src)/beremiz, where we run tests from.
    +#
    +
    +all: source_check cli_tests ide_tests runtime_tests
    +
    +src := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
    +workspace ?= $(abspath $(src)/../..)
    +
    +test_dir ?= $(HOME)/test
    +build_dir = $(test_dir)/build
    +
    +OWN_PROJECTS=beremiz matiec
    +
    +# sha1 checksum of source is used to force copy/compile on each change
    +
    +define make_checksum_assign
    +$(1)_checksum = $(shell tar --exclude=.hg --exclude=.git --exclude=.*.swp -c $(workspace)/$(1) | sha1sum | cut -d ' ' -f 1)
    +endef
    +$(foreach project,$(OWN_PROJECTS),$(eval $(call make_checksum_assign,$(project))))
    +
    +$(build_dir):
    + mkdir -p $(build_dir)
    +
    +define make_src_rule
    +$(build_dir)/$(1)/$($(1)_checksum).sha1: $(build_dir) $(workspace)/$(1)
    + rm -rf $(build_dir)/$(1)
    + cp -a $(workspace)/$(1) $(build_dir)/$(1)
    + touch $$@
    +endef
    +$(foreach project,$(OWN_PROJECTS),$(eval $(call make_src_rule,$(project))))
    +
    +$(build_dir)/matiec/iec2c: | $(build_dir)/matiec/$(matiec_checksum).sha1
    + cd $(build_dir)/matiec && \
    + autoreconf -i && \
    + ./configure && \
    + make
    +
    +# TODO: use packge (deb/snap ?)
    +own_apps: $(build_dir)/matiec/iec2c $(build_dir)/beremiz/$(beremiz_checksum).sha1
    + touch $@
    +
    +ide_tests = $(subst $(src)/ide_tests/,,$(wildcard $(src)/ide_tests/*.sikuli))
    +
    +define idetest_command
    + (fluxbox &); BEREMIZPATH=$(build_dir)/beremiz sikulix -r $(src)/ide_tests/$(1) | tee test_stdout.txt; exit $$$${pipestatus[0]}
    +endef
    +
    +# Xnest based interactive sessions for tests edit and debug.
    +# Would be nice with something equivalent to xvfb-run, waiting for USR1.
    +# Arbitrary "sleep 1" is probably enough for interactive use
    +define xnest_run
    + Xnest :42 -geometry 1920x1080+0+0 & export xnestpid=$$!; sleep 1; DISPLAY=:42 $(1); export res=$$?; kill $${xnestpid} 2>/dev/null; exit $${res}
    +endef
    +
    +# Manually invoked rule {testname}.sikuli
    +define make_idetest_rule
    +$(test_dir)/$(1)_idetest/.passed: own_apps
    + rm -rf $(test_dir)/$(1)_idetest
    + mkdir $(test_dir)/$(1)_idetest
    + cd $(test_dir)/$(1)_idetest; xvfb-run -s '-screen 0 1920x1080x24' bash -c '$$(call idetest_command, $(1))'
    + touch $$@
    +
    +$(1): $(test_dir)/$(1)_idetest/.passed
    +
    +# Manually invoked rule xnest_{testname}.sikuli
    +# runs test in xnest so that one can see what happens
    +# depends on docker arguments "-v /tmp/.X11-unix/X0:/tmp/.X11-unix/X0 -e DISPLAY=$DISPLAY"
    +xnest_$(1): own_apps
    + rm -rf $(test_dir)/$(1)_idetest
    + mkdir $(test_dir)/$(1)_idetest
    + cd $(test_dir)/$(1)_idetest; $$(call xnest_run, bash -c '$(call idetest_command, $(1))')
    +
    +ide_tests_targets += $(test_dir)/$(1)_idetest/.passed
    +endef
    +$(foreach idetest,$(ide_tests),$(eval $(call make_idetest_rule,$(idetest))))
    +
    +ide_tests : $(ide_tests_targets)
    + echo "$(ide_tests_targets)" : Passed
    +
    +xnest_xterm: own_apps
    + $(call xnest_run, bash -c '(fluxbox &);xterm')
    +
    +xnest_sikuli: own_apps
    + $(call xnest_run, bash -c '(fluxbox &);(xterm -e sikulix &);xterm')
    +
    +
    +# in case VNC would be used
    + #xvfb-run -s '-screen 0 1920x1080x24' bash -c '(fluxbox &);(x11vnc &);xterm;'
    +
    +
    +clean:
    + rm -rf $(ide_tests_targets) $(build_dir)
    +
    +
    +# TODOs
    +
    +source_check:
    + echo TODO $@
    +
    +cli_tests :
    + echo TODO $@
    +
    +runtime_tests:
    + echo TODO $@
    +
    +
    +
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/tests/ide_tests/run_python_exemple.sikuli/run_python_exemple.py Sun Feb 13 20:59:42 2022 +0100
    @@ -0,0 +1,57 @@
    +""" This test opens, builds and runs exemple project named "python".
    +Test succeeds if runtime's stdout behaves as expected
    +"""
    +
    +import os
    +import time
    +
    +# allow module import from current test directory's parent
    +addImportPath(os.path.dirname(getBundlePath()))
    +
    +# common test definitions module
    +from sikuliberemiz import *
    +
    +# Start the app
    +proc,app = StartBeremizApp(exemple="python")
    +
    +# To detect when actions did finish because IDE content isn't changing
    +# idle = IDEIdleObserver(app)
    +# screencap based idle detection was making many false positive. Test is more stable with stdout based idle detection
    +
    +stdoutIdle = stdoutIdleObserver(proc)
    +
    +# To send keyboard shortuts
    +k = KBDShortcut(app)
    +
    +k.Clean()
    +
    +stdoutIdle.Wait(2,15)
    +
    +k.Build()
    +
    +stdoutIdle.Wait(2,15)
    +
    +k.Connect()
    +
    +stdoutIdle.Wait(2,15)
    +
    +k.Transfer()
    +
    +stdoutIdle.Wait(2,15)
    +
    +#del idle
    +
    +del stdoutIdle
    +
    +k.Run()
    +
    +# wait 10 seconds for 10 Grumpfs
    +found = waitPatternInStdout(proc, "Grumpf", 10, 10)
    +
    +app.close()
    +
    +if found:
    + exit(0)
    +else:
    + exit(1)
    +
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/tests/ide_tests/sikuliberemiz.py Sun Feb 13 20:59:42 2022 +0100
    @@ -0,0 +1,218 @@
    +"Commons definitions for sikuli based beremiz IDE GUI tests"
    +
    +import os
    +import sys
    +import subprocess
    +from threading import Thread, Event
    +from sikuli import *
    +
    +home = os.environ["HOME"]
    +beremiz_path = os.environ["BEREMIZPATH"]
    +
    +opj = os.path.join
    +
    +
    +def StartBeremizApp(projectpath=None, exemple=None):
    + """
    + Starts Beremiz IDE, waits for main window to appear, maximize it.
    +
    + Parameters:
    + projectpath (str): path to project to open
    + exemple (str): path relative to exemples directory
    +
    + Returns:
    + Sikuli App class instance
    + """
    +
    + command = ["%s/beremizenv/bin/python"%home, opj(beremiz_path,"Beremiz.py"), "--log=/dev/stdout"]
    +
    + if exemple is not None:
    + command.append(opj(beremiz_path,"exemples",exemple))
    + elif projectpath is not None:
    + command.append(projectpath)
    +
    + # App class is broken in Sikuli 2.0.5: can't start process with arguments.
    + #
    + # Workaround : - use subprocess module to spawn IDE process,
    + # - use wmctrl to find IDE window details and maximize it
    + # - pass exact window title to App class constructor
    +
    + proc = subprocess.Popen(command, stdout=subprocess.PIPE, bufsize=0)
    +
    + # Window are macthed against process' PID
    + ppid = proc.pid
    +
    + # Timeout 5s
    + c = 50
    + while c > 0:
    + # equiv to "wmctrl -l -p | grep $pid"
    + try:
    + wlist = filter(lambda l:(len(l)>2 and l[2]==str(ppid)), map(lambda s:s.split(None,4), subprocess.check_output(["wmctrl", "-l", "-p"]).splitlines()))
    + except subprocess.CalledProcessError:
    + wlist = []
    +
    + # window with no title only has 4 fields do describe it
    + # beremiz splashcreen has no title
    + # we wity until main window is visible
    + if len(wlist) == 1 and len(wlist[0]) == 5:
    + windowID,_zero,wpid,_XID,wtitle = wlist[0]
    + break
    +
    + wait(0.1)
    + c = c - 1
    +
    + if c == 0:
    + raise Exception("Couldn't find Beremiz window")
    +
    + # Maximize window x and y
    + subprocess.check_call(["wmctrl", "-i", "-r", windowID, "-b", "add,maximized_vert,maximized_horz"])
    +
    + # switchApp creates an App object by finding window by title, is not supposed to spawn a process
    + return proc, switchApp(wtitle)
    +
    +class KBDShortcut:
    + """Send shortut to app by calling corresponding methods:
    + Stop
    + Run
    + Transfer
    + Connect
    + Clean
    + Build
    +
    + example:
    + k = KBDShortcut(app)
    + k.Clean()
    + """
    +
    + fkeys = {"Stop": Key.F4,
    + "Run": Key.F5,
    + "Transfer": Key.F6,
    + "Connect": Key.F7,
    + "Clean": Key.F9,
    + "Build": Key.F11}
    +
    + def __init__(self, app):
    + self.app = app
    +
    + def __getattr__(self, name):
    + fkey = self.fkeys[name]
    + app = self.app
    +
    + def PressShortCut():
    + app.focus()
    + type(fkey)
    +
    + return PressShortCut
    +
    +
    +class IDEIdleObserver:
    + "Detects when IDE is idle. This is particularly handy when staring an operation and witing for the en of it."
    +
    + def __init__(self, app):
    + """
    + Parameters:
    + app (class App): Sikuli app given by StartBeremizApp
    + """
    + self.r = Region(app.window())
    +
    + self.idechanged = False
    +
    + # 200 was selected because default 50 was still catching cursor blinking in console
    + # FIXME : remove blinking cursor in console
    + self.r.onChange(200,self._OnIDEWindowChange)
    + self.r.observeInBackground()
    +
    + def __del__(self):
    + self.r.stopObserver()
    +
    + def _OnIDEWindowChange(self, event):
    + print event
    + self.idechanged = True
    +
    + def Wait(self, period, timeout):
    + """
    + Wait for IDE to stop changing
    + Parameters:
    + period (int): how many seconds with no change to consider idle
    + timeout (int): how long to wait for idle, in seconds
    + """
    + c = timeout/period
    + while c > 0:
    + self.idechanged = False
    + wait(period)
    + if not self.idechanged:
    + break
    + c = c - 1
    +
    + if c == 0:
    + raise Exception("Window did not idle before timeout")
    +
    +
    +class stdoutIdleObserver:
    + "Detects when IDE's stdout is idle. Can be more reliable than pixel based version (false changes ?)"
    +
    + def __init__(self, proc):
    + """
    + Parameters:
    + proc (subprocess.Popen): Beremiz process, given by StartBeremizApp
    + """
    + self.proc = proc
    + self.stdoutchanged = False
    +
    + self.thread = Thread(target = self._waitStdoutProc).start()
    +
    + def _waitStdoutProc():
    + while True:
    + a = self.proc.stdout.read(1)
    + if len(a) == 0 or a is None:
    + break
    + sys.stdout.write(a)
    + self.idechanged = True
    +
    + def Wait(self, period, timeout):
    + """
    + Wait for IDE'stdout to stop changing
    + Parameters:
    + period (int): how many seconds with no change to consider idle
    + timeout (int): how long to wait for idle, in seconds
    + """
    + c = timeout/period
    + while c > 0:
    + self.idechanged = False
    + wait(period)
    + if not self.idechanged:
    + break
    + c = c - 1
    +
    + if c == 0:
    + raise Exception("Stdout did not idle before timeout")
    +
    +
    +def waitPatternInStdout(proc, pattern, timeout, count=1):
    +
    + success_event = Event()
    +
    + def waitPatternInStdoutProc():
    + found = 0
    + while True:
    + a = proc.stdout.readline()
    + if len(a) == 0 or a is None:
    + raise Exception("App finished before producing expected stdout pattern")
    + sys.stdout.write(a)
    + if a.find(pattern) >= 0:
    + found = found + 1
    + if found >= count:
    + success_event.set()
    + break
    +
    +
    + Thread(target = waitPatternInStdoutProc).start()
    +
    + if not success_event.wait(timeout):
    + # test timed out
    + return False
    + else:
    + return True
    +
    +
    +