beremiz

Tests: Enhance robustness of stdout driven waiting state in Sikuli based tests.
wxPython4
2022-03-05, Edouard Tisserant
ce366d67a5b7
Parents ccaabb9da623
Children 24fbd4d1fe80
Tests: Enhance robustness of stdout driven waiting state in Sikuli based tests.

Some tests were randomly passing, because from time to time waiting for idle was skiped. It was combination of multiple problems :
- buffering on stdout (now use readline + flush for each write to log)
- it is sometime required to wait for activity before waiting for timeout added "WaitForChangeAndIdle" to "stdoutIdleObserver"
3 files changed, 56 insertions(+), 21 deletions(-)
  • +1 -0
    BeremizIDE.py
  • +22 -13
    tests/ide_tests/edit_project.sikuli/edit_project.py
  • +33 -8
    tests/ide_tests/sikuliberemiz.py
    • --- a/BeremizIDE.py Mon Feb 28 21:53:14 2022 +0100
    +++ b/BeremizIDE.py Sat Mar 05 11:14:00 2022 +0100
    @@ -136,6 +136,7 @@
    def write(self, s, style=None):
    if self.logf is not None:
    self.logf.write(s)
    + self.logf.flush()
    self.StackLock.acquire()
    self.stack.append((s, style))
    self.StackLock.release()
    --- a/tests/ide_tests/edit_project.sikuli/edit_project.py Mon Feb 28 21:53:14 2022 +0100
    +++ b/tests/ide_tests/edit_project.sikuli/edit_project.py Sat Mar 05 11:14:00 2022 +0100
    @@ -15,46 +15,55 @@
    proc,app = StartBeremizApp(exemple="python")
    # To detect when actions did finish because IDE content isn't changing
    -# idle = IDEIdleObserver(app)
    -# screencap based idle detection was making many false positive. Test is more stable with stdout based idle detection
    +idle = IDEIdleObserver(app)
    doubleClick("1646062660770.png")
    +idle.Wait(1,15)
    +
    click("1646066794902.png")
    +idle.Wait(1,15)
    +
    type(Key.DOWN * 10, Key.CTRL)
    +idle.Wait(1,15)
    +
    doubleClick("1646066996620.png")
    +idle.Wait(1,15)
    +
    type(Key.TAB*3) # select text content
    -type("'sys.stdout.write(\"EDIT TEST OK\")'")
    +type("'sys.stdout.write(\"EDIT TEST OK\\n\")'")
    type(Key.ENTER)
    +idle.Wait(1,15)
    +
    +k = KBDShortcut(app)
    +
    +k.Save()
    +
    +del idle
    +
    stdoutIdle = stdoutIdleObserver(proc)
    -# To send keyboard shortuts
    -k = KBDShortcut(app)
    -
    k.Clean()
    -stdoutIdle.Wait(2,15)
    +stdoutIdle.WaitForChangeAndIdle(2,15)
    -k.Save()
    k.Build()
    -stdoutIdle.Wait(2,15)
    +stdoutIdle.WaitForChangeAndIdle(2,15)
    k.Connect()
    -stdoutIdle.Wait(2,15)
    +stdoutIdle.WaitForChangeAndIdle(2,15)
    k.Transfer()
    -stdoutIdle.Wait(2,15)
    -
    -#del idle
    +stdoutIdle.WaitForChangeAndIdle(2,15)
    del stdoutIdle
    --- a/tests/ide_tests/sikuliberemiz.py Mon Feb 28 21:53:14 2022 +0100
    +++ b/tests/ide_tests/sikuliberemiz.py Sat Mar 05 11:14:00 2022 +0100
    @@ -4,6 +4,7 @@
    import sys
    import subprocess
    from threading import Thread, Event
    +from time import time as timesec
    typeof=type
    @@ -142,7 +143,7 @@
    period (int): how many seconds with no change to consider idle
    timeout (int): how long to wait for idle, in seconds
    """
    - c = timeout/period
    + c = max(timeout/period,1)
    while c > 0:
    self.idechanged = False
    wait(period)
    @@ -165,15 +166,37 @@
    self.proc = proc
    self.stdoutchanged = False
    + self.changes = 0
    + self.last_change_count = 0
    +
    + self.event = Event()
    +
    self.thread = Thread(target = self._waitStdoutProc).start()
    def _waitStdoutProc(self):
    while True:
    - a = self.proc.stdout.read(1)
    + a = self.proc.stdout.readline()
    if len(a) == 0 or a is None:
    break
    - sys.stdout.write(a)
    - self.idechanged = True
    + # sys.stdout.write(a)
    + self.changes = self.changes + 1
    + self.event.set()
    +
    + def WaitForChangeAndIdle(self, period, timeout):
    + """
    + Wait for IDE'stdout to start changing
    + Parameters:
    + timeout (int): how long to wait for change, in seconds
    + """
    + start_time = timesec()
    + if self.changes == self.last_change_count:
    + if self.event.wait(timeout):
    + self.event.clear()
    + self.last_change_count = self.changes
    + else:
    + raise Exception("Stdout didn't become active before timeout")
    +
    + self.Wait(period, timeout - (timesec() - start_time))
    def Wait(self, period, timeout):
    """
    @@ -182,11 +205,12 @@
    period (int): how many seconds with no change to consider idle
    timeout (int): how long to wait for idle, in seconds
    """
    - c = timeout/period
    + c = max(timeout/period, 1)
    while c > 0:
    - self.idechanged = False
    + changes = self.changes
    wait(period)
    - if not self.idechanged:
    + if self.changes == changes:
    + self.last_change_count = self.changes
    break
    c = c - 1
    @@ -204,8 +228,9 @@
    a = proc.stdout.readline()
    if len(a) == 0 or a is None:
    raise Exception("App finished before producing expected stdout pattern")
    - sys.stdout.write(a)
    + # sys.stdout.write(a)
    if a.find(pattern) >= 0:
    + sys.stdout.write("found pattern in '" + a +"'")
    found = found + 1
    if found >= count:
    success_event.set()