beremiz

IDE: Cleaned up some useless tests in variable trace data handling code, changed from bare numpy arrays to RingBuffers inorder to avoid RAM outage and crash after long tracing session.

2021-06-14, Edouard Tisserant
3cc5663af196
Parents 610801fe1b88
Children 5f7445b582d4
IDE: Cleaned up some useless tests in variable trace data handling code, changed from bare numpy arrays to RingBuffers inorder to avoid RAM outage and crash after long tracing session.
5 files changed, 125 insertions(+), 67 deletions(-)
  • +22 -20
    ProjectController.py
  • +14 -13
    controls/DebugVariablePanel/DebugVariableItem.py
  • +33 -33
    controls/DebugVariablePanel/DebugVariablePanel.py
  • +55 -0
    controls/DebugVariablePanel/RingBuffer.py
  • +1 -1
    runtime/typemapping.py
    • --- a/ProjectController.py Fri Jun 11 11:56:07 2021 +0200
    +++ b/ProjectController.py Mon Jun 14 16:48:39 2021 +0200
    @@ -40,6 +40,7 @@
    from datetime import datetime
    from weakref import WeakKeyDictionary
    from functools import reduce
    +from itertools import izip
    from distutils.dir_util import copy_tree
    from six.moves import xrange
    @@ -1512,8 +1513,8 @@
    for debug_tick, debug_buff in Traces:
    debug_vars = UnpackDebugBuffer(
    debug_buff, self.TracedIECTypes)
    - if debug_vars is not None and len(debug_vars) == len(self.TracedIECPath):
    - for IECPath, values_buffer, value in zip(
    + if debug_vars is not None:
    + for IECPath, values_buffer, value in izip(
    self.TracedIECPath,
    self.DebugValuesBuffers,
    debug_vars):
    @@ -1606,8 +1607,8 @@
    WeakKeyDictionary(), # Callables
    [], # Data storage [(tick, data),...]
    "Registered", # Variable status
    - None,
    - buffer_list] # Forced value
    + None, # Forced value
    + buffer_list]
    self.IECdebug_datas[IECPath] = IECdebug_data
    else:
    IECdebug_data[4] |= buffer_list
    @@ -1681,27 +1682,28 @@
    return self._connector.RemoteExec(script, **kwargs)
    def DispatchDebugValuesProc(self, event):
    - self.debug_status, debug_ticks, buffers = self.SnapshotAndResetDebugValuesBuffers()
    + event.Skip()
    start_time = time.time()
    - if len(self.TracedIECPath) == len(buffers):
    - for IECPath, values in zip(self.TracedIECPath, buffers):
    - if len(values) > 0:
    - self.CallWeakcallables(
    - IECPath, "NewValues", debug_ticks, values)
    - if len(debug_ticks) > 0:
    - self.CallWeakcallables(
    - "__tick__", "NewDataAvailable", debug_ticks)
    + self.debug_status, debug_ticks, buffers = self.SnapshotAndResetDebugValuesBuffers()
    if self.debug_status == PlcStatus.Broken:
    self.logger.write_warning(
    _("Debug: token rejected - other debug took over - reconnect to recover\n"))
    - else:
    - delay = time.time() - start_time
    - next_refresh = max(REFRESH_PERIOD - delay, 0.2 * delay)
    - if self.DispatchDebugValuesTimer is not None:
    - self.DispatchDebugValuesTimer.Start(
    - int(next_refresh * 1000), oneShot=True)
    - event.Skip()
    + return
    +
    + for IECPath, values in zip(self.TracedIECPath, buffers):
    + if len(values) > 0:
    + self.CallWeakcallables(
    + IECPath, "NewValues", debug_ticks, values)
    + if len(debug_ticks) > 0:
    + self.CallWeakcallables(
    + "__tick__", "NewDataAvailable", debug_ticks)
    +
    + delay = time.time() - start_time
    + next_refresh = max(REFRESH_PERIOD - delay, 0.2 * delay)
    + if self.DispatchDebugValuesTimer is not None:
    + res = self.DispatchDebugValuesTimer.Start(
    + int(next_refresh * 1000), oneShot=True)
    def KillDebugThread(self):
    if self.DispatchDebugValuesTimer is not None:
    --- a/controls/DebugVariablePanel/DebugVariableItem.py Fri Jun 11 11:56:07 2021 +0200
    +++ b/controls/DebugVariablePanel/DebugVariableItem.py Mon Jun 14 16:48:39 2021 +0200
    @@ -26,8 +26,9 @@
    from __future__ import absolute_import
    from datetime import timedelta
    import binascii
    -import numpy
    +import numpy as np
    from graphics.DebugDataConsumer import DebugDataConsumer, TYPE_TRANSLATOR
    +from controls.DebugVariablePanel.RingBuffer import RingBuffer
    # -------------------------------------------------------------------------------
    # Constant for calculate CRC for string variables
    @@ -142,8 +143,8 @@
    @return: Data as numpy.array([(tick, value, forced),...])
    """
    # Return immediately if data empty or none
    - if self.Data is None or len(self.Data) == 0:
    - return self.Data
    + if self.Data is None or self.Data.count == 0:
    + return None
    # Find nearest data outside given range indexes
    start_idx = (self.GetNearestData(start_tick, -1)
    @@ -154,7 +155,7 @@
    else len(self.Data))
    # Return data between indexes
    - return self.Data[start_idx:end_idx]
    + return self.Data.view[start_idx:end_idx]
    def GetRawValue(self, index):
    """
    @@ -195,8 +196,8 @@
    if len(values) > 0:
    # Return value range for data in given tick range
    return (data,
    - data[numpy.argmin(values), 1],
    - data[numpy.argmax(values), 1])
    + data[np.argmin(values), 1],
    + data[np.argmax(values), 1])
    # Return default values
    return data, None, None
    @@ -207,7 +208,7 @@
    """
    if self.StoreData and self.IsNumVariable():
    # Init table storing data
    - self.Data = numpy.array([]).reshape(0, 3)
    + self.Data = RingBuffer(3)
    # Init table storing raw data if variable is strin
    self.RawData = ([]
    @@ -294,7 +295,7 @@
    [float(tick), num_value, extra_value])
    # Add New data to stored data table
    - self.Data = numpy.append(self.Data, data_values, axis=0)
    + self.Data.append(data_values)
    # Signal to debug variable panel to refresh
    self.Parent.HasNewData = True
    @@ -337,7 +338,7 @@
    if tick is not None and self.Data is not None:
    # Return current value and forced flag if data empty
    - if len(self.Data) == 0:
    + if self.Data.count == 0:
    return self.Value, self.IsForced()
    # Get index of nearest data from tick given
    @@ -345,9 +346,9 @@
    # Get value and forced flag at given index
    value, forced = \
    - self.RawData[int(self.Data[idx, 2])] \
    + self.RawData[int(self.Data.view[idx, 2])] \
    if self.VariableType in ["STRING", "WSTRING"] \
    - else self.Data[idx, 1:3]
    + else self.Data.view[idx, 1:3]
    if self.VariableType in ["TIME", "TOD", "DT", "DATE"]:
    value = timedelta(seconds=value)
    @@ -380,10 +381,10 @@
    return None
    # Extract data ticks
    - ticks = self.Data[:, 0]
    + ticks = self.Data.view[:, 0]
    # Get nearest data from tick
    - idx = numpy.argmin(abs(ticks - tick))
    + idx = np.searchsorted(ticks, tick)
    # Adjust data index according to constraint
    if adjust < 0 and ticks[idx] > tick and idx > 0 or \
    --- a/controls/DebugVariablePanel/DebugVariablePanel.py Fri Jun 11 11:56:07 2021 +0200
    +++ b/controls/DebugVariablePanel/DebugVariablePanel.py Mon Jun 14 16:48:39 2021 +0200
    @@ -26,7 +26,7 @@
    from __future__ import absolute_import
    from __future__ import division
    from functools import reduce
    -import numpy
    +import numpy as np
    import wx
    import wx.lib.buttons
    @@ -43,6 +43,7 @@
    from controls.DebugVariablePanel.DebugVariableItem import DebugVariableItem
    from controls.DebugVariablePanel.DebugVariableTextViewer import DebugVariableTextViewer
    from controls.DebugVariablePanel.DebugVariableGraphicViewer import *
    +from controls.DebugVariablePanel.RingBuffer import RingBuffer
    MILLISECOND = 1000000 # Number of nanosecond in a millisecond
    @@ -205,7 +206,7 @@
    main_sizer = wx.BoxSizer(wx.VERTICAL)
    - self.Ticks = numpy.array([]) # List of tick received
    + self.Ticks = RingBuffer() # List of tick received
    self.StartTick = 0 # Tick starting range of data displayed
    self.Fixed = False # Flag that range of data is fixed
    self.CursorTick = None # Tick of cursor for displaying values
    @@ -344,11 +345,11 @@
    tick = ticks[-1]
    # Save tick as start tick for range if data is still empty
    - if len(self.Ticks) == 0:
    + if self.Ticks.count == 0:
    self.StartTick = ticks[0]
    # Add tick to list of ticks received
    - self.Ticks = numpy.append(self.Ticks, ticks)
    + self.Ticks.append(ticks)
    # Update start tick for range if range follow ticks received
    if not self.Fixed or tick < self.StartTick + self.CurrentRange:
    @@ -357,7 +358,7 @@
    # Force refresh if graph is fixed because range of data received
    # is too small to fill data range selected
    if self.Fixed and \
    - self.Ticks[-1] - self.Ticks[0] < self.CurrentRange:
    + self.Ticks.view[-1] - self.Ticks.view[0] < self.CurrentRange:
    self.Force = True
    self.HasNewData = False
    @@ -385,17 +386,16 @@
    def MoveCursorTick(self, move):
    if self.CursorTick is not None:
    - cursor_tick = max(self.Ticks[0],
    - min(self.CursorTick + move, self.Ticks[-1]))
    - cursor_tick_idx = numpy.argmin(numpy.abs(self.Ticks - cursor_tick))
    - if self.Ticks[cursor_tick_idx] == self.CursorTick:
    + cursor_tick = max(self.Ticks.view[0],
    + min(self.CursorTick + move, self.Ticks.view[-1]))
    + cursor_tick_idx = np.searchsorted(self.Ticks.view, cursor_tick)
    + if self.Ticks.view[cursor_tick_idx] == self.CursorTick:
    cursor_tick_idx = max(0,
    min(cursor_tick_idx + abs(move) // move,
    - len(self.Ticks) - 1))
    - self.CursorTick = self.Ticks[cursor_tick_idx]
    + self.Ticks.count - 1))
    + self.CursorTick = self.Ticks.view[cursor_tick_idx]
    self.StartTick = max(
    - self.Ticks[numpy.argmin(
    - numpy.abs(self.Ticks - self.CursorTick + self.CurrentRange))],
    + self.Ticks.view[np.searchsorted(self.Ticks.view, self.CursorTick + self.CurrentRange)],
    min(self.StartTick, self.CursorTick))
    self.RefreshCanvasPosition()
    self.UpdateCursorTick()
    @@ -547,8 +547,8 @@
    if self.CursorTick is not None:
    tick = self.CursorTick
    - elif len(self.Ticks) > 0:
    - tick = self.Ticks[-1]
    + elif self.Ticks.count > 0:
    + tick = self.Ticks.view[-1]
    else:
    tick = None
    if tick is not None:
    @@ -604,16 +604,16 @@
    self.RefreshGraphicsSizer()
    def SetCanvasPosition(self, tick):
    - tick = max(self.Ticks[0], min(tick, self.Ticks[-1] - self.CurrentRange))
    - self.StartTick = self.Ticks[numpy.argmin(numpy.abs(self.Ticks - tick))]
    + tick = max(self.Ticks.view[0], min(tick, self.Ticks.view[-1] - self.CurrentRange))
    + self.StartTick = self.Ticks.view[np.searchsorted(self.Ticks.view, tick)]
    self.Fixed = True
    self.RefreshCanvasPosition()
    self.ForceRefresh()
    def RefreshCanvasPosition(self):
    - if len(self.Ticks) > 0:
    - pos = int(self.StartTick - self.Ticks[0])
    - range = int(self.Ticks[-1] - self.Ticks[0])
    + if len(self.Ticks.view) > 0:
    + pos = int(self.StartTick - self.Ticks.view[0])
    + range = int(self.Ticks.view[-1] - self.Ticks.view[0])
    else:
    pos = 0
    range = 0
    @@ -626,23 +626,23 @@
    if new_range_idx != current_range_idx:
    self.CanvasRange.SetSelection(new_range_idx)
    self.CurrentRange = self.RANGE_VALUES[new_range_idx][1] / self.Ticktime
    - if len(self.Ticks) > 0:
    + if self.Ticks.count > 0:
    if tick is None:
    tick = self.StartTick + self.CurrentRange / 2.
    new_start_tick = min(tick - (tick - self.StartTick) * self.CurrentRange / current_range,
    - self.Ticks[-1] - self.CurrentRange)
    - self.StartTick = self.Ticks[numpy.argmin(numpy.abs(self.Ticks - new_start_tick))]
    - self.Fixed = new_start_tick < self.Ticks[-1] - self.CurrentRange
    + self.Ticks.view[-1] - self.CurrentRange)
    + self.StartTick = self.Ticks.view[np.searchsorted(self.Ticks.view, - new_start_tick)]
    + self.Fixed = new_start_tick < self.Ticks.view[-1] - self.CurrentRange
    self.ForceRefresh()
    def RefreshRange(self):
    - if len(self.Ticks) > 0:
    - if self.Fixed and self.Ticks[-1] - self.Ticks[0] < self.CurrentRange:
    + if self.Ticks.count > 0:
    + if self.Fixed and self.Ticks.view[-1] - self.Ticks.view[0] < self.CurrentRange:
    self.Fixed = False
    if self.Fixed:
    - self.StartTick = min(self.StartTick, self.Ticks[-1] - self.CurrentRange)
    + self.StartTick = min(self.StartTick, self.Ticks.view[-1] - self.CurrentRange)
    else:
    - self.StartTick = max(self.Ticks[0], self.Ticks[-1] - self.CurrentRange)
    + self.StartTick = max(self.Ticks.view[0], self.Ticks.view[-1] - self.CurrentRange)
    self.ForceRefresh()
    def OnRangeChanged(self, event):
    @@ -654,8 +654,8 @@
    event.Skip()
    def OnCurrentButton(self, event):
    - if len(self.Ticks) > 0:
    - self.StartTick = max(self.Ticks[0], self.Ticks[-1] - self.CurrentRange)
    + if self.Ticks.count > 0:
    + self.StartTick = max(self.Ticks.view[0], self.Ticks.view[-1] - self.CurrentRange)
    self.ResetCursorTick()
    event.Skip()
    @@ -695,8 +695,8 @@
    event.Skip()
    def OnPositionChanging(self, event):
    - if len(self.Ticks) > 0:
    - self.StartTick = self.Ticks[0] + event.GetPosition()
    + if self.Ticks.count > 0:
    + self.StartTick = self.Ticks.view[0] + event.GetPosition()
    self.Fixed = True
    self.ForceRefresh()
    event.Skip()
    @@ -908,7 +908,7 @@
    self.ForceRefresh()
    def ResetGraphicsValues(self):
    - self.Ticks = numpy.array([])
    + self.Ticks = RingBuffer()
    self.StartTick = 0
    for panel in self.GraphicPanels:
    panel.ResetItemsData()
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/controls/DebugVariablePanel/RingBuffer.py Mon Jun 14 16:48:39 2021 +0200
    @@ -0,0 +1,55 @@
    +#!/usr/bin/env python
    +# -*- coding: utf-8 -*-
    +
    +# This file is part of Beremiz
    +# Copyright (C) 2021: Edouard TISSERANT
    +#
    +# See COPYING file for copyrights details.
    +
    +# Based on Eelco Hoogendoorn stackoverflow answer about RingBuffer with numpy
    +
    +import numpy as np
    +
    +
    +class RingBuffer(object):
    + def __init__(self, width=None, size=65536, padding=None):
    + self.size = size
    + self.padding = size if padding is None else padding
    + shape = (self.size+self.padding,)
    + if width :
    + shape += (width,)
    + self.buffer = np.zeros(shape)
    + self.counter = 0
    + self.full = False
    +
    + def append(self, data):
    + """this is an O(n) operation"""
    + data = data[-self.padding:]
    + n = len(data)
    + if self.remaining < n: self.compact()
    + self.buffer[self.counter+self.size:][:n] = data
    + self.counter += n
    +
    + @property
    + def count(self):
    + return self.counter if not self.full else self.size
    +
    + @property
    + def remaining(self):
    + return self.padding-self.counter
    +
    + @property
    + def view(self):
    + """this is always an O(1) operation"""
    + return self.buffer[self.counter:][:self.size]
    +
    + def compact(self):
    + """
    + note: only when this function is called, is an O(size) performance hit incurred,
    + and this cost is amortized over the whole padding space
    + """
    + print 'compacting'
    + self.buffer[:self.size] = self.view
    + self.counter = 0
    + self.full = True
    +
    --- a/runtime/typemapping.py Fri Jun 11 11:56:07 2021 +0200
    +++ b/runtime/typemapping.py Mon Jun 14 16:48:39 2021 +0200
    @@ -92,7 +92,7 @@
    buffoffset += sizeof(c_type) if iectype != "STRING" else len(value)+1
    res.append(value)
    else:
    - break
    + return None
    if buffoffset and buffoffset == buffsize:
    return res
    return None