beremiz

Chunk based transfer for PLC binary and extra files, and some collateral code refactoring.

2018-12-04, Edouard Tisserant
8742337a9fe3
Parents ed6b0e905fcb
Children 10437c6c294e
Chunk based transfer for PLC binary and extra files, and some collateral code refactoring.
9 files changed, 159 insertions(+), 115 deletions(-)
  • +43 -62
    ProjectController.py
  • +21 -0
    connectors/ConnectorBase.py
  • +1 -1
    connectors/PYRO/__init__.py
  • +1 -7
    connectors/WAMP/__init__.py
  • +15 -9
    connectors/__init__.py
  • +63 -16
    runtime/PLCObject.py
  • +1 -0
    runtime/Worker.py
  • +11 -17
    targets/toolchain_gcc.py
  • +3 -3
    targets/toolchain_makefile.py
    • --- a/ProjectController.py Tue Nov 27 13:34:14 2018 +0100
    +++ b/ProjectController.py Tue Dec 04 11:31:58 2018 +0100
    @@ -891,19 +891,6 @@
    self._builder = targetclass(self)
    return self._builder
    - def ResetBuildMD5(self):
    - builder = self.GetBuilder()
    - if builder is not None:
    - builder.ResetBinaryCodeMD5()
    - self.EnableMethod("_Transfer", False)
    -
    - def GetLastBuildMD5(self):
    - builder = self.GetBuilder()
    - if builder is not None:
    - return builder.GetBinaryCodeMD5()
    - else:
    - return None
    -
    #
    #
    # C CODE GENERATION METHODS
    @@ -1131,7 +1118,6 @@
    # If IEC code gen fail, bail out.
    if not IECGenRes:
    self.logger.write_error(_("PLC code generation failed !\n"))
    - self.ResetBuildMD5()
    return False
    # Reset variable and program list that are parsed from
    @@ -1147,7 +1133,6 @@
    builder = self.GetBuilder()
    if builder is None:
    self.logger.write_error(_("Fatal : cannot get builder.\n"))
    - self.ResetBuildMD5()
    return False
    # Build
    @@ -1156,9 +1141,9 @@
    self.logger.write_error(_("C Build failed.\n"))
    return False
    except Exception:
    + builder.ResetBinaryMD5()
    self.logger.write_error(_("C Build crashed !\n"))
    self.logger.write_error(traceback.format_exc())
    - self.ResetBuildMD5()
    return False
    self.logger.write(_("Successfully built.\n"))
    @@ -1178,7 +1163,6 @@
    self.logger.write_error(
    _("Runtime IO extensions C code generation failed !\n"))
    self.logger.write_error(traceback.format_exc())
    - self.ResetBuildMD5()
    return False
    # Generate C code and compilation params from liraries
    @@ -1189,7 +1173,6 @@
    self.logger.write_error(
    _("Runtime library extensions C code generation failed !\n"))
    self.logger.write_error(traceback.format_exc())
    - self.ResetBuildMD5()
    return False
    self.LocationCFilesAndCFLAGS = LibCFilesAndCFLAGS + \
    @@ -1239,7 +1222,6 @@
    except Exception:
    self.logger.write_error(name + _(" generation failed !\n"))
    self.logger.write_error(traceback.format_exc())
    - self.ResetBuildMD5()
    return False
    self.logger.write(_("C code generated successfully.\n"))
    return True
    @@ -1820,26 +1802,18 @@
    def CompareLocalAndRemotePLC(self):
    if self._connector is None:
    return
    - # We are now connected. Update button status
    - MD5 = self.GetLastBuildMD5()
    + builder = self.GetBuilder()
    + if builder is None :
    + return
    + MD5 = builder.GetBinaryMD5()
    + if MD5 is None:
    + return
    # Check remote target PLC correspondance to that md5
    - if MD5 is not None:
    - if not self._connector.MatchMD5(MD5):
    - # self.logger.write_warning(
    - # _("Latest build does not match with target, please
    - # transfer.\n"))
    - self.EnableMethod("_Transfer", True)
    - else:
    - # self.logger.write(
    - # _("Latest build matches target, no transfer needed.\n"))
    - self.EnableMethod("_Transfer", True)
    - # warns controller that program match
    - self.ProgramTransferred()
    - # self.EnableMethod("_Transfer", False)
    + if self._connector.MatchMD5(MD5):
    + self.ProgramTransferred()
    else:
    - # self.logger.write_warning(
    - # _("Cannot compare latest build to target. Please build.\n"))
    - self.EnableMethod("_Transfer", False)
    + self.logger.write(
    + _("Latest build does not match with connected target.\n"))
    def _Disconnect(self):
    self._SetConnector(None)
    @@ -1855,8 +1829,13 @@
    else:
    return
    - # Get the last build PLC's
    - MD5 = self.GetLastBuildMD5()
    + builder = self.GetBuilder()
    + if builder is None:
    + self.logger.write_error(_("Fatal : cannot get builder.\n"))
    + return False
    +
    + # recover md5 from last build
    + MD5 = builder.GetBinaryMD5()
    # Check if md5 file is empty : ask user to build PLC
    if MD5 is None:
    @@ -1869,35 +1848,37 @@
    self.logger.write(
    _("Latest build already matches current target. Transfering anyway...\n"))
    - # Get temprary directory path
    + # purge any non-finished transfer
    + # note: this would abord any runing transfer with error
    + self._connector.PurgeBlobs()
    +
    + # transfer extra files
    extrafiles = []
    for extrafilespath in [self._getExtraFilesPath(),
    self._getProjectFilesPath()]:
    - extrafiles.extend(
    - [(name, open(os.path.join(extrafilespath, name),
    - 'rb').read())
    - for name in os.listdir(extrafilespath)])
    + for name in os.listdir(extrafilespath):
    + extrafiles.append((
    + name,
    + self._connector.BlobFromFile(
    + os.path.join(extrafilespath, name))))
    # Send PLC on target
    - builder = self.GetBuilder()
    - if builder is not None:
    - data = builder.GetBinaryCode()
    - if data is not None:
    - if self._connector.NewPLC(MD5, data, extrafiles) and self.GetIECProgramsAndVariables():
    - self.UnsubscribeAllDebugIECVariable()
    - self.ProgramTransferred()
    - if self.AppFrame is not None:
    - self.AppFrame.CloseObsoleteDebugTabs()
    - self.AppFrame.RefreshPouInstanceVariablesPanel()
    - self.logger.write(_("Transfer completed successfully.\n"))
    - self.AppFrame.LogViewer.ResetLogCounters()
    - else:
    - self.logger.write_error(_("Transfer failed\n"))
    - self.HidePLCProgress()
    - else:
    - self.logger.write_error(
    - _("No PLC to transfer (did build succeed ?)\n"))
    + object_path = builder.GetBinaryPath()
    + object_blob = self._connector.BlobFromFile(object_path)
    +
    + if self._connector.NewPLC(MD5, object_blob, extrafiles):
    + self.ProgramTransferred()
    + self.AppFrame.CloseObsoleteDebugTabs()
    + self.AppFrame.LogViewer.ResetLogCounters()
    + if self.GetIECProgramsAndVariables():
    + self.UnsubscribeAllDebugIECVariable()
    + self.AppFrame.RefreshPouInstanceVariablesPanel()
    + self.logger.write(_("Transfer completed successfully.\n"))
    + else:
    + self.logger.write_error(_("Transfer failed\n"))
    +
    + self.HidePLCProgress()
    wx.CallAfter(self.UpdateMethodsFromPLCStatus)
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/connectors/ConnectorBase.py Tue Dec 04 11:31:58 2018 +0100
    @@ -0,0 +1,21 @@
    +#!/usr/bin/env python
    +# -*- coding: utf-8 -*-
    +
    +# See COPYING file for copyrights details.
    +
    +import md5
    +
    +class ConnectorBase(object):
    +
    + chuncksize = 16384
    + def BlobFromFile(self, filepath):
    + s = md5.new()
    + blobID = s.digest() # empty md5, to support empty blob
    + with open(filepath, "rb") as f:
    + while True:
    + chunk = f.read(self.chuncksize)
    + if len(chunk) == 0: return blobID
    + blobID = self.AppendChunkToBlob(chunk, blobID)
    + s.update(chunk)
    + if blobID != s.digest(): return None
    +
    --- a/connectors/PYRO/__init__.py Tue Nov 27 13:34:14 2018 +0100
    +++ b/connectors/PYRO/__init__.py Tue Dec 04 11:31:58 2018 +0100
    @@ -150,4 +150,4 @@
    self.__dict__[attrName] = member
    return member
    - return PyroProxyProxy()
    + return PyroProxyProxy
    --- a/connectors/WAMP/__init__.py Tue Nov 27 13:34:14 2018 +0100
    +++ b/connectors/WAMP/__init__.py Tue Dec 04 11:31:58 2018 +0100
    @@ -156,10 +156,4 @@
    # TODO : GetPLCID()
    # TODO : PSK.UpdateID()
    - # Try to get the proxy object
    - try:
    - return WampPLCObjectProxy()
    - except Exception:
    - confnodesroot.logger.write_error(_("WAMP connection to '%s' failed.\n") % location)
    - confnodesroot.logger.write_error(traceback.format_exc())
    - return None
    + return WampPLCObjectProxy
    --- a/connectors/__init__.py Tue Nov 27 13:34:14 2018 +0100
    +++ b/connectors/__init__.py Tue Dec 04 11:31:58 2018 +0100
    @@ -29,6 +29,8 @@
    from __future__ import absolute_import
    from os import listdir, path
    import util.paths as paths
    +from connectors.ConnectorBase import ConnectorBase
    +from types import ClassType
    connectors_packages = ["PYRO","WAMP"]
    @@ -64,8 +66,8 @@
    Return a connector corresponding to the URI
    or None if cannot connect to URI
    """
    - scheme = uri.split("://")[0].upper()
    - if scheme == "LOCAL":
    + _scheme = uri.split("://")[0].upper()
    + if _scheme == "LOCAL":
    # Local is special case
    # pyro connection to local runtime
    # started on demand, listening on random port
    @@ -73,17 +75,21 @@
    runtime_port = confnodesroot.AppFrame.StartLocalRuntime(
    taskbaricon=True)
    uri = "PYROLOC://127.0.0.1:" + str(runtime_port)
    - elif scheme in connectors:
    - pass
    - elif scheme[-1] == 'S' and scheme[:-1] in connectors:
    - scheme = scheme[:-1]
    + elif _scheme in connectors:
    + scheme = _scheme
    + elif _scheme[-1] == 'S' and _scheme[:-1] in connectors:
    + scheme = _scheme[:-1]
    else:
    return None
    - # import module according to uri type
    - connectorclass = connectors[scheme]()
    - return connectorclass(uri, confnodesroot)
    + # import module according to uri type and get connector specific baseclass
    + # first call to import the module,
    + # then call with parameters to create the class
    + connector_specific_class = connectors[scheme]()(uri, confnodesroot)
    + # new class inheriting from generic and specific connector base classes
    + return ClassType(_scheme + "_connector",
    + (ConnectorBase, connector_specific_class), {})()
    def EditorClassFromScheme(scheme):
    _Import_Dialogs()
    --- a/runtime/PLCObject.py Tue Nov 27 13:34:14 2018 +0100
    +++ b/runtime/PLCObject.py Tue Dec 04 11:31:58 2018 +0100
    @@ -34,6 +34,9 @@
    import Pyro.core as pyro
    import six
    from six.moves import _thread, xrange
    +import md5
    +from tempfile import mkstemp
    +import shutil
    from runtime.typemapping import TypeTranslator
    from runtime.loglevels import LogLevelsDefault, LogLevelsCount
    @@ -41,6 +44,8 @@
    from runtime import PlcStatus
    from runtime import MainWorker
    +empty_md5_digest = md5.new().digest()
    +
    if os.name in ("nt", "ce"):
    dlopen = _ctypes.LoadLibrary
    dlclose = _ctypes.FreeLibrary
    @@ -74,7 +79,11 @@
    class PLCObject(object):
    def __init__(self, WorkingDir, argv, statuschange, evaluator, pyruntimevars):
    - self.workingdir = WorkingDir
    + self.workingdir = WorkingDir # must exits already
    + self.tmpdir = os.path.join(WorkingDir, 'tmp')
    + if os.path.exists(self.tmpdir):
    + shutil.rmtree(self.tmpdir)
    + os.mkdir(self.tmpdir)
    # FIXME : is argv of any use nowadays ?
    self.argv = [WorkingDir] + argv # force argv[0] to be "path" to exec...
    self.statuschange = statuschange
    @@ -91,6 +100,8 @@
    self.TraceLock = Lock()
    self.Traces = []
    + self._init_blobs()
    +
    # First task of worker -> no @RunInMain
    def AutoLoad(self, autostart):
    # Get the last transfered PLC
    @@ -447,8 +458,52 @@
    def GetPLCID(self):
    return getPSKID()
    + def _init_blobs(self):
    + self.blobs = {}
    + if os.path.exists(self.tmpdir):
    + shutil.rmtree(self.tmpdir)
    + os.mkdir(self.tmpdir)
    +
    @RunInMain
    - def NewPLC(self, md5sum, data, extrafiles):
    + def AppendChunkToBlob(self, data, blobID):
    + blob = ((mkstemp(dir=self.tmpdir) if data else None)\
    + + (md5.new(),)) \
    + if blobID == empty_md5_digest \
    + else self.blobs.pop(blobID, None)
    +
    + if blob is None:
    + return None
    +
    + fobj, path, md5sum = blob
    + md5sum.update(data)
    + newBlobID = md5sum.digest()
    + if data:
    + os.write(fobj,data)
    + self.blobs[newBlobID] = blob
    + return newBlobID
    +
    + @RunInMain
    + def PurgeBlobs(self):
    + for fobj, path, md5sum in self.blobs:
    + os.close(fobj)
    + self._init_blobs()
    +
    + def _BlobAsFile(self, blobID, newpath):
    + blob = self.blobs.pop(blobID, None)
    +
    + if blob is None:
    + if blobID == md5.new().digest():
    + # create empty file
    + open(newpath,'r').close()
    + return
    + raise Exception(_("Missing data to create file: {}").format(newpath))
    +
    + fobj, path, md5sum = blob
    + os.close(fobj)
    + shutil.move(path, newpath)
    +
    + @RunInMain
    + def NewPLC(self, md5sum, plc_object, extrafiles):
    if self.PLCStatus in [PlcStatus.Stopped, PlcStatus.Empty, PlcStatus.Broken]:
    NewFileName = md5sum + lib_ext
    extra_files_log = os.path.join(self.workingdir, "extra_files.txt")
    @@ -458,18 +513,13 @@
    else None
    new_PLC_filename = os.path.join(self.workingdir, NewFileName)
    - # Some platform (i.e. Xenomai) don't like reloading same .so file
    - replace_PLC_shared_object = new_PLC_filename != old_PLC_filename
    -
    - if replace_PLC_shared_object:
    - self.UnLoadPLC()
    + self.UnLoadPLC()
    self.LogMessage("NewPLC (%s)" % md5sum)
    self.PLCStatus = PlcStatus.Empty
    try:
    - if replace_PLC_shared_object:
    - os.remove(old_PLC_filename)
    + os.remove(old_PLC_filename)
    for filename in open(extra_files_log, "rt").readlines() + [extra_files_log]:
    try:
    os.remove(os.path.join(self.workingdir, filename.strip()))
    @@ -480,17 +530,16 @@
    try:
    # Create new PLC file
    - if replace_PLC_shared_object:
    - open(new_PLC_filename, 'wb').write(data)
    + self._BlobAsFile(plc_object, new_PLC_filename)
    # Store new PLC filename based on md5 key
    open(self._GetMD5FileName(), "w").write(md5sum)
    # Then write the files
    log = open(extra_files_log, "w")
    - for fname, fdata in extrafiles:
    + for fname, blobID in extrafiles:
    fpath = os.path.join(self.workingdir, fname)
    - open(fpath, "wb").write(fdata)
    + self._BlobAsFile(blobID, fpath)
    log.write(fname+'\n')
    # Store new PLC filename
    @@ -501,9 +550,7 @@
    PLCprint(traceback.format_exc())
    return False
    - if not replace_PLC_shared_object:
    - self.PLCStatus = PlcStatus.Stopped
    - elif self.LoadPLC():
    + if self.LoadPLC():
    self.PLCStatus = PlcStatus.Stopped
    else:
    self.PLCStatus = PlcStatus.Broken
    --- a/runtime/Worker.py Tue Nov 27 13:34:14 2018 +0100
    +++ b/runtime/Worker.py Tue Dec 04 11:31:58 2018 +0100
    @@ -9,6 +9,7 @@
    from __future__ import absolute_import
    import sys
    +import six
    import thread
    from threading import Lock, Condition
    --- a/targets/toolchain_gcc.py Tue Nov 27 13:34:14 2018 +0100
    +++ b/targets/toolchain_gcc.py Tue Dec 04 11:31:58 2018 +0100
    @@ -72,23 +72,20 @@
    """
    return self.CTRInstance.GetTarget().getcontent().getLinker()
    - def GetBinaryCode(self):
    - try:
    - return open(self.exe_path, "rb").read()
    - except Exception:
    - return None
    + def GetBinaryPath(self):
    + return self.bin_path
    def _GetMD5FileName(self):
    return os.path.join(self.buildpath, "lastbuildPLC.md5")
    - def ResetBinaryCodeMD5(self):
    + def ResetBinaryMD5(self):
    self.md5key = None
    try:
    os.remove(self._GetMD5FileName())
    except Exception:
    pass
    - def GetBinaryCodeMD5(self):
    + def GetBinaryMD5(self):
    if self.md5key is not None:
    return self.md5key
    else:
    @@ -100,8 +97,8 @@
    def SetBuildPath(self, buildpath):
    if self.buildpath != buildpath:
    self.buildpath = buildpath
    - self.exe = self.CTRInstance.GetProjectName() + self.extension
    - self.exe_path = os.path.join(self.buildpath, self.exe)
    + self.bin = self.CTRInstance.GetProjectName() + self.extension
    + self.bin_path = os.path.join(self.buildpath, self.bin)
    self.md5key = None
    self.srcmd5 = {}
    @@ -152,9 +149,6 @@
    wholesrcdata += self.concat_deps(CFileName)
    return hashlib.md5(wholesrcdata).hexdigest()
    - def calc_md5(self):
    - return hashlib.md5(self.GetBinaryCode()).hexdigest()
    -
    def build(self):
    # Retrieve compiler and linker
    self.compiler = self.getCompiler()
    @@ -165,7 +159,7 @@
    # ----------------- GENERATE OBJECT FILES ------------------------
    obns = []
    objs = []
    - relink = self.GetBinaryCode() is None
    + relink = not os.path.exists(self.bin_path)
    for Location, CFilesAndCFLAGS, _DoCalls in self.CTRInstance.LocationCFilesAndCFLAGS:
    if CFilesAndCFLAGS:
    if Location:
    @@ -213,14 +207,14 @@
    ALLldflags = ' '.join(self.getBuilderLDFLAGS())
    - self.CTRInstance.logger.write(" [CC] " + ' '.join(obns)+" -> " + self.exe + "\n")
    + self.CTRInstance.logger.write(" [CC] " + ' '.join(obns)+" -> " + self.bin + "\n")
    status, _result, _err_result = ProcessLogger(
    self.CTRInstance.logger,
    "\"%s\" %s -o \"%s\" %s" %
    (self.linker,
    listobjstring,
    - self.exe_path,
    + self.bin_path,
    ALLldflags)
    ).spin()
    @@ -228,10 +222,10 @@
    return False
    else:
    - self.CTRInstance.logger.write(" [pass] " + ' '.join(obns)+" -> " + self.exe + "\n")
    + self.CTRInstance.logger.write(" [pass] " + ' '.join(obns)+" -> " + self.bin + "\n")
    # Calculate md5 key and get data for the new created PLC
    - self.md5key = self.calc_md5()
    + self.md5key = hashlib.md5(open(self.bin_path, "rb").read()).hexdigest()
    # Store new PLC filename based on md5 key
    f = open(self._GetMD5FileName(), "w")
    --- a/targets/toolchain_makefile.py Tue Nov 27 13:34:14 2018 +0100
    +++ b/targets/toolchain_makefile.py Tue Dec 04 11:31:58 2018 +0100
    @@ -47,20 +47,20 @@
    self.buildpath = buildpath
    self.md5key = None
    - def GetBinaryCode(self):
    + def GetBinaryPath(self):
    return None
    def _GetMD5FileName(self):
    return os.path.join(self.buildpath, "lastbuildPLC.md5")
    - def ResetBinaryCodeMD5(self):
    + def ResetBinaryMD5(self):
    self.md5key = None
    try:
    os.remove(self._GetMD5FileName())
    except Exception:
    pass
    - def GetBinaryCodeMD5(self):
    + def GetBinaryMD5(self):
    if self.md5key is not None:
    return self.md5key
    else: