beremiz

Reworked chunk based transfer to support duplicated files (i.e. files with same content, but different names)

2019-02-13, Edouard Tisserant
6a4f9a061994
Parents 44c2a4e2b84d
Children 889c43872f4c
Reworked chunk based transfer to support duplicated files (i.e. files with same content, but different names)
4 files changed, 22 insertions(+), 19 deletions(-)
  • +5 -2
    ProjectController.py
  • +4 -4
    connectors/ConnectorBase.py
  • +12 -13
    runtime/PLCObject.py
  • +1 -0
    runtime/WampClient.py
    • --- a/ProjectController.py Fri Feb 01 14:14:13 2019 +0100
    +++ b/ProjectController.py Wed Feb 13 09:41:35 2019 +0100
    @@ -1873,11 +1873,14 @@
    extrafiles.append((
    name,
    self._connector.BlobFromFile(
    - os.path.join(extrafilespath, name))))
    + # use file name as a seed to avoid collisions
    + # with files having same content
    + os.path.join(extrafilespath, name),name)))
    # Send PLC on target
    object_path = builder.GetBinaryPath()
    - object_blob = self._connector.BlobFromFile(object_path)
    + # arbitrarily use MD5 as a seed, could be any string
    + object_blob = self._connector.BlobFromFile(object_path, MD5)
    self.HidePLCProgress()
    --- a/connectors/ConnectorBase.py Fri Feb 01 14:14:13 2019 +0100
    +++ b/connectors/ConnectorBase.py Wed Feb 13 09:41:35 2019 +0100
    @@ -9,14 +9,14 @@
    #chuncksize = 16384
    chuncksize = 1024*1024
    - def BlobFromFile(self, filepath):
    + def BlobFromFile(self, filepath, seed):
    s = md5.new()
    - blobID = s.digest() # empty md5, to support empty blob
    + s.update(seed)
    + blobID = self.SeedBlob(seed)
    with open(filepath, "rb") as f:
    - while True:
    + while blobID == s.digest():
    chunk = f.read(self.chuncksize)
    if len(chunk) == 0: return blobID
    blobID = self.AppendChunkToBlob(chunk, blobID)
    s.update(chunk)
    - if blobID != s.digest(): return None
    --- a/runtime/PLCObject.py Fri Feb 01 14:14:13 2019 +0100
    +++ b/runtime/PLCObject.py Wed Feb 13 09:41:35 2019 +0100
    @@ -45,8 +45,6 @@
    from runtime import PlcStatus
    from runtime import MainWorker
    -empty_md5_digest = md5.new().digest()
    -
    if os.name in ("nt", "ce"):
    dlopen = _ctypes.LoadLibrary
    dlclose = _ctypes.FreeLibrary
    @@ -468,11 +466,17 @@
    os.mkdir(self.tmpdir)
    @RunInMain
    + def SeedBlob(self, seed):
    + blob = (mkstemp(dir=self.tmpdir) + (md5.new(),))
    + fobj, path, md5sum = blob
    + md5sum.update(seed)
    + newBlobID = md5sum.digest()
    + self.blobs[newBlobID] = blob
    + return newBlobID
    +
    + @RunInMain
    def AppendChunkToBlob(self, data, blobID):
    - blob = ((mkstemp(dir=self.tmpdir) if data else None)\
    - + (md5.new(),)) \
    - if blobID == empty_md5_digest \
    - else self.blobs.pop(blobID, None)
    + blob = self.blobs.pop(blobID, None)
    if blob is None:
    return None
    @@ -480,9 +484,8 @@
    fobj, path, md5sum = blob
    md5sum.update(data)
    newBlobID = md5sum.digest()
    - if data:
    - os.write(fobj,data)
    - self.blobs[newBlobID] = blob
    + os.write(fobj,data)
    + self.blobs[newBlobID] = blob
    return newBlobID
    @RunInMain
    @@ -495,10 +498,6 @@
    blob = self.blobs.pop(blobID, None)
    if blob is None:
    - if blobID == md5.new().digest():
    - # create empty file
    - open(newpath,'r').close()
    - return
    raise Exception(_("Missing data to create file: {}").format(newpath))
    fobj, path, md5sum = blob
    --- a/runtime/WampClient.py Fri Feb 01 14:14:13 2019 +0100
    +++ b/runtime/WampClient.py Wed Feb 13 09:41:35 2019 +0100
    @@ -57,6 +57,7 @@
    ("StopPLC", {}),
    ("GetPLCstatus", {}),
    ("GetPLCID", {}),
    + ("SeedBlob", {}),
    ("AppendChunkToBlob", {}),
    ("PurgeBlobs", {}),
    ("NewPLC", {}),