HGKeeper

beremiz

MQTT: add JSON payload corresponding to IEC Structured types

21 months ago, Edouard Tisserant

6337c9c2c379

Parents fdbb3c4ebbf6
Children 7f4226b2f867

MQTT: add JSON payload corresponding to IEC Structured types

- uses https://github.com/cesanta/frozen
- frozen.c is added as a resulting c file in generat_C (fix toolchain_gcc.py as it was breaking dependency checking)
- structured types are converted to C #define, in turn generating json_scanf and json_printf statements

4 files changed, 188 insertions(+), 30 deletions(-)

+15 -5

mqtt/client.py

+73 -6

mqtt/mqtt_client_gen.py

+98 -19

mqtt/mqtt_template.c

+2 -0

targets/toolchain_gcc.py

~~--- a/mqtt/client.py Thu Sep 05 15:55:46 2024 +0200~~

+++ b/mqtt/client.py Fri Sep 13 14:26:55 2024 +0200

@@ -16,9 +16,12 @@

# and cmake build was invoked from this directory

PahoMqttCLibraryPath = paths.ThirdPartyPath("paho.mqtt.c", "build", "src")

~~-PahoMqttCIncludePaths = [~~

+frozen_path = paths.ThirdPartyPath("frozen")

+MqttCIncludePaths = [

paths.ThirdPartyPath("paho.mqtt.c", "build"), # VersionInfo.h

~~- paths.ThirdPartyPath("paho.mqtt.c", "src")~~

+ paths.ThirdPartyPath("paho.mqtt.c", "src"),

+ frozen_path

]

class MQTTClientEditor(ConfTreeNodeEditor):

@@ -94,6 +97,10 @@

datatype_candidates = self.GetCTRoot().GetDataTypes()

return datatype_candidates

+ def GetDataTypeInfos(self, typename):

+ tagname = "D::"+typename

+ return self.GetCTRoot().GetDataTypeInfos(tagname)

def GetConfig(self):

def cfg(path):

try:

@@ -150,7 +157,7 @@

#include "beremiz.h"

"""

config = self.GetConfig()

~~- c_code += self.modeldata.GenerateC(c_path, locstr, config)~~

+ c_code += self.modeldata.GenerateC(c_path, locstr, config, self.GetDataTypeInfos)

with open(c_path, 'w') as c_file:

c_file.write(c_code)

@@ -164,9 +171,12 @@

LDFLAGS = [' "' + os.path.join(PahoMqttCLibraryPath, static_lib) + '"'] + libs

~~- CFLAGS = ' '.join(['-I"' + path + '"' for path in PahoMqttCIncludePaths])~~

+ CFLAGS = ' '.join(['-I"' + path + '"' for path in MqttCIncludePaths])

~~- return [(c_path, CFLAGS)], LDFLAGS, True~~

+ # TODO: add frozen only if using JSON

+ frozen_c_path = os.path.join(frozen_path, "frozen.c")

+ return [(c_path, CFLAGS), (frozen_c_path, CFLAGS)], LDFLAGS, True

def GetVariableLocationTree(self):

current_location = self.GetCurrentLocation()

~~--- a/mqtt/mqtt_client_gen.py Thu Sep 05 15:55:46 2024 +0200~~

+++ b/mqtt/mqtt_client_gen.py Fri Sep 13 14:26:55 2024 +0200

@@ -4,7 +4,7 @@

import csv

import functools

from threading import Thread

~~-from collections import OrderedDict~~

+from collections import OrderedDict as OD

import wx

import wx.dataview as dv

@@ -336,12 +336,14 @@

for row in data:

writer.writerow([direction] + row)

~~- def GenerateC(self, path, locstr, config):~~

+ def GenerateC(self, path, locstr, config, datatype_info_getter):

c_template_filepath = paths.AbsNeighbourFile(__file__, "mqtt_template.c")

c_template_file = open(c_template_filepath , 'rb')

c_template = c_template_file.read()

c_template_file.close()

+ json_types = OD()

formatdict = dict(

locstr = locstr,

uri = config["URI"],

@@ -353,7 +355,8 @@

init_pubsub = "",

retrieve = "",

publish = "",

~~- publish_changes = ""~~

+ publish_changes = "",

+ json_decl = ""

)

@@ -384,19 +387,22 @@

if iec_type in MQTT_IEC_types:

C_type, iec_size_prefix = MQTT_IEC_types[iec_type]

c_loc_name = "__Q" + iec_size_prefix + locstr + "_" + str(iec_number)

+ encoding = "SIMPLE"

else:

C_type = iec_type.upper();

c_loc_name = "__Q" + locstr + "_" + str(iec_number)

+ json_types.setdefault(iec_type,OD()).setdefault("OUTPUT",[]).append(c_loc_name)

+ encoding = "JSON"

formatdict["decl"] += """

DECL_VAR({iec_type}, {C_type}, {c_loc_name})""".format(**locals())

formatdict["init_pubsub"] += """

~~- INIT_PUBLICATION({Topic}, {QoS}, {C_type}, {c_loc_name}, {Retained})""".format(**locals())~~

+ INIT_PUBLICATION({encoding}, {Topic}, {QoS}, {C_type}, {c_loc_name}, {Retained})""".format(**locals())

formatdict["publish"] += """

WRITE_VALUE({c_loc_name}, {C_type})""".format(**locals())

formatdict["publish_changes"] += """

~~- PUBLISH_CHANGE({Topic}, {QoS}, {C_type}, {c_loc_name}, {Retained})""".format(**locals())~~

+ PUBLISH_CHANGE({encoding}, {Topic}, {QoS}, {C_type}, {c_loc_name}, {Retained})""".format(**locals())

# inputs need to be sorted for bisection search

for row in sorted(self["input"]):

@@ -404,19 +410,80 @@

if iec_type in MQTT_IEC_types:

C_type, iec_size_prefix = MQTT_IEC_types[iec_type]

c_loc_name = "__I" + iec_size_prefix + locstr + "_" + str(iec_number)

+ init_topic_call = "INIT_TOPIC"

else:

C_type = iec_type.upper();

c_loc_name = "__I" + locstr + "_" + str(iec_number)

+ init_topic_call = "INIT_JSON_TOPIC"

+ json_types.setdefault(iec_type,OD()).setdefault("INPUT",[]).append(c_loc_name)

formatdict["decl"] += """

DECL_VAR({iec_type}, {C_type}, {c_loc_name})""".format(**locals())

formatdict["topics"] += """

~~- INIT_TOPIC({Topic}, {iec_type}, {c_loc_name})""".format(**locals())~~

+ {init_topic_call}({Topic}, {iec_type}, {c_loc_name})""".format(**locals())

formatdict["init_pubsub"] += """

INIT_SUBSCRIPTION({Topic}, {QoS})""".format(**locals())

formatdict["retrieve"] += """

READ_VALUE({c_loc_name}, {C_type})""".format(**locals())

+ def recurseJsonTypes(datatype, basetypes):

+ basetypes.append(datatype)

+ # add derivated type first fo we can expect the list to be sorted

+ # with base types in last position

+ infos = datatype_info_getter(datatype)

+ for element in infos["elements"]:

+ field_datatype = element["Type"]

+ if field_datatype not in MQTT_IEC_types:

+ recurseJsonTypes(field_datatype, basetypes)

+ print(json_types)

+ # collect all type dependencies

+ basetypes=[] # use a list to keep them ordered

+ for iec_type,_instances in json_types.items():

+ recurseJsonTypes(iec_type, basetypes)

+ done_types = set()

+ # go backard to get most derivated type definition last

+ # so that CPP can always find base type deinition before

+ for iec_type in reversed(basetypes):

+ # avoid repeating type definition

+ if iec_type in done_types:

+ continue

+ done_types.add(iec_type)

+ C_type = iec_type.upper()

+ json_decl = "#define TYPE_" + C_type + "(_P, _A) \\\n"

+ infos = datatype_info_getter(iec_type)

+ elements = infos["elements"]

+ last = len(elements) - 1

+ for idx, element in enumerate(elements):

+ field_iec_type = element["Type"]

+ field_C_type = field_iec_type.upper()

+ field_name = element["Name"].upper()

+ if field_iec_type in MQTT_IEC_types:

+ decl_type = "SIMPLE"

+ else:

+ decl_type = "OBJECT"

+ json_decl += " _P##_"+decl_type+"(" + field_C_type + ", " + field_name + ", _A)"

+ if idx != last:

+ json_decl += " _P##_separator \\"

+ else:

+ json_decl += "\n"

+ formatdict["json_decl"] += json_decl

+ for iec_type, instances in json_types.items():

+ C_type = iec_type.upper()

+ for direction, instance_list in instances.items():

+ for c_loc_name in instance_list:

+ formatdict["json_decl"] += "DECL_JSON_"+direction+"("+C_type+", "+c_loc_name+")\n"

Ccode = c_template.format(**formatdict)

return Ccode

~~--- a/mqtt/mqtt_template.c Thu Sep 05 15:55:46 2024 +0200~~

+++ b/mqtt/mqtt_template.c Fri Sep 13 14:26:55 2024 +0200

@@ -5,10 +5,15 @@

#include <pthread.h>

#include <string.h>

#include <stdio.h>

+#include <errno.h>

+#include "frozen.h"

#include "MQTTClient.h"

#include "MQTTClientPersistence.h"

+#include "POUS.h"

#define _Log(level, ...) \

{{ \

char mstr[256]; \

@@ -42,13 +47,62 @@

#define UNCHANGED 0

#define DECL_VAR(iec_type, C_type, c_loc_name) \

~~-static C_type PLC_##c_loc_name##_buf = 0; \~~

~~-static C_type MQTT_##c_loc_name##_buf = 0; \~~

+static C_type PLC_##c_loc_name##_buf; \

+static C_type MQTT_##c_loc_name##_buf; \

static int MQTT_##c_loc_name##_state = UNCHANGED; /* systematically published at init */ \

C_type *c_loc_name = &PLC_##c_loc_name##_buf;

{decl}

+/* JSON topic content encoding macros matching "json_decl" in substitution*/

+#define format_BOOL "%B"

+#define format_SINT "%hhd"

+#define format_USINT "%uhhd"

+#define format_INT "%hd"

+#define format_UINT "%uhd"

+#define format_DINT "%d"

+#define format_UDINT "%ud"

+#define format_LINT "%ld"

+#define format_ULINT "%uld"

+#define format_REAL "%f"

+#define format_LREAL "%Lf"

+#define format_STRING "%*s"

+#define format_separator ", "

+#define format_SIMPLE(C_type, name, _A) #name " : " format_##C_type

+#define format_OBJECT(C_type, name, _A) #name " : {{ " TYPE_##C_type(format, _A) " }}"

+#define arg_separator ,

+#define arg_SIMPLE(C_type, name, data_ptr) data_ptr->name

+#define arg_OBJECT(C_type, name, data_ptr) TYPE_##C_type(arg, (&data_ptr->name))

+#define DECL_JSON_INPUT(C_type, c_loc_name) \

+int json_parse_##c_loc_name(char *json, const int len, void *void_ptr) {{ \

+ C_type *struct_ptr = (C_type *)void_ptr; \

+ return json_scanf(json, len, "{{" TYPE_##C_type(format,) "}}", TYPE_##C_type(arg, struct_ptr)); \

+}}

+/* Pre-allocated json output buffer for json_printf */

+#define json_out_size 1<<12 // 4K

+static char json_out_buf[json_out_size] = {{0,}};

+static int json_out_len = 0;

+#define DECL_JSON_OUTPUT(C_type, c_loc_name) \

+int json_gen_##c_loc_name(C_type *struct_ptr) {{ \

+ struct json_out out = JSON_OUT_BUF(json_out_buf, json_out_size); \

+ json_out_len = json_printf(&out, "{{" TYPE_##C_type(format,) "}}", TYPE_##C_type(arg, struct_ptr)); \

+ if(json_out_len > json_out_size){{ \

+ json_out_len = 0; \

+ return -EOVERFLOW; \

+ }} \

+ return 0; \

+}}

+{json_decl}

static MQTTClient client;

#ifdef USE_MQTT_5

static MQTTClient_connectOptions conn_opts = MQTTClient_connectOptions_initializer5;

@@ -79,14 +133,23 @@

/* thread that handles publish and reconnection */

static pthread_t MQTT_thread;

~~-#define INIT_TOPIC(topic, iec_type, c_loc_name) \~~

~~-{{#topic, &MQTT_##c_loc_name##_buf, &MQTT_##c_loc_name##_state, iec_type##_ENUM}},~~

+#define INIT_TOPIC(topic, iec_type, c_loc_name) \

+{{#topic, &MQTT_##c_loc_name##_buf, &MQTT_##c_loc_name##_state, 0, .vartype = iec_type##_ENUM}},

+#define INIT_JSON_TOPIC(topic, iec_type, c_loc_name) \

+{{#topic, &MQTT_##c_loc_name##_buf, &MQTT_##c_loc_name##_state, 1, .json_parse_func=json_parse_##c_loc_name}},

+typedef int (*json_parse_func_t)(char *json, int len, void *void_ptr);

static struct {{

const char *topic; //null terminated topic string

void *mqtt_pdata; // pointer to data from/for MQTT stack

int *mqtt_pchanged; // pointer to changed flag

~~- __IEC_types_enum vartype;~~

+ int is_json_type;

+ union {{

+ __IEC_types_enum vartype;

+ json_parse_func_t json_parse_func;

+ }};

}} topics [] = {{

{topics}

}};

@@ -135,6 +198,7 @@

int size = sizeof(topics) / sizeof(topics[0]);

int high = size - 1;

int mid;

+ int is_json_type;

// bisect topic among subscribed topics

while (low <= high) {{

@@ -159,9 +223,15 @@

goto exit;

found:

~~- if(__get_type_enum_size(topics[mid].vartype) == message->payloadlen){{~~

+ is_json_type = topics[mid].is_json_type;

+ if(is_json_type || __get_type_enum_size(topics[mid].vartype) == message->payloadlen){{

if (pthread_mutex_lock(&MQTT_retrieve_mutex) == 0){{

~~- memcpy(topics[mid].mqtt_pdata, (char*)message->payload, message->payloadlen);~~

+ if(is_json_type){{

+ (topics[mid].json_parse_func)((char*)message->payload, message->payloadlen, topics[mid].mqtt_pdata);

+ }} else {{

+ memcpy(topics[mid].mqtt_pdata, (char*)message->payload, message->payloadlen);

+ }}

*topics[mid].mqtt_pchanged = 1;

pthread_mutex_unlock(&MQTT_retrieve_mutex);

}}

@@ -179,7 +249,7 @@

LogInfo("MQTT Init no auth\n");

#define INIT_x509(Verify, KeyStore, TrustStore) \

~~- LogInfo("MQTT Init x509 with %s,%s\n", KeyStore, TrustStore) \~~

+ LogInfo("MQTT Init x509 with %s,%s\n", KeyStore?KeyStore:"NULL", TrustStore?TrustStore:"NULL")\

ssl_opts.verify = Verify; \

ssl_opts.keyStore = KeyStore; \

ssl_opts.trustStore = TrustStore; \

@@ -220,21 +290,30 @@

#ifdef USE_MQTT_5

~~-#define _PUBLISH(Topic, QoS, C_type, c_loc_name, Retained) \~~

~~- MQTTResponse response = MQTTClient_publish5(client, #Topic, sizeof(C_type), \~~

~~- &MQTT_##c_loc_name##_buf, QoS, Retained, NULL, NULL); \~~

+#define _PUBLISH(Topic, QoS, cstring_size, cstring_ptr, Retained) \

+ MQTTResponse response = MQTTClient_publish5(client, #Topic, cstring_size, \

+ cstring_ptr, QoS, Retained, NULL, NULL); \

rc = response.reasonCode; \

MQTTResponse_free(response);

#else

~~-#define _PUBLISH(Topic, QoS, C_type, c_loc_name, Retained) \~~

~~- rc = MQTTClient_publish(client, #Topic, sizeof(C_type), \~~

~~- &PLC_##c_loc_name##_buf, QoS, Retained, NULL);~~

+#define _PUBLISH(Topic, QoS, cstring_size, cstring_ptr, Retained) \

+ rc = MQTTClient_publish(client, #Topic, cstring_size, \

+ cstring_ptr, QoS, Retained, NULL);

#endif

~~-#define INIT_PUBLICATION(Topic, QoS, C_type, c_loc_name, Retained) \~~

+#define PUBLISH_SIMPLE(Topic, QoS, C_type, c_loc_name, Retained) \

+ _PUBLISH(Topic, QoS, sizeof(C_type), &MQTT_##c_loc_name##_buf, Retained)

+#define PUBLISH_JSON(Topic, QoS, C_type, c_loc_name, Retained) \

+ int res = json_gen_##c_loc_name(&MQTT_##c_loc_name##_buf); \

+ if(res == 0) {{ \

+ _PUBLISH(Topic, QoS, json_out_len, json_out_buf, Retained) \

+ }}

+#define INIT_PUBLICATION(encoding, Topic, QoS, C_type, c_loc_name, Retained) \

{{ \

int rc; \

~~- _PUBLISH(Topic, QoS, C_type, c_loc_name, Retained) \~~

+ PUBLISH_##encoding(Topic, QoS, C_type, c_loc_name, Retained) \

if (rc != MQTTCLIENT_SUCCESS) \

{{ \

LogError("MQTT client failed to init publication of '%s', return code %d\n", #Topic, rc);\

@@ -242,11 +321,11 @@

}} \

}}

~~-#define PUBLISH_CHANGE(Topic, QoS, C_type, c_loc_name, Retained) \~~

+#define PUBLISH_CHANGE(encoding, Topic, QoS, C_type, c_loc_name, Retained) \

if(MQTT_##c_loc_name##_state == CHANGED) \

{{ \

int rc; \

~~- _PUBLISH(Topic, QoS, C_type, c_loc_name, Retained) \~~

+ PUBLISH_##encoding(Topic, QoS, C_type, c_loc_name, Retained) \

if (rc != MQTTCLIENT_SUCCESS) \

{{ \

LogError("MQTT client failed to publish '%s', return code %d\n", #Topic, rc); \

@@ -399,7 +478,7 @@

#define WRITE_VALUE(c_loc_name, C_type) \

/* TODO care about endianess */ \

~~- if(MQTT_##c_loc_name##_buf != PLC_##c_loc_name##_buf){{ \~~

+ if(memcmp(&MQTT_##c_loc_name##_buf, &PLC_##c_loc_name##_buf, sizeof(C_type))){{ \

MQTT_##c_loc_name##_buf = PLC_##c_loc_name##_buf; \

MQTT_##c_loc_name##_state = CHANGED; \

MQTT_any_pub_var_changed = 1; \

~~--- a/targets/toolchain_gcc.py Thu Sep 05 15:55:46 2024 +0200~~

+++ b/targets/toolchain_gcc.py Fri Sep 13 14:26:55 2024 +0200

@@ -149,6 +149,8 @@

oldhash, deps = self.srcmd5.get(bn, (None, []))

# read source

src = os.path.join(self.buildpath, bn)

+ if not os.path.exists(src):

+ return False

# compute new hash

newhash = compute_file_md5(src)

# compare