input UTF-8 values are transformed to unicode code point

[lmussier]

Hi,

We have encountered a strange behavior when using messages for things. When sending a message through either the sandbox or the HTTP API, any UTF-8 character in the message is being transformed into what appears to be its Unicode code point.

How to reproduce?

Send a message to a thing/feature with "こんにちは" as the payload, with the content type set to application/json or text/plain.
Nothe this also true for keys in a json object.

props: [ContentType : application/vnd.eclipse.ditto+json, CorrelationData : b'89dde36c-a5cc-4b44-996a-df36604cfc39', UserProperty : [('subject', 'hello'), ('response-required', 'true')]]
payload: 
 {
  "topic": "my.sensors/sensor01/things/live/messages/hello",
  "headers": {
    "version": 2,
    "sec-fetch-mode": "cors",
    "referer": "http://localhost:8080/apidoc/?url=/apidoc/openapi/ditto-api-2.yml",
    "x-ditto-pre-authenticated": "nginx:ditto",
    "sec-fetch-site": "same-origin",
    "accept-language": "en-US, en;q=0.5",
    "origin": "http://localhost:8080",
    "x-forwarded-for": "172.22.0.1",
    "accept": "*/*",
    "authorization": "Basic ZGl0dG86ZGl0dG8=",
    "x-real-ip": "172.22.0.1",
    "x-forwarded-user": "ditto",
    "host": "localhost:8080",
    "sec-fetch-dest": "empty",
    "user-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0",
    "channel": "live",
    "timeout": "10",
    "response-required": true,
    "ditto-originator": "nginx:ditto",
    "requested-acks": [],
    "ditto-message-direction": "TO",
    "ditto-message-subject": "hello",
    "ditto-message-thing-id": "my.sensors:sensor01",
    "content-type": "text/plain",
    "timestamp": "2023-08-02T10:24:30.467036546+02:00",
    "correlation-id": "89dde36c-a5cc-4b44-996a-df36604cfc39"
  },
  "path": "/inbox/messages/hello",
  "value": "\u3053\u3093\u306b\u3061\u306f"
}

[lmussier]

Some additionnal tests. I got it working like like intended from a thing to ditto (exactly from a websocket listener).
The key was to set content type inside the enveloppe and not as the MQTT content type.

        if json_payload['headers']['response-required']:
            logger.info(f"Answering back: こんにちは, I don't know I am only a coffee machine")

            ditto_msg = {
                "headers": {
                    "content-type": "application/json"
                },
                "topic": f"{json_payload['topic']}",
                "path": f"{json_payload['path'].replace('inbox','outbox')}",
                "status": 418,
                "value": "こんにちは, I don't know I am only a coffee machine"
            }

            publish_properties = Properties(PacketTypes.PUBLISH)
            publish_properties.CorrelationData = message.properties.CorrelationData
            publish_properties.ContentType = "application/vnd.eclipse.ditto+json"
            
            self.client.publish(json_payload['topic'], payload=json.dumps(ditto_msg),  properties=publish_properties)

Doing the same thing on websocket side leads to a different behaviour , we still have the unicode point in the value.

import WebSocket from 'ws';

// connect to the WebSocket
const ws = new WebSocket('ws://ditto:ditto@localhost:8080/ws/2');

const ditto_msg = {
    "topic": "se.etp.efr.sensors/sensor01/things/live/messages/hello",
    "headers": {
        "correlation-id": "my correlation id",
        "content-type": "application/json",
    },
    "path": "/inbox/messages/hello",
    "value": "こんにちは"
};

ws.on('open', function open() {
    ws.send('START-SEND-MESSAGES');
});

ws.on('message', function message(data) {
  if ( data == "START-SEND-MESSAGES:ACK") {
    console.log(' < received:\n %s', data);
    console.log(' > Sending :\n %s', JSON.stringify(ditto_msg, null, 2));
    ws.send(JSON.stringify(ditto_msg))
  } else {
    console.log(' < received:\n %s', JSON.stringify(JSON.parse(data), null, 2));
  }
});

ws.on('error', console.error);