runtime: [Bug] System.Text.Json can't parse Json ( returned from server using Python json.dumps() ) ?

Description

Python server returns the following Json string ( using Python json.dumps() ): “{"code":0,"data":"[{\"Names\":\"[\\\"\\u5f91\\u9a78\\\", \\\"\\u5f25\\u6622\\u6656\\\"]\", \"registry\": null}]"}”

System.Text.Json fails to deserialize it, due to its strict format requirement:
(backward slash)" should be " “[ should be [ ]” should be ]

But doing such string replacement is not safe. json.dumps() is widely used in Python server, so I’m wondering if anyone faces such issues? Are there any ways to directly parse the above Json format, without the need of any custom converter or string replacement?

Reproduction Steps

Please run the attached demo project: maapp.zip

Code clips:

    public string JsonReturnedFromServer { get; set; } = "{\"code\":0,\"data\":\"[{\\\"Names\\\":\\\"[\\\\\\\"\\\\u5f91\\\\u9a78\\\\\\\", \\\\\\\"\\\\u5f25\\\\u6622\\\\u6656\\\\\\\"]\\\", \\\"registry\\\": null}]\"}";
    public string DataStrUseSystemTextJsonServerJson { get; set; }
    public string DataStrUseNewtonsoftServerJson { get; set; }
    public string DataStrUseSystemTextJsonSerialization { get; set; }


    private void UseSystemTextJson(object sender, EventArgs e)
    {
        try
        {
            JsonSerializerOptions options = new JsonSerializerOptions()
            {
                AllowTrailingCommas = true,
                NumberHandling = JsonNumberHandling.AllowReadingFromString
            };
            options.Converters.Add(new StringConverter());
            var dict = System.Text.Json.JsonSerializer.Deserialize<Dictionary<string, object>>(JsonReturnedFromServer, options);
            DataStrUseSystemTextJsonServerJson = dict["data"].ToString();
            // NOT SAFE to do such replacement for injection / parsing failure possibilities
            string parsedData = DataStrUseSystemTextJsonServerJson.Replace("\\\"", "\"").Replace("\"[", "[").Replace("]\"", "]");
            var dat = System.Text.Json.JsonSerializer.Deserialize<ObservableCollection<MyPoco>>(parsedData);
            Console.WriteLine(dat.Count);
        }
        catch (Exception ex)
        { Toast.Make(ex.Message).Show(); }
    }

    private void UseNewtonsoft(object sender, EventArgs e)
    {
        try
        {
            var dict = JsonConvert.DeserializeObject<Dictionary<string, string>>(JsonReturnedFromServer);
            DataStrUseNewtonsoftServerJson = dict["data"].ToString();
            var dat = JsonConvert.DeserializeObject<ObservableCollection<MyPoco>>(DataStrUseNewtonsoftServerJson);
            Console.WriteLine(dat.Count);
        }
        catch (Exception ex)
        { Toast.Make(ex.Message).Show(); }
    }

    private void SystemTextJsonTest(object sender, EventArgs e)
    {
        try
        {
            var value = new ObservableCollection<MyPoco>() { new MyPoco() };
            Dictionary<string, object> data = new Dictionary<string, object>() { { "code", 0 }, { "data", value } };

            JsonSerializerOptions options = new JsonSerializerOptions()
            {
                AllowTrailingCommas = true,
                NumberHandling = JsonNumberHandling.AllowReadingFromString
            };
            //options.Converters.Add(new StringConverter());
            string json = System.Text.Json.JsonSerializer.Serialize(data, options);
            Console.WriteLine(json);

            var dict = System.Text.Json.JsonSerializer.Deserialize<Dictionary<string, object>>(json, options);
            DataStrUseSystemTextJsonSerialization = dict["data"].ToString();
            var dat = System.Text.Json.JsonSerializer.Deserialize<ObservableCollection<MyPoco>>(DataStrUseSystemTextJsonSerialization);
            Console.WriteLine(dat.Count);
        }
        catch (Exception ex)
        { Toast.Make(ex.Message).Show(); }
    }
}

public class MyPoco
{
    //[System.Text.Json.Serialization.JsonConverter(typeof(ObservableConverter<string>))]
    public ObservableCollection<string> Names { get; set; } = new ObservableCollection<string> { "徑驸", "弥昢晖" };
}

public class ObservableConverter<T> : System.Text.Json.Serialization.JsonConverter<ObservableCollection<T>>
{
    public override ObservableCollection<T> Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
    {
        string str = reader.GetString();
        str = str != "" ? str : "[]";
        List<T> ls = System.Text.Json.JsonSerializer.Deserialize<List<T>>(str);
        ObservableCollection<T> obs = new ObservableCollection<T>(ls);
        return obs;
    }

    public override void Write(Utf8JsonWriter writer, ObservableCollection<T> value, JsonSerializerOptions options)
    {
        writer.WriteStringValue(System.Text.Json.JsonSerializer.Serialize(((value as ObservableCollection<T>).ToList())));
    }
}


public class StringConverter : System.Text.Json.Serialization.JsonConverter<string>
{
    public override string Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
    {

        if (reader.TokenType == JsonTokenType.Number)
        {
            var stringValue = reader.GetInt32();
            return stringValue.ToString();
        }
        else if (reader.TokenType == JsonTokenType.String)
        {
            return reader.GetString();
        }
        else if (reader.TokenType == JsonTokenType.StartArray)
        {
            return reader.GetString();
        }

        throw new System.Text.Json.JsonException();
    }

    public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options)
    {
        writer.WriteStringValue(value);

    }

}

Expected behavior

Such Json string from Python server should be able to be parsed directly, without any manual parsing of the original string.

Actual behavior

Custom converter needs to be added on each and every ObservableCollection<T> properties.

Regression?

No response

Known Workarounds

No response

Configuration

.Net 7 Windows 11 preview beta x64

Other information

No response

About this issue

  • Original URL
  • State: closed
  • Created 2 years ago
  • Comments: 24 (14 by maintainers)

Most upvoted comments

It seems to me that the actual problem is that the core data is triple (!) JSON encoded. The proper way to read it is:

using System.Text.Json;

string jsonReturnedFromServer = "{\"code\":0,\"data\":\"[{\\\"Names\\\":\\\"[\\\\\\\"\\\\u5f91\\\\u9a78\\\\\\\", \\\\\\\"\\\\u5f25\\\\u6622\\\\u6656\\\\\\\"]\\\", \\\"registry\\\": null}]\"}";

var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };

var serverResponse = JsonSerializer.Deserialize<ServerResponse>(jsonReturnedFromServer, options);

var data = JsonSerializer.Deserialize<DataItem[]>(serverResponse.Data, options);

foreach (DataItem item in data)
{
    var names = JsonSerializer.Deserialize<string[]>(item.Names, options);
    foreach (string name in names)
    {
        Console.WriteLine(name);
    }
    Console.WriteLine();
}

record ServerResponse(int Code, string Data);
record DataItem(string Names, object Registry);