Skip to content

Reject non-ASCII BINARY strings during JSON generation #1098

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
byroot merged 1 commit into from
Oct 6, 2026
Merged

Reject non-ASCII BINARY strings during JSON generation #1098

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 1 addition & 16 deletions ext/json/ext/generator/generator.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ static void generate_json_bignum(FBuffer *buffer, struct generate_json_data *dat
static void generate_json_float(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
static void generate_json_fragment(FBuffer *buffer, struct generate_json_data *data, VALUE obj);

static int usascii_encindex, utf8_encindex, binary_encindex;
static int usascii_encindex, utf8_encindex;

NORETURN(static void) raise_generator_error_str(VALUE invalid_object, VALUE str)
{
Expand Down Expand Up @@ -865,20 +865,6 @@ NOINLINE(static) VALUE convert_invalid_encoding(struct generate_json_data *data,
}
}

if (RB_ENCODING_GET_INLINED(str) == binary_encindex) {
VALUE utf8_string = rb_enc_associate_index(rb_str_dup(str), utf8_encindex);
switch (rb_enc_str_coderange(utf8_string)) {
case ENC_CODERANGE_7BIT:
return utf8_string;
case ENC_CODERANGE_VALID:
// For historical reason, we silently reinterpret binary strings as UTF-8 if it would work.
// TODO: Raise in 3.0.0
rb_warn("JSON.generate: UTF-8 string passed as BINARY, this will raise an encoding error in json 3.0");
return utf8_string;
break;
}
}

return rb_rescue(encode_json_string_try, str, encode_json_string_rescue, str);
}

Expand Down Expand Up @@ -2140,7 +2126,6 @@ void Init_generator(void)

usascii_encindex = rb_usascii_encindex();
utf8_encindex = rb_utf8_encindex();
binary_encindex = rb_ascii8bit_encindex();

rb_require("json/ext/generator/state");

Expand Down
21 changes: 2 additions & 19 deletions java/src/json/ext/StringEncoder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
import java.nio.charset.StandardCharsets;

import org.jcodings.Encoding;
import org.jcodings.specific.ASCIIEncoding;
import org.jcodings.specific.USASCIIEncoding;
import org.jcodings.specific.UTF8Encoding;
import org.jruby.Ruby;
Expand Down Expand Up @@ -229,28 +228,12 @@ static RubyString ensureValidEncoding(ThreadContext context, RubyString str) {
return str;
}

return tryWeirdEncodings(context, str, encoding);
return tryWeirdEncodings(context, str);
}

private static RubyString tryWeirdEncodings(ThreadContext context, RubyString str, Encoding encoding) {
private static RubyString tryWeirdEncodings(ThreadContext context, RubyString str) {
Ruby runtime = context.runtime;

RubyString utf8String;

if (encoding == ASCIIEncoding.INSTANCE) {
utf8String = str.strDup(runtime);
utf8String.setEncoding(UTF8Encoding.INSTANCE);
switch (utf8String.getCodeRange()) {
case StringSupport.CR_7BIT:
return utf8String;
case StringSupport.CR_VALID:
// For historical reason, we silently reinterpret binary strings as UTF-8 if it would work.
// TODO: Raise in 3.0.0
runtime.getWarnings().warn("JSON.generate: UTF-8 string passed as BINARY, this will raise an encoding error in json 3.0");
return utf8String;
}
}

try {
str = (RubyString) str.encode(context, runtime.getEncodingService().convertEncodingToRubyEncoding(UTF8Encoding.INSTANCE));
} catch (RaiseException re) {
Expand Down
29 changes: 16 additions & 13 deletions test/json/json_generator_test.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1049,22 +1049,25 @@ def to_s
end
end

if defined?(JSON::Ext::Generator) and RUBY_PLATFORM != "java"
def test_valid_utf8_in_different_encoding
utf8_string = "€™"
wrong_encoding_string = utf8_string.b
# This behavior is historical. Not necessary desirable. We should deprecated it.
# The pure and java version of the gem already don't behave this way.
assert_warning(/UTF-8 string passed as BINARY, this will raise an encoding error in json 3.0/) do
assert_equal utf8_string.to_json, wrong_encoding_string.to_json
end

assert_warning(/UTF-8 string passed as BINARY, this will raise an encoding error in json 3.0/) do
assert_equal JSON.dump(utf8_string), JSON.dump(wrong_encoding_string)
end
def test_valid_utf8_in_binary_encoding
string = "€™".b.freeze
assert_raise(JSON::GeneratorError) { string.to_json }
assert_raise(JSON::GeneratorError) { JSON.dump(string) }
[string, [string], { string => 1 }, { value: string }].each do |object|
error = assert_raise(JSON::GeneratorError) { JSON.generate(object) }
assert_same string, error.invalid_object
assert_kind_of Encoding::UndefinedConversionError, error.cause
end
end

def test_ascii_in_binary_encoding
string = "ascii".b
assert_equal '"ascii"', string.to_json
assert_equal '"ascii"', JSON.dump(string)
assert_equal '["ascii"]', JSON.generate([string])
assert_equal '{"ascii":1}', JSON.generate(string => 1)
end

def test_nonutf8_encoding
assert_equal("\"5\u{b0}\"", "5\xb0".dup.force_encoding(Encoding::ISO_8859_1).to_json)
end
Expand Down
Loading