blob: 1f3e6db18d77f230b1e2e7230d1c0978dcd3374b [file] [log] [blame] [edit]
// Copyright 2011 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "base/strings/utf_offset_string_conversions.h"
#include <stddef.h>
#include <algorithm>
#include "base/strings/string_piece.h"
#include "testing/gtest/include/gtest/gtest.h"
namespace gurl_base {
namespace {
static const size_t kNpos = std::u16string::npos;
} // namespace
TEST(UTFOffsetStringConversionsTest, AdjustOffset) {
struct UTF8ToUTF16Case {
const char* utf8;
size_t input_offset;
size_t output_offset;
} utf8_to_utf16_cases[] = {
{"", 0, 0},
{"", kNpos, kNpos},
{"\xe4\xbd\xa0\xe5\xa5\xbd", 1, kNpos},
{"\xe4\xbd\xa0\xe5\xa5\xbd", 3, 1},
{"\xed\xb0\x80z", 3, 3},
{"A\xF0\x90\x8C\x80z", 1, 1},
{"A\xF0\x90\x8C\x80z", 2, kNpos},
{"A\xF0\x90\x8C\x80z", 5, 3},
{"A\xF0\x90\x8C\x80z", 6, 4},
{"A\xF0\x90\x8C\x80z", kNpos, kNpos},
};
for (const auto& i : utf8_to_utf16_cases) {
const size_t offset = i.input_offset;
std::vector<size_t> offsets;
offsets.push_back(offset);
UTF8ToUTF16AndAdjustOffsets(i.utf8, &offsets);
EXPECT_EQ(i.output_offset, offsets[0]);
}
struct UTF16ToUTF8Case {
char16_t utf16[10];
size_t input_offset;
size_t output_offset;
} utf16_to_utf8_cases[] = {
{{}, 0, 0},
// Converted to 3-byte utf-8 sequences
{{0x5909, 0x63DB}, 3, kNpos},
{{0x5909, 0x63DB}, 2, 6},
{{0x5909, 0x63DB}, 1, 3},
{{0x5909, 0x63DB}, 0, 0},
// Converted to 2-byte utf-8 sequences
{{'A', 0x00bc, 0x00be, 'z'}, 1, 1},
{{'A', 0x00bc, 0x00be, 'z'}, 2, 3},
{{'A', 0x00bc, 0x00be, 'z'}, 3, 5},
{{'A', 0x00bc, 0x00be, 'z'}, 4, 6},
// Surrogate pair
{{'A', 0xd800, 0xdf00, 'z'}, 1, 1},
{{'A', 0xd800, 0xdf00, 'z'}, 2, kNpos},
{{'A', 0xd800, 0xdf00, 'z'}, 3, 5},
{{'A', 0xd800, 0xdf00, 'z'}, 4, 6},
};
for (size_t i = 0; i < std::size(utf16_to_utf8_cases); ++i) {
size_t offset = utf16_to_utf8_cases[i].input_offset;
std::vector<size_t> offsets;
offsets.push_back(offset);
UTF16ToUTF8AndAdjustOffsets(utf16_to_utf8_cases[i].utf16, &offsets);
EXPECT_EQ(utf16_to_utf8_cases[i].output_offset, offsets[0]) << i;
}
}
TEST(UTFOffsetStringConversionsTest, LimitOffsets) {
const OffsetAdjuster::Adjustments kNoAdjustments;
const size_t kLimit = 10;
const size_t kItems = 20;
std::vector<size_t> size_ts;
for (size_t t = 0; t < kItems; ++t) {
size_ts.push_back(t);
OffsetAdjuster::AdjustOffset(kNoAdjustments, &size_ts.back(), kLimit);
}
size_t unlimited_count = 0;
for (auto ti : size_ts) {
if (ti != kNpos)
++unlimited_count;
}
EXPECT_EQ(11U, unlimited_count);
// Reverse the values in the vector and try again.
size_ts.clear();
for (size_t t = kItems; t > 0; --t) {
size_ts.push_back(t - 1);
OffsetAdjuster::AdjustOffset(kNoAdjustments, &size_ts.back(), kLimit);
}
unlimited_count = 0;
for (auto ti : size_ts) {
if (ti != kNpos)
++unlimited_count;
}
EXPECT_EQ(11U, unlimited_count);
}
TEST(UTFOffsetStringConversionsTest, AdjustOffsets) {
// Imagine we have strings as shown in the following cases where the
// X's represent encoded characters.
// 1: abcXXXdef ==> abcXdef
{
std::vector<size_t> offsets;
for (size_t t = 0; t <= 9; ++t)
offsets.push_back(t);
OffsetAdjuster::Adjustments adjustments;
adjustments.push_back(OffsetAdjuster::Adjustment(3, 3, 1));
OffsetAdjuster::AdjustOffsets(adjustments, &offsets);
size_t expected_1[] = {0, 1, 2, 3, kNpos, kNpos, 4, 5, 6, 7};
EXPECT_EQ(offsets.size(), std::size(expected_1));
for (size_t i = 0; i < std::size(expected_1); ++i)
EXPECT_EQ(expected_1[i], offsets[i]);
}
// 2: XXXaXXXXbcXXXXXXXdefXXX ==> XaXXbcXXXXdefX
{
std::vector<size_t> offsets;
for (size_t t = 0; t <= 23; ++t)
offsets.push_back(t);
OffsetAdjuster::Adjustments adjustments;
adjustments.push_back(OffsetAdjuster::Adjustment(0, 3, 1));
adjustments.push_back(OffsetAdjuster::Adjustment(4, 4, 2));
adjustments.push_back(OffsetAdjuster::Adjustment(10, 7, 4));
adjustments.push_back(OffsetAdjuster::Adjustment(20, 3, 1));
OffsetAdjuster::AdjustOffsets(adjustments, &offsets);
size_t expected_2[] = {
0, kNpos, kNpos, 1, 2, kNpos, kNpos, kNpos, 4, 5, 6, kNpos, kNpos, kNpos,
kNpos, kNpos, kNpos, 10, 11, 12, 13, kNpos, kNpos, 14
};
EXPECT_EQ(offsets.size(), std::size(expected_2));
for (size_t i = 0; i < std::size(expected_2); ++i)
EXPECT_EQ(expected_2[i], offsets[i]);
}
// 3: XXXaXXXXbcdXXXeXX ==> aXXXXbcdXXXe
{
std::vector<size_t> offsets;
for (size_t t = 0; t <= 17; ++t)
offsets.push_back(t);
OffsetAdjuster::Adjustments adjustments;
adjustments.push_back(OffsetAdjuster::Adjustment(0, 3, 0));
adjustments.push_back(OffsetAdjuster::Adjustment(4, 4, 4));
adjustments.push_back(OffsetAdjuster::Adjustment(11, 3, 3));
adjustments.push_back(OffsetAdjuster::Adjustment(15, 2, 0));
OffsetAdjuster::AdjustOffsets(adjustments, &offsets);
size_t expected_3[] = {
0, kNpos, kNpos, 0, 1, kNpos, kNpos, kNpos, 5, 6, 7, 8, kNpos, kNpos, 11,
12, kNpos, 12
};
EXPECT_EQ(offsets.size(), std::size(expected_3));
for (size_t i = 0; i < std::size(expected_3); ++i)
EXPECT_EQ(expected_3[i], offsets[i]);
}
}
TEST(UTFOffsetStringConversionsTest, UnadjustOffsets) {
// Imagine we have strings as shown in the following cases where the
// X's represent encoded characters.
// 1: abcXXXdef ==> abcXdef
{
std::vector<size_t> offsets;
for (size_t t = 0; t <= 7; ++t)
offsets.push_back(t);
OffsetAdjuster::Adjustments adjustments;
adjustments.push_back(OffsetAdjuster::Adjustment(3, 3, 1));
OffsetAdjuster::UnadjustOffsets(adjustments, &offsets);
size_t expected_1[] = {0, 1, 2, 3, 6, 7, 8, 9};
EXPECT_EQ(offsets.size(), std::size(expected_1));
for (size_t i = 0; i < std::size(expected_1); ++i)
EXPECT_EQ(expected_1[i], offsets[i]);
}
// 2: XXXaXXXXbcXXXXXXXdefXXX ==> XaXXbcXXXXdefX
{
std::vector<size_t> offsets;
for (size_t t = 0; t <= 14; ++t)
offsets.push_back(t);
OffsetAdjuster::Adjustments adjustments;
adjustments.push_back(OffsetAdjuster::Adjustment(0, 3, 1));
adjustments.push_back(OffsetAdjuster::Adjustment(4, 4, 2));
adjustments.push_back(OffsetAdjuster::Adjustment(10, 7, 4));
adjustments.push_back(OffsetAdjuster::Adjustment(20, 3, 1));
OffsetAdjuster::UnadjustOffsets(adjustments, &offsets);
size_t expected_2[] = {
0, 3, 4, kNpos, 8, 9, 10, kNpos, kNpos, kNpos, 17, 18, 19, 20, 23
};
EXPECT_EQ(offsets.size(), std::size(expected_2));
for (size_t i = 0; i < std::size(expected_2); ++i)
EXPECT_EQ(expected_2[i], offsets[i]);
}
// 3: XXXaXXXXbcdXXXeXX ==> aXXXXbcdXXXe
{
std::vector<size_t> offsets;
for (size_t t = 0; t <= 12; ++t)
offsets.push_back(t);
OffsetAdjuster::Adjustments adjustments;
adjustments.push_back(OffsetAdjuster::Adjustment(0, 3, 0));
adjustments.push_back(OffsetAdjuster::Adjustment(4, 4, 4));
adjustments.push_back(OffsetAdjuster::Adjustment(11, 3, 3));
adjustments.push_back(OffsetAdjuster::Adjustment(15, 2, 0));
OffsetAdjuster::UnadjustOffsets(adjustments, &offsets);
size_t expected_3[] = {
0, // this could just as easily be 3
4, kNpos, kNpos, kNpos, 8, 9, 10, 11, kNpos, kNpos, 14,
15 // this could just as easily be 17
};
EXPECT_EQ(offsets.size(), std::size(expected_3));
for (size_t i = 0; i < std::size(expected_3); ++i)
EXPECT_EQ(expected_3[i], offsets[i]);
}
}
// MergeSequentialAdjustments is used by net/base/escape.{h,cc} and
// net/base/net_util.{h,cc}. The two tests EscapeTest.AdjustOffset and
// NetUtilTest.FormatUrlWithOffsets test its behavior extensively. This
// is simply a short, additional test.
TEST(UTFOffsetStringConversionsTest, MergeSequentialAdjustments) {
// Pretend the input string is "abcdefghijklmnopqrstuvwxyz".
// Set up |first_adjustments| to
// - remove the leading "a"
// - combine the "bc" into one character (call it ".")
// - remove the "f"
// - remove the "tuv"
// The resulting string should be ".deghijklmnopqrswxyz".
OffsetAdjuster::Adjustments first_adjustments;
first_adjustments.push_back(OffsetAdjuster::Adjustment(0, 1, 0));
first_adjustments.push_back(OffsetAdjuster::Adjustment(1, 2, 1));
first_adjustments.push_back(OffsetAdjuster::Adjustment(5, 1, 0));
first_adjustments.push_back(OffsetAdjuster::Adjustment(19, 3, 0));
// Set up |adjustments_on_adjusted_string| to
// - combine the "." character that replaced "bc" with "d" into one character
// (call it "?")
// - remove the "egh"
// - expand the "i" into two characters (call them "12")
// - combine the "jkl" into one character (call it "@")
// - expand the "z" into two characters (call it "34")
// The resulting string should be "?12@mnopqrswxy34".
OffsetAdjuster::Adjustments adjustments_on_adjusted_string;
adjustments_on_adjusted_string.push_back(OffsetAdjuster::Adjustment(
0, 2, 1));
adjustments_on_adjusted_string.push_back(OffsetAdjuster::Adjustment(
2, 3, 0));
adjustments_on_adjusted_string.push_back(OffsetAdjuster::Adjustment(
5, 1, 2));
adjustments_on_adjusted_string.push_back(OffsetAdjuster::Adjustment(
6, 3, 1));
adjustments_on_adjusted_string.push_back(OffsetAdjuster::Adjustment(
19, 1, 2));
// Now merge the adjustments and check the results.
OffsetAdjuster::MergeSequentialAdjustments(first_adjustments,
&adjustments_on_adjusted_string);
// The merged adjustments should look like
// - combine abcd into "?"
// - note: it's also reasonable for the Merge function to instead produce
// two adjustments instead of this, one to remove a and another to
// combine bcd into "?". This test verifies the current behavior.
// - remove efgh
// - expand i into "12"
// - combine jkl into "@"
// - remove tuv
// - expand z into "34"
ASSERT_EQ(6u, adjustments_on_adjusted_string.size());
EXPECT_EQ(0u, adjustments_on_adjusted_string[0].original_offset);
EXPECT_EQ(4u, adjustments_on_adjusted_string[0].original_length);
EXPECT_EQ(1u, adjustments_on_adjusted_string[0].output_length);
EXPECT_EQ(4u, adjustments_on_adjusted_string[1].original_offset);
EXPECT_EQ(4u, adjustments_on_adjusted_string[1].original_length);
EXPECT_EQ(0u, adjustments_on_adjusted_string[1].output_length);
EXPECT_EQ(8u, adjustments_on_adjusted_string[2].original_offset);
EXPECT_EQ(1u, adjustments_on_adjusted_string[2].original_length);
EXPECT_EQ(2u, adjustments_on_adjusted_string[2].output_length);
EXPECT_EQ(9u, adjustments_on_adjusted_string[3].original_offset);
EXPECT_EQ(3u, adjustments_on_adjusted_string[3].original_length);
EXPECT_EQ(1u, adjustments_on_adjusted_string[3].output_length);
EXPECT_EQ(19u, adjustments_on_adjusted_string[4].original_offset);
EXPECT_EQ(3u, adjustments_on_adjusted_string[4].original_length);
EXPECT_EQ(0u, adjustments_on_adjusted_string[4].output_length);
EXPECT_EQ(25u, adjustments_on_adjusted_string[5].original_offset);
EXPECT_EQ(1u, adjustments_on_adjusted_string[5].original_length);
EXPECT_EQ(2u, adjustments_on_adjusted_string[5].output_length);
}
} // namespace base