Add UTF-16 surrogate pair decode support

- Add tests for UTF-16 decoding and failures
- Add getutf8.pl to assist with UTF-16 decode testing

- Re-add test_decode_cycle() which was accidentally removed earlier
- Rename bytestring.dat to octets-escaped.dat

1 files changed, 25 insertions, 0 deletions

diff --git a/tests/genutf8.pl b/tests/genutf8.pl
new file mode 100755
index 0000000..4960663
--- /dev/null
+++ b/tests/genutf8.pl
@@ -0,0 +1,25 @@
+#!/usr/bin/perl -w
+
+# Create test comparison data using a different UTF-8 implementation.
+
+use strict;
+use Text::Iconv;
+use FileHandle;
+
+# 0xD800 - 0xDFFF are used to encode supplementary codepoints
+# 0x10000 - 0x10FFFF are supplementary codepoints
+my (@codepoints) = (0 .. 0xD7FF, 0xE000 .. 0x10FFFF);
+
+my ($utf32be) = pack("N*", @codepoints);
+my $iconv = Text::Iconv->new("UTF-32BE", "UTF-8");
+my ($utf8) = $iconv->convert($utf32be);
+defined($utf8) or die "Unable create UTF-8 string\n";
+
+my $fh = FileHandle->new();
+$fh->open("utf8.dat", ">")
+    or die "Unable to open utf8.dat: $!\n";
+$fh->print($utf8)
+    or die "Unable to write utf.dat\n";
+$fh->close();
+
+# vi:ai et sw=4 ts=4: