using System.Collections;
using System.Collections.Generic;
using System.Threading.Tasks;
using System.Runtime.InteropServices;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Serialization;
public static partial class TextExtensions
public static Encoding PlatformCompatibleUnicode => BitConverter.IsLittleEndian ? Encoding.Unicode : Encoding.BigEndianUnicode;
static bool IsPlatformCompatibleUnicode(this Encoding encoding) => BitConverter.IsLittleEndian ? encoding.CodePage == 1200 : encoding.CodePage == 1201;
public static Stream AsStream(this string @string, Encoding encoding = default) =>
(@string ?? throw new ArgumentNullException(nameof(@string))).AsMemory().AsStream(encoding);
public static Stream AsStream(this ReadOnlyMemory<char> charBuffer, Encoding encoding = default) =>
((encoding ??= Encoding.UTF8).IsPlatformCompatibleUnicode())
? new UnicodeStream(charBuffer)
: Encoding.CreateTranscodingStream(new UnicodeStream(charBuffer), PlatformCompatibleUnicode, encoding, false);
sealed class UnicodeStream : Stream
const int BytesPerChar = 2;
ReadOnlyMemory<char> charMemory;
Task<int> _cachedResultTask;
public UnicodeStream(string @string) : this((@string ?? throw new ArgumentNullException(nameof(@string))).AsMemory()) { }
public UnicodeStream(ReadOnlyMemory<char> charMemory) => this.charMemory = charMemory;
public override int Read(Span<byte> buffer)
var charPosition = position / BytesPerChar;
var byteSlice = MemoryMarshal.AsBytes(charMemory.Slice(charPosition, Math.Min(charMemory.Length - charPosition, 1 + buffer.Length / BytesPerChar)).Span);
var slicePosition = position % BytesPerChar;
var nRead = Math.Min(buffer.Length, byteSlice.Length - slicePosition);
byteSlice.Slice(slicePosition, nRead).CopyTo(buffer);
public override int Read(byte[] buffer, int offset, int count)
ValidateBufferArgs(buffer, offset, count);
return Read(buffer.AsSpan(offset, count));
public override int ReadByte()
Span<byte> span = stackalloc byte[1];
return Read(span) == 0 ? -1 : span[0];
public override ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default)
if (cancellationToken.IsCancellationRequested)
return ValueTask.FromCanceled<int>(cancellationToken);
return new ValueTask<int>(Read(buffer.Span));
catch (Exception exception)
return ValueTask.FromException<int>(exception);
public override Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
ValidateBufferArgs(buffer, offset, count);
var valueTask = ReadAsync(buffer.AsMemory(offset, count));
if (!valueTask.IsCompletedSuccessfully)
return valueTask.AsTask();
var lastResultTask = _cachedResultTask;
return (lastResultTask != null && lastResultTask.Result == valueTask.Result) ? lastResultTask : (_cachedResultTask = Task.FromResult<int>(valueTask.Result));
throw new ObjectDisposedException(GetType().Name);
public override void Flush() { }
public override Task FlushAsync(CancellationToken cancellationToken) => cancellationToken.IsCancellationRequested ? Task.FromCanceled(cancellationToken) : Task.CompletedTask;
public override bool CanRead => true;
public override bool CanSeek => false;
public override bool CanWrite => false;
public override long Length => throw new NotSupportedException();
public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); }
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
protected override void Dispose(bool disposing)
_cachedResultTask = null;
static void ValidateBufferArgs(byte[] buffer, int offset, int count)
throw new ArgumentNullException(nameof(buffer));
if (offset < 0 || count < 0)
throw new ArgumentOutOfRangeException();
if (count > buffer.Length - offset)
throw new ArgumentException();
static string hokke = "\U00029E3D";
public static void Test()
var s = string.Concat(new [] { "x" }.Concat(Enumerable.Repeat(hokke, 2047)));
TestReadSingle(s, Encoding.Unicode, 131);
static void TestReadByte(string s)
var bytes = TextExtensions.PlatformCompatibleUnicode.GetBytes(s);
using var stream = new UnicodeStream(s);
Assert.AreEqual((int)b, stream.ReadByte());
Assert.IsTrue(stream.ReadByte() == -1);
static void TestExceptions()
Assert.Throws<ArgumentNullException>(() => new MemoryStream(null));
Assert.Throws<ArgumentNullException>(() => new UnicodeStream((string)null));
Assert.Throws<ArgumentNullException>(() => new MemoryStream().Read(null, 0, 0));
Assert.Throws<ArgumentNullException>(() => new UnicodeStream("").Read(null, 0, 0));
Assert.Throws<ArgumentOutOfRangeException>(() => new MemoryStream().Read(new byte[1], -1, 2));
Assert.Throws<ArgumentOutOfRangeException>(() => new UnicodeStream("").Read(new byte[1], -1, 2));
Assert.Throws<ArgumentOutOfRangeException>(() => new MemoryStream().Read(new byte[1], 1, -1));
Assert.Throws<ArgumentOutOfRangeException>(() => new UnicodeStream("").Read(new byte[1], 1, -1));
Assert.Throws<ArgumentException>(() => new MemoryStream().Read(new byte[1], 0, 2));
Assert.Throws<ArgumentException>(() => new UnicodeStream("").Read(new byte[1], 0, 2));
Assert.Throws<ObjectDisposedException>(() => { var stream = new MemoryStream(); stream.Dispose(); stream.Read(new byte[1], 0, 1); });
Assert.Throws<ObjectDisposedException>(() => { var stream = new UnicodeStream(""); stream.Dispose(); stream.Read(new byte[1], 0, 1); });
Assert.Throws<ArgumentException>(() => { var stream = new MemoryStream(); stream.Dispose(); stream.Read(new byte[1], 0, 2); });
Assert.Throws<ArgumentException>(() => { var stream = new UnicodeStream(""); stream.Dispose(); stream.Read(new byte[1], 0, 2); });
static void Test(string s, int bufferSize = 1024)
TestReadSingle(s, null, bufferSize);
foreach (var encoding in Encoding.GetEncodings())
TestReadSingle(s, encoding.GetEncoding(), bufferSize);
static void TestReadSingle(string s, Encoding encoding = default, int bufferSize = 1024)
var result = (encoding == null ? s : encoding.GetString(encoding.GetBytes(s)));
using (var stream = s.AsStream(encoding))
using (var reader = new StreamReader(stream, encoding, true, bufferSize))
Assert.AreEqual(result, reader.ReadToEnd());
public static void Main()
Console.WriteLine("Environment version: {0} ({1})", System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription , GetNetCoreVersion());
Console.WriteLine("{0} version: {1}", typeof(JsonSerializer).Assembly.GetName().Name, typeof(JsonSerializer).Assembly.FullName);
Console.WriteLine("Failed with unhandled exception: ");
public static string GetNetCoreVersion()
var assembly = typeof(System.Runtime.GCSettings).GetTypeInfo().Assembly;
var assemblyPath = assembly.Location.Split(new[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries);
int netCoreAppIndex = Array.IndexOf(assemblyPath, "Microsoft.NETCore.App");
if (netCoreAppIndex > 0 && netCoreAppIndex < assemblyPath.Length - 2)
return assemblyPath[netCoreAppIndex + 1];