char/avril

git clone https://git.t4t.associates/char/avril

Charlotte Somport encoding extension to typescriptf2e5e0e2026-10-19

mod common;

#[test]
fn utf8_encoding_and_encode_into_respect_code_points_and_view_bounds() {
    let out = common::run(
        r#"
        const encoder=new TextEncoder();if(encoder.encoding!=='utf-8')throw Error('Encoding');
        console.log([...encoder.encode('Aé😀\ud800')].join(','));
        if(encoder.encode().length!==0||new TextDecoder().decode(encoder.encode(null))!=='null')throw Error('Defaults');
        const source='Aé😀\ud800Z';
        for(let capacity=0;capacity<=12;capacity++) {
            const storage=new Uint8Array(capacity+2).fill(99), dest=storage.subarray(1,capacity+1);
            const result=encoder.encodeInto(source,dest);
            if(storage[0]!==99||storage[capacity+1]!==99||result.written>capacity)throw Error('Bounds');
            const expected=encoder.encode(source.slice(0,result.read));
            if(expected.length!==result.written||expected.some((b,i)=>b!==dest[i]))throw Error('Counts');
            if(result.read===3)throw Error('Split scalar');
        }
        for (const source of ['\0', '\ud800', '\udc00', '\ud800\ud800', '\udc00\ud800', 'é\ud800😀\0z']) {
            for (let capacity = 0; capacity < 20; capacity++) {
                const expected = encoder.encode(source);
                const storage = new Uint8Array(capacity + 4).fill(99);
                const destination = storage.subarray(2, capacity + 2);
                const {read, written} = encoder.encodeInto(source, destination);
                if (written !== encoder.encode(source.slice(0, read)).length ||
                    expected.slice(0, written).some((byte, i) => byte !== destination[i]) ||
                    storage.slice(0, 2).some(byte => byte !== 99) ||
                    storage.slice(written + 2).some(byte => byte !== 99)) throw Error('UTF8 prefix bounds');
                if (read < source.length && encoder.encode(String.fromCodePoint(source.codePointAt(read))).length <= capacity - written) {
                    throw Error('Stopped before a complete scalar');
                }
            }
        }
        const view=new Uint8Array(new SharedArrayBuffer(4));const result=encoder.encodeInto('😀x',view);
        if(result.read!==2||result.written!==4||[...view].join(',')!=='240,159,152,128')throw Error('Shared destination');
        for(const bad of [new Uint16Array(4),new DataView(new ArrayBuffer(4)),[],null]) {try{encoder.encodeInto('x',bad);throw Error('Accepted destination');}catch(e){if(!(e instanceof TypeError))throw e;}}
        try{encoder.encode(Symbol());throw Error('Symbol source');}catch(e){if(!(e instanceof TypeError))throw e;}
    "#,
        &[],
    );
    assert_eq!(out.stdout, b"65,195,169,240,159,152,128,239,191,189\n");
}

#[test]
fn decoding_labels_legacy_encodings_and_fatal_errors() {
    let out = common::run(
        r#"
        const cases=[
            [' UTF-8\t','utf-8',[195,169],'é'],
            ['unicode-1-1-utf-8','utf-8',[240,159,152,128],'😀'],
            ['ASCII','windows-1252',[128,233],'ێ'],
            ['latin1','windows-1252',[128],'€'],
            ['shift_jis','shift_jis',[130,160],'あ'],
            ['euc-jp','euc-jp',[164,162],'あ'],
            ['gb18030','gb18030',[196,227],'你'],
            ['big5','big5',[167,65],'你'],
            ['koi8-r','koi8-r',[225],'А'],
            ['utf-16','utf-16le',[65,0,61,216,0,222],'A😀'],
            ['utf-16be','utf-16be',[0,65,216,61,222,0],'A😀'],
            ['x-user-defined','x-user-defined',[128],'\uf780'],
        ];
        for(const [label,encoding,input,text]of cases) {const d=new TextDecoder(label);if(d.encoding!==encoding||d.decode(new Uint8Array(input))!==text)throw Error(label);}
        for(const label of ['bogus','replacement','iso-2022-cn','iso-2022-kr','utf-7','utf\u00a0-8']) {try{new TextDecoder(label);throw Error('Accepted label');}catch(e){if(!(e instanceof RangeError))throw e;}}
        const malformed=new Uint8Array([240,128,128,65,255]);if(new TextDecoder().decode(malformed)!=='���A�')throw Error('Replacement');
        const fatal=new TextDecoder('utf-8',{fatal:true});try{fatal.decode(malformed);throw Error('Fatal accepted');}catch(e){if(!(e instanceof TypeError))throw e;}
        if(fatal.decode(new Uint8Array([65]))!=='A'||!fatal.fatal||fatal.ignoreBOM)throw Error('Reset');
        console.log('labels');
    "#,
        &[],
    );
    assert_eq!(out.stdout, b"labels\n");
}

#[test]
fn streaming_decode_preserves_partial_sequences_and_bom_state() {
    let out = common::run(
        r#"
        const d=new TextDecoder();const pieces=[[239],[187],[191,240],[159],[152,128,239],[187,191]];
        let text='';for(const piece of pieces)text+=d.decode(new Uint8Array(piece),{stream:true});text+=d.decode();
        if(text!=='😀\ufeff')throw Error('UTF8 streaming/BOM');
        if(d.decode(new Uint8Array([239,187,191,65]))!=='A')throw Error('BOM reset');
        if(new TextDecoder('utf-8',{ignoreBOM:true}).decode(new Uint8Array([239,187,191,65]))!=='\ufeffA')throw Error('ignoreBOM');
        if(new TextDecoder('windows-1252').decode(new Uint8Array([239,187,191]))!=='')throw Error('Legacy BOM');
        const utf16=new TextDecoder('utf-16be');let result='';for(const byte of [254,255,216,61,222,0])result+=utf16.decode(new Uint8Array([byte]),{stream:true});result+=utf16.decode();if(result!=='😀')throw Error('UTF16 streaming');
        const incomplete=new TextDecoder();if(incomplete.decode(new Uint8Array([226]),{stream:true})!==''||incomplete.decode()!=='�'||incomplete.decode()!=='')throw Error('Flush/reset');
        const fatal=new TextDecoder('utf-8',{fatal:true});fatal.decode(new Uint8Array([226]),{stream:true});try{fatal.decode();throw Error('Incomplete fatal');}catch(e){if(!(e instanceof TypeError))throw e;}if(fatal.decode(new Uint8Array([65]))!=='A')throw Error('Fatal reset');
        const recovering=new TextDecoder('utf-8',{fatal:true});try{recovering.decode(new Uint8Array([255,65]),{stream:true});throw Error('Malformed stream');}catch(e){if(!(e instanceof TypeError))throw e;}if(recovering.decode()!=='')throw Error('Fatal chunk recovery');
        const shift=new TextDecoder('iso-2022-jp');let shifted='';for(const byte of [27,36,66,36,34,27,40,66,65])shifted+=shift.decode(new Uint8Array([byte]),{stream:true});shifted+=shift.decode();if(shifted!=='あA')throw Error('Stateful encoding');
        console.log('streaming');
    "#,
        &[],
    );
    assert_eq!(out.stdout, b"streaming\n");
}

#[test]
fn buffer_sources_include_shared_resizable_detached_and_offset_views() {
    let out = common::run(
        r#"
        const d=new TextDecoder(), encoder=new TextEncoder();
        const ab=new Uint8Array([0,65,66,0]).buffer;if(d.decode(new DataView(ab,1,2))!=='AB'||d.decode(new Uint8Array(ab,1,2))!=='AB')throw Error('View range');
        const shared=new SharedArrayBuffer(2);new Uint8Array(shared).set([65,66]);if(d.decode(shared)!=='AB'||d.decode(new DataView(shared))!=='AB')throw Error('Shared');
        const rab=new ArrayBuffer(4,{maxByteLength:8});new Uint8Array(rab).set([65,66,67,68]);const fixed=new Uint8Array(rab,0,4);rab.resize(2);
        for(const operation of [()=>d.decode(fixed),()=>encoder.encodeInto('x',fixed)]){try{operation();throw Error('Resizable view');}catch(e){if(!(e instanceof TypeError))throw e;}}
        for(const value of [rab,new DataView(rab),new Uint8Array(rab),new SharedArrayBuffer(2,{maxByteLength:8})]) {try{d.decode(value);throw Error('Resizable accepted');}catch(e){if(!(e instanceof TypeError))throw e;}}
        const detached=new ArrayBuffer(4), typed=new Uint8Array(detached), dataView=new DataView(detached);detached.transfer();
        for(const value of [detached,typed,dataView])if(d.decode(value)!=='')throw Error('Detached source');
        const zero=encoder.encodeInto('x',typed);if(zero.read!==0||zero.written!==0)throw Error('Detached destination');
        const transferred=new Uint8Array([65]);if(d.decode(transferred,{get stream(){transferred.buffer.transfer();return false;}})!=='')throw Error('Detached in conversion');
        for(const bad of [null,{},[65],'A',1]) {try{d.decode(bad);throw Error('Invalid BufferSource');}catch(e){if(!(e instanceof TypeError))throw e;}}
        console.log('buffers');
    "#,
        &[],
    );
    assert_eq!(out.stdout, b"buffers\n");
}

#[test]
fn conversions_branding_and_captured_methods_follow_web_interfaces() {
    let out = common::run(
        r#"
        const decoder=new TextDecoder(), bytes=new Uint8Array([65]);
        if(decoder.decode(bytes,{get stream(){bytes[0]=66;return false;}})!=='B')throw Error('Option conversion/copy order');
        const log=[];new TextDecoder({toString(){log.push('label');return 'utf-8';}},{get fatal(){log.push('fatal');return false;},get ignoreBOM(){log.push('bom');return false;}});if(log.join(',')!=='label,fatal,bom')throw Error('Conversion order');
        for(const call of [()=>TextEncoder(),()=>TextDecoder(),()=>TextEncoder.prototype.encode.call({}),()=>TextDecoder.prototype.decode.call({}),()=>new TextDecoder(undefined,1),()=>decoder.decode(undefined,true)]) {try{call();throw Error('Invalid invocation');}catch(e){if(!(e instanceof TypeError))throw e;}}
        if(Object.prototype.toString.call(decoder)!=='[object TextDecoder]'||!Object.getOwnPropertyDescriptor(TextDecoder.prototype,'decode').enumerable)throw Error('Interface');
        const encode=new TextEncoder(), decode=new TextDecoder();const poison=()=>{throw Error('Poisoned');};
        Uint8Array.prototype.set=poison;ArrayBuffer.isView=poison;String.prototype.toWellFormed=poison;Object.getOwnPropertyDescriptor=poison;
        if(decode.decode(encode.encode('hello'))!=='hello')throw Error('Captured functions');const dest=new Uint8Array(3);if(encode.encodeInto('éx',dest).written!==3)throw Error('Captured set');
        console.log('interfaces');
    "#,
        &[],
    );
    assert_eq!(out.stdout, b"interfaces\n");
}

#[test]
fn encoding_transform_streams_handle_split_scalars_and_backpressure() {
    let out = common::run(
        r#"
        const encoder=new TextEncoderStream(), decoder=new TextDecoderStream();
        if(encoder.encoding!=='utf-8'||decoder.encoding!=='utf-8'||decoder.fatal||decoder.ignoreBOM)throw Error('Stream properties');
        const received=[];const consumer=(async()=>{for await(const text of encoder.readable.pipeThrough(decoder))received.push(text);})();
        const writer=encoder.writable.getWriter();await writer.write('A\ud83d');await writer.write('');await writer.write('\ude00B\ud800');await writer.close();await consumer;
        if(received.join('')!=='A😀B�')throw Error('Surrogate carry');
        const legacy=new TextDecoderStream('shift_jis');let text='';const collecting=(async()=>{for await(const chunk of legacy.readable)text+=chunk;})();const output=legacy.writable.getWriter();await output.write(new Uint8Array([130]));await output.write(new Uint8Array([160]));await output.close();await collecting;if(text!=='あ')throw Error('Legacy transform');
        console.log('transforms');
    "#,
        &[],
    );
    assert_eq!(out.stdout, b"transforms\n");
}

#[test]
fn stream_wrappers_keep_captured_accessors_and_controller_methods() {
    let out = common::run(
        r#"
        const prototype = TransformStream.prototype;
        const readable = Object.getOwnPropertyDescriptor(prototype, 'readable');
        const writable = Object.getOwnPropertyDescriptor(prototype, 'writable');
        const controller = TransformStreamDefaultController.prototype;
        const enqueue = Object.getOwnPropertyDescriptor(controller, 'enqueue');
        const poison = () => { throw Error('Poisoned stream hook'); };
        try {
            Object.defineProperty(prototype, 'readable', {get: poison, configurable: true});
            Object.defineProperty(prototype, 'writable', {get: poison, configurable: true});
            Object.defineProperty(controller, 'enqueue', {value: poison, configurable: true});
            const encoder = new TextEncoderStream(), decoder = new TextDecoderStream();
            const chunks = [];
            const consumed = (async () => {
                for await (const chunk of encoder.readable.pipeThrough(decoder)) chunks.push(chunk);
            })();
            const writer = encoder.writable.getWriter();
            await writer.write('A\ud83d');
            await writer.write('\ude00B');
            await writer.close();
            await consumed;
            if (chunks.join('') !== 'A😀B') throw Error('Captured stream pipeline');
            if (TextEncoder.length !== 0 || TextDecoder.length !== 0 ||
                TextEncoderStream.length !== 0 || TextDecoderStream.length !== 0 ||
                TextEncoder.prototype.encode.length !== 0 ||
                TextEncoder.prototype.encodeInto.length !== 2 ||
                TextDecoder.prototype.decode.length !== 0) throw Error('Callable arity');
        } finally {
            Object.defineProperty(prototype, 'readable', readable);
            Object.defineProperty(prototype, 'writable', writable);
            Object.defineProperty(controller, 'enqueue', enqueue);
        }
        console.log('captured streams');
    "#,
        &[],
    );
    assert_eq!(out.stdout, b"captured streams\n");
}

#[test]
fn fatal_decoder_stream_errors_both_sides_without_unhandled_rejections() {
    let out = common::run(
        r#"
        const stream=new TextDecoderStream('utf-8',{fatal:true});const reader=stream.readable.getReader(),writer=stream.writable.getWriter();
        const reading=reader.read().then(()=>{throw Error('Expected read error');},e=>{if(!(e instanceof TypeError))throw e;});
        try{await writer.write(new Uint8Array([255]));throw Error('Expected write error');}catch(e){if(!(e instanceof TypeError))throw e;}
        await reading;
        const bad=new TextDecoderStream(),r=bad.readable.getReader(),w=bad.writable.getWriter();const rejected=r.read().catch(e=>{if(!(e instanceof TypeError))throw e;});try{await w.write(undefined);throw Error('Invalid stream chunk');}catch(e){if(!(e instanceof TypeError))throw e;}await rejected;
        console.log('errors');
    "#,
        &[],
    );
    assert_eq!(out.stdout, b"errors\n");
}

#[test]
fn default_options_and_internal_transformers_ignore_prototype_pollution() {
    let out = common::run(
        r#"
        Object.prototype.fatal=true;Object.prototype.ignoreBOM=true;Object.prototype.stream=true;
        Object.prototype.start=c=>c.enqueue('injected');Object.prototype.readableType='bytes';Object.prototype.highWaterMark=-1;
        const decoder=new TextDecoder();if(decoder.fatal||decoder.ignoreBOM||decoder.decode(new Uint8Array([226,130]))!=='�')throw Error('Default dictionary');
        const pipeline=new TextDecoderStream(),encoder=new TextEncoderStream();
        if(pipeline.fatal||pipeline.ignoreBOM)throw Error('Stream defaults');
        delete Object.prototype.highWaterMark;
        const result=[];const reading=(async()=>{for await(const chunk of pipeline.readable)result.push(chunk);})();
        const writer=pipeline.writable.getWriter();await writer.write(new Uint8Array([65]));await writer.close();await reading;if(result.join('')!=='A')throw Error('Inherited transformer hook');
        console.log('defaults');
    "#,
        &[],
    );
    assert_eq!(out.stdout, b"defaults\n");
}