diff --git a/vlib/veb/tests/handle_split_requests_in_chrome_test.v b/vlib/veb/tests/handle_split_requests_in_chrome_test.v new file mode 100644 index 0000000000..816a35b782 --- /dev/null +++ b/vlib/veb/tests/handle_split_requests_in_chrome_test.v @@ -0,0 +1,95 @@ +import log +import veb +import time +import net +import x.json2 +import net.http + +const port = 31228 + +pub struct Context { + veb.Context +} + +pub struct App { +pub mut: + started chan bool +} + +pub fn (mut app App) before_accept_loop() { + app.started <- true +} + +@['/data/:filename'; post] +fn (mut app App) data(mut ctx Context, filename string) veb.Result { + content_type := ctx.get_header(http.CommonHeader.content_type) or { return ctx.no_content() } + f := http.FileData{ + filename: filename + content_type: content_type + data: ctx.req.data + } + log.info('Received ${filename} with content_type ${content_type} and length ${f.data.len}') + return ctx.json(f) +} + +const svg_image_content = '' + +fn test_veb_app_start() { + log.info('starting watchdog ...') + spawn fn () { + log.info('watchdog running') + time.sleep(10 * time.second) + log.info('exiting...') + exit(0) + }() + mut app := &App{} + spawn veb.run_at[App, Context](mut app, port: port) + _ := <-app.started + log.info('app started') +} + +fn test_make_request() { + mut client := net.dial_tcp('127.0.0.1:${port}')! + defer { client.close() or {} } + client.write_string('POST /data/Seeker.svg HTTP/1.1\r +Host: localhost:8090\r +User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0\r +Accept: */*\r +Accept-Language: en-US,en;q=0.5\r +Accept-Encoding: gzip, deflate, br, zstd\r +Content-Type: image/svg+xml\r +Content-Length: 618\r +Origin: null\r +Connection: close\r +Sec-Fetch-Dest: empty\r +Sec-Fetch-Mode: cors\r +Sec-Fetch-Site: cross-site\r +Priority: u=4\r +\r +')! // " + + time.sleep(25 * time.millisecond) + client.write_string(svg_image_content)! + mut res := []u8{} + mut buf := []u8{len: 512} + for { + read_len := client.read(mut buf) or { break } + if read_len == 0 { + break + } + res << buf[0..read_len] + } + response := res.bytestr() + assert response.starts_with('HTTP/1.1 200 OK') + assert response.contains('Content-Length: 706') + assert response.contains('Content-Type: application/json') + payload := response.all_after('\r\n\r\n') + r := json2.decode[http.FileData](payload)! + dump(r.filename) + dump(r.content_type) + assert r.filename == 'Seeker.svg' + assert r.content_type == 'image/svg+xml' + assert r.data.starts_with('') + assert r.data == svg_image_content +} diff --git a/vlib/veb/veb.v b/vlib/veb/veb.v index 0d3f02aa2f..6f99d9f083 100644 --- a/vlib/veb/veb.v +++ b/vlib/veb/veb.v @@ -134,6 +134,9 @@ mut: pub fn (mut params RequestParams) request_done(fd int) { params.incomplete_requests[fd] = http.Request{} params.idx[fd] = 0 + $if trace_handle_read ? { + eprintln('>>>>> fd: ${fd} | request_done.') + } } interface BeforeAcceptApp { @@ -240,10 +243,8 @@ fn handle_timeout(mut pv picoev.Picoev, mut params RequestParams, fd int) { handle: fd is_blocking: false } - fast_send_resp(mut conn, http_408) or {} pv.close_conn(fd) - params.request_done(fd) } @@ -263,19 +264,16 @@ fn handle_write_file(mut pv picoev.Picoev, mut params RequestParams, fd int) { if bytes_to_write > max_write { bytes_to_write = max_write } - data := unsafe { malloc(bytes_to_write) } defer { unsafe { free(data) } } - mut conn := &net.TcpConn{ sock: net.tcp_socket_from_handle_raw(fd) handle: fd is_blocking: false write_timeout: params.timeout_in_seconds * time.second } - params.file_responses[fd].file.read_into_ptr(data, bytes_to_write) or { params.file_responses[fd].done() pv.close_conn(fd) @@ -301,17 +299,14 @@ fn handle_write_file(mut pv picoev.Picoev, mut params RequestParams, fd int) { @[direct_array_access] fn handle_write_string(mut pv picoev.Picoev, mut params RequestParams, fd int) { mut bytes_to_write := int(params.string_responses[fd].str.len - params.string_responses[fd].pos) - if bytes_to_write > max_write { bytes_to_write = max_write } - mut conn := &net.TcpConn{ sock: net.tcp_socket_from_handle_raw(fd) handle: fd is_blocking: false } - // pointer magic to start at the correct position in the buffer data := unsafe { params.string_responses[fd].str.str + params.string_responses[fd].pos } actual_written := send_string_ptr(mut conn, data, bytes_to_write) or { @@ -342,7 +337,6 @@ fn handle_read[A, X](mut pv picoev.Picoev, mut params RequestParams, fd int) { handle: fd is_blocking: false } - // cap the max_read to 8KB mut reader := io.new_buffered_reader(reader: conn, cap: max_read) defer { @@ -350,12 +344,14 @@ fn handle_read[A, X](mut pv picoev.Picoev, mut params RequestParams, fd int) { reader.free() } } - // take the previous incomplete request mut req := params.incomplete_requests[fd] - // check if there is an incomplete request for this file descriptor if params.idx[fd] == 0 { + $if trace_handle_read ? { + eprintln('>>>>> fd: ${fd} | start of request parsing') + } + // this is the start of a new request, setup the connection, and read the headers: // set the read and write timeout according to picoev settings when the // connection is first encountered conn.set_read_timeout(params.timeout_in_seconds) @@ -370,7 +366,7 @@ fn handle_read[A, X](mut pv picoev.Picoev, mut params RequestParams, fd int) { // the buffered reader was empty meaning that the client probably // closed the connection. pv.close_conn(fd) - params.incomplete_requests[fd] = http.Request{} + params.request_done(fd) return } if reader.total_read >= max_read { @@ -379,37 +375,45 @@ fn handle_read[A, X](mut pv picoev.Picoev, mut params RequestParams, fd int) { eprintln('[veb] error parsing request: too large') fast_send_resp(mut conn, http_413) or {} pv.close_conn(fd) - params.incomplete_requests[fd] = http.Request{} + params.request_done(fd) return } } - + if params.idx[fd] == -1 { + // this is for sure a continuation of a previous request, where the first part contained only headers; + // make sure that we are ready to accept the body and account for every byte in it, by setting the counter to 0: + params.idx[fd] = 0 + $if trace_handle_read ? { + eprintln('>>>>> fd: ${fd} | continuation of request, where the first part contained headers') + } + } // check if the request has a body content_length := req.header.get(.content_length) or { '0' } - if content_length.int() > 0 { + content_length_i := content_length.int() + if content_length_i > 0 { mut max_bytes_to_read := max_read - reader.total_read - mut bytes_to_read := content_length.int() - params.idx[fd] + mut bytes_to_read := content_length_i - params.idx[fd] // cap the bytes to read to 8KB for the body, including the request headers if any if bytes_to_read > max_read - reader.total_read { bytes_to_read = max_read - reader.total_read } - mut buf_ptr := params.buf unsafe { buf_ptr += fd * max_read // pointer magic } // convert to []u8 for BufferedReader mut buf := unsafe { buf_ptr.vbytes(max_bytes_to_read) } - n := reader.read(mut buf) or { if reader.total_read > 0 { - // the headers were parsed in this cycle, but the body has not been - // sent yet. No need to error + // The headers were parsed in this cycle, but the body has not been sent yet. No need to error. + params.idx[fd] = -1 // avoid reparsing the headers on the next call. + params.incomplete_requests[fd] = req + $if trace_handle_read ? { + eprintln('>>>>> fd: ${fd} | request headers were parsed, but the body has not been parsed yet | params.idx[fd]: ${params.idx[fd]} | content_length_i: ${content_length_i}') + } return } - eprintln('[veb] error reading request body: ${err}') - if err is io.Eof { // we expect more data to be send, but an Eof error occurred, meaning // that there is no more data to be read from the socket. @@ -423,17 +427,14 @@ fn handle_read[A, X](mut pv picoev.Picoev, mut params RequestParams, fd int) { ).join(headers_close) )) or {} } - pv.close_conn(fd) - params.incomplete_requests[fd] = http.Request{} - params.idx[fd] = 0 + params.request_done(fd) return } - // there is no more data to be sent, but it is less than the Content-Length header // so it is a mismatch of body length and content length. // Or if there is more data received then the Content-Length header specified - if (n == 0 && params.idx[fd] != 0) || params.idx[fd] + n > content_length.int() { + if (n == 0 && params.idx[fd] != 0) || params.idx[fd] + n > content_length_i { fast_send_resp(mut conn, http.new_response( status: .bad_request body: 'Mismatch of body length and Content-Length header' @@ -442,29 +443,31 @@ fn handle_read[A, X](mut pv picoev.Picoev, mut params RequestParams, fd int) { value: 'text/plain' ).join(headers_close) )) or {} - pv.close_conn(fd) - params.incomplete_requests[fd] = http.Request{} - params.idx[fd] = 0 + params.request_done(fd) return - } else if n < bytes_to_read || params.idx[fd] + n < content_length.int() { + } else if n < bytes_to_read || params.idx[fd] + n < content_length_i { // request is incomplete wait until the socket becomes ready to read again - params.idx[fd] += n // TODO: change this to a memcpy function? req.data += buf[0..n].bytestr() params.incomplete_requests[fd] = req + params.idx[fd] += n + $if trace_handle_read ? { + eprintln('>>>>> request is NOT complete, fd: ${fd} | n: ${n} | req.data.len: ${req.data.len} | params.idx[fd]: ${params.idx[fd]}') + } return } else { // request is complete: n = bytes_to_read - params.idx[fd] += n req.data += buf[0..n].bytestr() + params.idx[fd] += n + $if trace_handle_read ? { + eprintln('>>>>> request is NOW COMPLETE, fd: ${fd} | n: ${n} | req.data.len: ${req.data.len}') + } } } - defer { params.request_done(fd) } - if completed_context := handle_request[A, X](mut conn, req, params) { if completed_context.takeover { // the connection should be kept open, but removed from the picoev loop. @@ -473,13 +476,11 @@ fn handle_read[A, X](mut pv picoev.Picoev, mut params RequestParams, fd int) { pv.delete(fd) return } - // TODO: At this point the Context can safely be freed when this function returns. // The user will have to clone the context if the context object should be kept. // defer { // completed_context.free() // } - match completed_context.return_type { .normal { // small optimization: if the response is small write it immediately diff --git a/vlib/x/json2/decoder2/decode.v b/vlib/x/json2/decoder2/decode.v index b3900dfe51..d0a5d1b752 100644 --- a/vlib/x/json2/decoder2/decode.v +++ b/vlib/x/json2/decoder2/decode.v @@ -412,11 +412,41 @@ fn (mut decoder Decoder) decode_value[T](mut val T) ! { string_buffer << `\t` } `u` { - string_buffer << rune(strconv.parse_uint(decoder.json[ + unicode_point := rune(strconv.parse_uint(decoder.json[ string_info.position + string_index..string_info.position + - string_index + 4], 16, 32)!).bytes() + string_index + 4], 16, 32)!) string_index += 4 + + if unicode_point < 0xD800 { // normal utf-8 + string_buffer << unicode_point.bytes() + } else if unicode_point >= 0xDC00 { // trail surrogate -> invalid + decoder.decode_error('Got trail surrogate: ${u32(unicode_point):04X} before head surrogate.')! + } else { // head surrogate -> treat as utf-16 + if string_index > string_info.length - 6 { + decoder.decode_error('Expected a trail surrogate after a head surrogate, but got no valid escape sequence.')! + } + if decoder.json[string_info.position + string_index.. + string_info.position + string_index + 2] != '\\u' { + decoder.decode_error('Expected a trail surrogate after a head surrogate, but got no valid escape sequence.')! + } + + string_index += 2 + + unicode_point2 := rune(strconv.parse_uint(decoder.json[ + string_info.position + string_index..string_info.position + + string_index + 4], 16, 32)!) + + string_index += 4 + + if unicode_point2 < 0xDC00 { + decoder.decode_error('Expected a trail surrogate after a head surrogate, but got ${u32(unicode_point):04X}.')! + } + + final_unicode_point := (unicode_point2 & 0x3FF) + + ((unicode_point & 0x3FF) << 10) + 0x10000 + string_buffer << final_unicode_point.bytes() + } } else {} // has already been checked } diff --git a/vlib/x/json2/decoder2/tests/decode_escaped_string_test.v b/vlib/x/json2/decoder2/tests/decode_escaped_string_test.v index e79a61c3fb..d3e437895a 100644 --- a/vlib/x/json2/decoder2/tests/decode_escaped_string_test.v +++ b/vlib/x/json2/decoder2/tests/decode_escaped_string_test.v @@ -10,3 +10,50 @@ fn test_decode_escaped_string() { assert escaped_strings == decoded_strings } + +fn test_surrogate() { + assert decoder2.decode[string](r'"\ud83d\ude00"')! == '😀' + assert decoder2.decode[string](r'"\ud83d\ude00 text"')! == '😀 text' +} + +fn test_invalid_surrogate() { + if x := decoder2.decode[string](r'"\ud83d"') { + assert false + } else { + if err is decoder2.JsonDecodeError { + assert err.line == 1 + assert err.character == 1 + assert err.message == 'Data: Expected a trail surrogate after a head surrogate, but got no valid escape sequence.' + } + } + + if x := decoder2.decode[string](r'"\ud83d\n\n\n\n"') { + assert false + } else { + if err is decoder2.JsonDecodeError { + assert err.line == 1 + assert err.character == 1 + assert err.message == 'Data: Expected a trail surrogate after a head surrogate, but got no valid escape sequence.' + } + } + + if x := decoder2.decode[string](r'"\ud83d\ud83d"') { + assert false + } else { + if err is decoder2.JsonDecodeError { + assert err.line == 1 + assert err.character == 1 + assert err.message == 'Data: Expected a trail surrogate after a head surrogate, but got D83D.' + } + } + + if x := decoder2.decode[string](r'"\ude00\ud83d"') { + assert false + } else { + if err is decoder2.JsonDecodeError { + assert err.line == 1 + assert err.character == 1 + assert err.message == 'Data: Got trail surrogate: DE00 before head surrogate.' + } + } +}