From 819836e61f3e55ab1b153decf3dc6eaa0ac4f940 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Mon, 25 Mar 2024 19:21:54 +0000 Subject: [PATCH 1/5] Add simple http_client for use in black box tests (in following commits) BUG: https://bugzilla.samba.org/show_bug.cgi?id=15611 Signed-off-by: Noel Power Reviewed-by: Andrew Bartlett (cherry picked from commit cd6c075476c820b4fe8bdc10a24d8fc8ac74e9c9) --- source4/client/http_test.c | 401 +++++++++++++++++++++++++++++++++++++ source4/wscript_build | 5 + 2 files changed, 406 insertions(+) create mode 100644 source4/client/http_test.c diff --git a/source4/client/http_test.c b/source4/client/http_test.c new file mode 100644 index 00000000000..ca542ba1c6b --- /dev/null +++ b/source4/client/http_test.c @@ -0,0 +1,401 @@ +#include "includes.h" +#include "version.h" +#include "libcli/libcli.h" +#include "lib/events/events.h" +#include "libcli/resolve/resolve.h" +#include "param/param.h" +#include "libcli/raw/raw_proto.h" +#include "libcli/http/http.h" +#include "credentials.h" +#include "util/tevent_ntstatus.h" +#include "lib/tls/tls.h" +#include "lib/cmdline/cmdline.h" + + +struct http_client_info { + struct http_conn *http_conn; + uint16_t server_port; + const char *server_addr; + struct tstream_tls_params *tls_params; + struct cli_credentials *creds; + struct loadparm_context *lp_ctx; + const char *uri; +}; + +static bool send_http_request(TALLOC_CTX *mem_ctx, + struct tevent_context *ev_ctx, + struct http_client_info* es, + size_t response_size, + NTSTATUS *pstatus) +{ + struct http_request *http_req = NULL; + struct tevent_req *req = NULL; + char *uri = NULL; + struct http_request *http_response = NULL; + NTSTATUS status; + + http_req = talloc_zero(mem_ctx, struct http_request); + if (!http_req) { + DBG_ERR("no memory\n"); + return false; + } + + uri = talloc_strdup(mem_ctx, es->uri); + + http_req->type = HTTP_REQ_POST; + http_req->uri = uri; + http_req->body = data_blob_null; + http_req->major = '1'; + http_req->minor = '1'; + + http_add_header(mem_ctx, &http_req->headers, + "User-Agent", "Samba/http_test"); + http_add_header(mem_ctx, &http_req->headers, + "Accept", "*/*"); + + req = http_send_auth_request_send(mem_ctx, + ev_ctx, + es->http_conn, + http_req, + es->creds, + es->lp_ctx, + HTTP_AUTH_BASIC); + if (!tevent_req_set_endtime(req, ev_ctx, timeval_current_ofs(10, 0))) { + DBG_ERR("Failed to set timeout\n"); + return false; + } + + if (!tevent_req_poll_ntstatus(req, ev_ctx, pstatus)) { + DBG_ERR("Failed to connect: %s\n", nt_errstr(*pstatus)); + return false; + } + + status = http_send_auth_request_recv(req); + if (!NT_STATUS_IS_OK(status)) { + DBG_ERR("Auth request failed: %s\n", nt_errstr(status)); + return false; + } + + req = http_read_response_send(mem_ctx, + ev_ctx, + es->http_conn, + response_size); + if (!req) { + DBG_ERR("no memory\n"); + return -1; + } + + if (!tevent_req_set_endtime(req, ev_ctx, timeval_current_ofs(10, 0))) { + DBG_ERR("Failed to set timeout\n"); + return false; + } + + if (!tevent_req_poll_ntstatus(req, ev_ctx, pstatus)) { + DBG_ERR("Failed to read_resonse: %s\n", nt_errstr(*pstatus)); + return false; + } + + *pstatus = http_read_response_recv(req, mem_ctx, &http_response); + + if (!NT_STATUS_IS_OK(*pstatus)) { + DBG_ERR("Failed to receive response: %s\n", nt_errstr(*pstatus)); + return false; + } + /* following are not 'hard' errors */ + if (http_response->response_code != 200) { + fprintf(stdout, "HTTP server response: %u\n", + http_response->response_code); + fflush(stdout); + return false; + + } + if (http_response->body.length == 0) { + fprintf(stdout, "unexpected 0 len response\n"); + fflush(stdout); + return false; + } + DBG_ERR("response: len (%d)\n%s\n", + (int)http_response->body.length, + talloc_strndup(mem_ctx, + (char *)http_response->body.data, + http_response->body.length)); + fprintf(stdout,"%s", talloc_strndup(mem_ctx, + (char *)http_response->body.data, + http_response->body.length)); + fflush(stdout); + return true; +} + +int main(int argc, const char *argv[]) + +{ + TALLOC_CTX *mem_ctx; + struct tevent_context *ev_ctx; + int retries = 4; + int count = 0; + struct http_client_info *http_info = NULL; + bool use_tls = false; + int res; + NTSTATUS status; + struct tevent_req *req = NULL; + bool connected = false; + poptContext pc; + const char **const_argv = discard_const_p(const char *, argv); + int opt; + bool ok; + const char *ca_file = NULL; + int port = 0; + size_t response_size = 8192000; + struct cli_credentials *cli_creds; + + struct poptOption long_options[] = { + POPT_AUTOHELP + + { + .longName = "usetls", + .shortName = 't', + .argInfo = POPT_ARG_NONE, + .arg = NULL, + .val = 't', + .descrip = "Use tls", + .argDescrip = "enable tls", + }, + { + .longName = "ip-address", + .shortName = 'I', + .argInfo = POPT_ARG_STRING, + .arg = NULL, + .val = 'I', + .descrip = "Use this IP to connect to", + .argDescrip = "IP", + }, + { + .longName = "port", + .shortName = 'p', + .argInfo = POPT_ARG_INT, + .arg = &port, + .val = 'p', + .descrip = "port to connect to", + .argDescrip = "port", + }, + { + .longName = "cacart", + .shortName = 'c', + .argInfo = POPT_ARG_STRING, + .arg = NULL, + .val = 'c', + .descrip = "CA certificate to verify peer against", + .argDescrip = "ca cert", + }, + { + .longName = "uri", + .shortName = 'u', + .argInfo = POPT_ARG_STRING, + .arg = NULL, + .val = 'u', + .descrip = "uri to send as part of http request", + .argDescrip = "uri", + }, + { + .longName = "rsize", + .argInfo = POPT_ARG_LONG, + .arg = &response_size, + .descrip = "response size", + }, + POPT_COMMON_SAMBA + POPT_COMMON_CREDENTIALS + POPT_TABLEEND + }; + + mem_ctx = talloc_init("http_test"); + + if (!mem_ctx) { + DBG_ERR("Not enough memory\n"); + res = -1; + goto done; + } + + http_info = talloc_zero(mem_ctx, struct http_client_info); + + if (http_info == NULL) { + DBG_ERR("Not enough memory\n"); + res = -1; + goto done; + } + + ok = samba_cmdline_init(mem_ctx, + SAMBA_CMDLINE_CONFIG_CLIENT, + false /* require_smbconf */); + if (!ok) { + DBG_ERR("Failed to init cmdline parser!\n"); + res = -1; + goto done; + } + + pc = samba_popt_get_context(getprogname(), + argc, + const_argv, + long_options, + 0); + if (pc == NULL) { + DBG_ERR("Failed to setup popt context!\n"); + res = -1; + goto done; + } + + /* some defaults */ + + http_info->server_addr = "localhost"; + http_info->uri = "/_search?pretty"; + + while ((opt = poptGetNextOpt(pc)) != -1) { + switch (opt) { + case 't': + use_tls = true; + break; + case 'c': { + ca_file = talloc_strdup(mem_ctx, + poptGetOptArg(pc)); + if (ca_file == NULL) { + DBG_ERR("Not enough memory\n"); + res = -1; + goto done; + } + break; + } + case 'I': { + http_info->server_addr = talloc_strdup(mem_ctx, + poptGetOptArg(pc)); + if (http_info->server_addr == NULL) { + DBG_ERR("Not enough memory\n"); + res = -1; + goto done; + } + break; + } + case 'u': { + http_info->uri = talloc_strdup(mem_ctx, + poptGetOptArg(pc)); + if (http_info->uri == NULL) { + DBG_ERR("Not enough memory\n"); + res = -1; + goto done; + } + break; + } + } + } + + if (use_tls && ca_file == NULL) { + DBG_ERR("No cacert\n"); + res = -1; + poptPrintUsage(pc, stderr, 0); + goto done; + } + + if (!port) { + port = 8080; + } + http_info->server_port = port; + + ev_ctx = s4_event_context_init(mem_ctx); + if (!ev_ctx) { + DBG_ERR("Not enough memory\n"); + res = -1; + goto done; + } + + + cli_creds = samba_cmdline_get_creds(); + if (!cli_credentials_is_anonymous(cli_creds)) { + http_info->creds = cli_credentials_init(mem_ctx); + cli_credentials_set_username( + http_info->creds, + cli_credentials_get_username(cli_creds), + CRED_SPECIFIED); + cli_credentials_set_password(http_info->creds, + cli_credentials_get_password(cli_creds), + CRED_SPECIFIED); + } else { + DBG_DEBUG("Anonymous creds!!!\n"); + http_info->creds = cli_creds; + } + if (http_info->creds == NULL) { + DBG_ERR("Failed to create creds\n"); + res = -1; + goto done; + } + http_info->lp_ctx = samba_cmdline_get_lp_ctx(); + + DBG_ERR("retries = %d/%d, Using server %s, port %d, using tls %s\n", + count, retries, + http_info->server_addr, + http_info->server_port, + use_tls ? "true" : "false"); + + while (count < retries) { + int error; + DBG_ERR("Connecting to HTTP [%s] port [%"PRIu16"]%s\n", + http_info->server_addr, http_info->server_port, + use_tls ? " with tls" : " without tls"); + if (use_tls) { + const char *crl_file = NULL; + const char *tls_priority = "NORMAL:-VERS-SSL3.0"; + enum tls_verify_peer_state verify_peer = + TLS_VERIFY_PEER_CA_ONLY; + + status = tstream_tls_params_client(mem_ctx, + ca_file, + crl_file, + tls_priority, + verify_peer, + http_info->server_addr, + &http_info->tls_params); + if (!NT_STATUS_IS_OK(status)) { + DBG_ERR("Failed tstream_tls_params_client - %s\n", + nt_errstr(status)); + res = -1; + goto done; + } + } + + req = http_connect_send(mem_ctx, + ev_ctx, + http_info->server_addr, + http_info->server_port, + http_info->creds, + http_info->tls_params); + if (!tevent_req_poll_ntstatus(req, ev_ctx, &status)) { + res = -1; + goto done; + } + + error = http_connect_recv(req, + mem_ctx, + &http_info->http_conn); + if (error != 0) { + count++; + DBG_ERR("HTTP connection failed retry %d/%d: %s\n", count, retries, strerror(error)); + } else { + DBG_ERR("HTTP connection succeeded\n"); + connected = true; + break; + } + } + + if (!connected) { + DBG_ERR("Leaving early\n"); + res = -1; + goto done; + } + + if (!send_http_request(mem_ctx, ev_ctx, http_info, response_size, &status)) { + DBG_ERR("Failure\n"); + res = -1; + goto done; + } + res = 0; +done: + TALLOC_FREE(mem_ctx); + return res; +} diff --git a/source4/wscript_build b/source4/wscript_build index d20444135eb..db2ad3eda5b 100644 --- a/source4/wscript_build +++ b/source4/wscript_build @@ -6,6 +6,11 @@ bld.SAMBA_BINARY('client/smbclient' + bld.env.suffix4, install=False ) +bld.SAMBA_BINARY('client/http_test', + source='client/http_test.c', + deps='samba-hostconfig SMBREADLINE samba-util LIBCLI_SMB RPC_NDR_SRVSVC LIBCLI_LSA popt CMDLINE_S4', + for_selftest=True, + ) bld.SAMBA_BINARY('client/cifsdd', source='client/cifsdd.c client/cifsddio.c', -- 2.35.3 From c9c0890c8b263f30a5a7243a3e952d5d0b2828f4 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Mon, 25 Mar 2024 16:25:55 +0000 Subject: [PATCH 2/5] selftest: Add basic content-lenght http tests very simple test of basic http request/response plus some checks to ensure http response doesn't exceed the response max length set by the client call. BUG: https://bugzilla.samba.org/show_bug.cgi?id=15611 Signed-off-by: Noel Power Reviewed-by: Andrew Bartlett (cherry picked from commit 74cdebeae3d1bc35eea96b51b9491f6c52844b10) --- python/samba/tests/blackbox/http_content.py | 95 +++++++++++++++++++++ selftest/tests.py | 1 + 2 files changed, 96 insertions(+) create mode 100644 python/samba/tests/blackbox/http_content.py diff --git a/python/samba/tests/blackbox/http_content.py b/python/samba/tests/blackbox/http_content.py new file mode 100644 index 00000000000..9ecb6ffe279 --- /dev/null +++ b/python/samba/tests/blackbox/http_content.py @@ -0,0 +1,95 @@ +# Blackbox tests for http_test +# +# Copyright (C) Noel Power noel.power@suse.com +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import time +import threading +import logging +import json +from http.server import HTTPServer, BaseHTTPRequestHandler +from samba.logger import get_samba_logger +from samba.tests import BlackboxTestCase, BlackboxProcessError + +logger = get_samba_logger(name=__name__) +COMMAND = "bin/http_test" + +# simple handler, spits back the 'path' passed in +# GET or POST and a chunked encoded http response +# where the chunk size is 10 octets +class ContentHTTPRequestHandler(BaseHTTPRequestHandler): + def handle_req(self): + msg = bytes(self.path, encoding="utf-8") + + self.send_response(200) + self.send_header('content-type', 'application/json; charset=UTF-8') + self.send_header('content-length', len(msg)) + self.end_headers() + self.wfile.write(msg) + + def do_POST(self): + self.handle_req() + def do_GET(self): + self.handle_req() + +class HttpContentBlackboxTests(BlackboxTestCase): + def setUp(self): + self.server = HTTPServer((os.getenv("SERVER_IP", "localhost"), 8080), + ContentHTTPRequestHandler, + bind_and_activate=False) + self.t = threading.Thread(target=HttpContentBlackboxTests.http_server, args=(self,)) + self.t.setDaemon(True) + self.t.start() + time.sleep(1) + + def tearDown(self): + super().tearDown() + + def http_server(self): + self.server.server_bind() + self.server.server_activate() + self.server.serve_forever() + + def notest_simple_msg(self): + try: + msg = "simplemessage" + resp = self.check_output("%s -U%% -I%s --uri %s" % (COMMAND, os.getenv("SERVER_IP", "localhost"), msg)) + self.assertEqual(msg, resp.decode('utf-8')) + except BlackboxProcessError as e: + print("Failed with: %s" % e) + self.fail(str(e)) + + def test_exceed_request_size(self): + try: + msg = "012345678" # 9 bytes + # limit response to 8 bytes + resp = self.check_output("%s -d11 -U%% -I%s --rsize 8 --uri %s" % (COMMAND, os.getenv("SERVER_IP", "localhost"), msg)) + self.fail(str(e)) + except BlackboxProcessError as e: + if "unexpected 0 len response" not in e.stdout.decode('utf-8'): + self.fail(str(e)) + if "http_parse_headers: Skipping body for code 200" not in e.stderr.decode('utf-8'): + self.fail(str(e)) + + def test_exact_request_size(self): + try: + msg = "012345678" # 9 bytes + # limit response to 9 bytes + resp = self.check_output("%s -U%% -I%s --rsize 9 --uri %s" % (COMMAND, os.getenv("SERVER_IP", "localhost"), msg)) + self.assertEqual(msg, resp.decode('utf-8')) + except BlackboxProcessError as e: + print("Failed with: %s" % e) + self.fail(str(e)) diff --git a/selftest/tests.py b/selftest/tests.py index 2cafe2faa4e..1efa8e79b09 100644 --- a/selftest/tests.py +++ b/selftest/tests.py @@ -220,6 +220,7 @@ plantestsuite( "samba4.blackbox.test_special_group", "none", cmdline('test_special_group.sh', '$PREFIX_ABS/provision')) +planpythontestsuite("fileserver", "samba.tests.blackbox.http_content") planpythontestsuite("none", "samba.tests.upgradeprovision") planpythontestsuite("none", "samba.tests.xattr") planpythontestsuite("none", "samba.tests.ntacls") -- 2.35.3 From 60bb165fa119fa427d42e204ab8335606304b06c Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 22 Mar 2024 08:55:49 +0000 Subject: [PATCH 3/5] libcli/http: Optimise reading for content-length Instead of reading byte-by-byte we know the content length we want to read so lets use it. Signed-off-by: Noel Power Reviewed-by: Andrew Bartlett BUG: https://bugzilla.samba.org/show_bug.cgi?id=15611 (cherry picked from commit 5f03d84e3b52bf5a31a0f885cb83bdcb48ec96f7) --- libcli/http/http.c | 53 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/libcli/http/http.c b/libcli/http/http.c index d20fc25f9e2..00bbde534ea 100644 --- a/libcli/http/http.c +++ b/libcli/http/http.c @@ -527,20 +527,59 @@ static int http_read_response_next_vector(struct tstream_context *stream, *_count = 1; } break; - case HTTP_MORE_DATA_EXPECTED: - /* TODO Optimize, allocating byte by byte */ - state->buffer.data = talloc_realloc(state, state->buffer.data, - uint8_t, state->buffer.length + 1); + case HTTP_MORE_DATA_EXPECTED: { + size_t toread = 1; + size_t total; + if (state->parser_state == HTTP_READING_BODY) { + struct http_request *resp = state->response; + toread = resp->remaining_content_length - + state->buffer.length; + } + + total = toread + state->buffer.length; + + if (total < state->buffer.length) { + DBG_ERR("adding %zu to buf len %zu " + "will overflow\n", + toread, + state->buffer.length); + return -1; + } + + /* + * test if content-length message exceeds the + * specified max_content_length + * Note: This check wont be hit at the moment + * due to an existing check in parse_headers + * which will skip the body. Check is here + * for completeness and to cater for future + * code changes. + */ + if (state->parser_state == HTTP_READING_BODY) { + if (total > state->max_content_length) { + DBG_ERR("content size %zu exceeds " + "max content len %zu\n", + total, + state->max_content_length); + return -1; + } + } + + state->buffer.data = + talloc_realloc(state, state->buffer.data, + uint8_t, + state->buffer.length + toread); if (!state->buffer.data) { return -1; } - state->buffer.length++; + state->buffer.length += toread; vector[0].iov_base = (void *)(state->buffer.data + - state->buffer.length - 1); - vector[0].iov_len = 1; + state->buffer.length - toread); + vector[0].iov_len = toread; *_vector = vector; *_count = 1; break; + } case HTTP_DATA_CORRUPTED: case HTTP_REQUEST_CANCELED: case HTTP_DATA_TOO_LONG: -- 2.35.3 From 1c7ecdbfbb77e191f513e6646410d320049b62d2 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Thu, 23 Sep 2021 12:18:22 +0100 Subject: [PATCH 4/5] tests: add test for chunked encoding with http cli library Adds http test client to excercise the http client library and a blackbox test to run the client. This client is built only with selftest also adds a knownfail for the test Signed-off-by: Noel Power Reviewed-by: Andrew Bartlett BUG: https://bugzilla.samba.org/show_bug.cgi?id=15611 (cherry picked from commit 30acd609f560352d3edb0c931b9a864110025b2c) --- python/samba/tests/blackbox/http_chunk.py | 116 ++++++++++++++++++++++ selftest/knownfail.d/http_chunks | 5 + selftest/tests.py | 1 + 3 files changed, 122 insertions(+) create mode 100644 python/samba/tests/blackbox/http_chunk.py create mode 100644 selftest/knownfail.d/http_chunks diff --git a/python/samba/tests/blackbox/http_chunk.py b/python/samba/tests/blackbox/http_chunk.py new file mode 100644 index 00000000000..1b5cc174788 --- /dev/null +++ b/python/samba/tests/blackbox/http_chunk.py @@ -0,0 +1,116 @@ +# Blackbox tests for http_test +# +# Copyright (C) Noel Power noel.power@suse.com +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import time +import threading +import logging +import json +from http.server import HTTPServer, BaseHTTPRequestHandler +from samba.logger import get_samba_logger +from samba.tests import BlackboxTestCase, BlackboxProcessError + +logger = get_samba_logger(name=__name__) +COMMAND = "bin/http_test" +def make_chunks(msg, chunk_size): + chunks = [] + while len(msg) > chunk_size: + chunk = msg[:chunk_size] + chunks.append(chunk) + msg = msg[chunk_size:] + if len(msg): + chunks.append(msg) + return chunks + +# simple handler, spits back the 'path' passed in +# GET or POST and a chunked encoded http response +# where the chunk size is 10 octets +class ChunkHTTPRequestHandler(BaseHTTPRequestHandler): + def handle_req(self): + msg = bytes(self.path, encoding="utf-8") + chunks = make_chunks(msg, 10) + + self.send_response(200) + self.send_header('content-type', 'application/json; charset=UTF-8') + self.send_header('Transfer-Encoding', 'chunked') + self.end_headers() + resp = bytes() + for chunk in chunks: + resp = resp + ("%x" % len(chunk)).encode("utf-8") + b'\r\n' + chunk + b'\r\n' + resp += b'0\r\n\r\n' + self.wfile.write(resp) + + def do_POST(self): + self.handle_req() + def do_GET(self): + self.handle_req() + +class HttpChunkBlackboxTests(BlackboxTestCase): + def setUp(self): + self.server = HTTPServer((os.getenv("SERVER_IP", "localhost"), 8080), + ChunkHTTPRequestHandler, + bind_and_activate=False) + self.t = threading.Thread(target=HttpChunkBlackboxTests.http_server, args=(self,)) + self.t.setDaemon(True) + self.t.start() + time.sleep(1) + + def tearDown(self): + super().tearDown() + + def http_server(self): + self.server.server_bind() + self.server.server_activate() + self.server.serve_forever() + + def test_single_chunk(self): + try: + msg = "one_chunk" + resp = self.check_output("%s -U%% -I%s --uri %s" % (COMMAND, os.getenv("SERVER_IP", "localhost"), msg)) + self.assertEqual(msg, resp.decode('utf-8')) + except BlackboxProcessError as e: + print("Failed with: %s" % e) + self.fail(str(e)) + + def test_multi_chunks(self): + try: + msg = "snglechunksnglechunksnglechunksnglechunksnglechunk" + resp = self.check_output("%s -U%% -I%s --uri %s" % (COMMAND, os.getenv("SERVER_IP", "localhost"), msg)) + self.assertEqual(msg, resp.decode('utf-8')) + except BlackboxProcessError as e: + print("Failed with: %s" % e) + self.fail(str(e)) + + def test_exceed_request_size(self): + try: + msg = "snglechunksnglechunksnglechunksnglechunksnglechunk" + resp = self.check_output("%s -d11 -U%% -I%s --rsize 49 --uri %s" % (COMMAND, os.getenv("SERVER_IP", "localhost"), msg)) + self.fail(str(e)) + except BlackboxProcessError as e: + if "http_read_chunk: size 50 exceeds max content len 49 skipping body" not in e.stderr.decode('utf-8'): + self.fail(str(e)) + if "unexpected 0 len response" not in e.stdout.decode('utf-8'): + self.fail(str(e)) + + def test_exact_request_size(self): + try: + msg = "snglechunksnglechunksnglechunksnglechunksnglechunk" + resp = self.check_output("%s -U%% -I%s --rsize 50 --uri %s" % (COMMAND, os.getenv("SERVER_IP", "localhost"), msg)) + self.assertEqual(msg, resp.decode('utf-8')) + except BlackboxProcessError as e: + print("Failed with: %s" % e) + self.fail(str(e)) diff --git a/selftest/knownfail.d/http_chunks b/selftest/knownfail.d/http_chunks new file mode 100644 index 00000000000..e58e002abdb --- /dev/null +++ b/selftest/knownfail.d/http_chunks @@ -0,0 +1,5 @@ +^samba.tests.blackbox.http_chunk.samba.tests.blackbox.http_chunk.HttpChunkBlackboxTests.test_multi_chunks +^samba.tests.blackbox.http_chunk.samba.tests.blackbox.http_chunk.HttpChunkBlackboxTests.test_single_chunk +^samba.tests.blackbox.http_chunk.samba.tests.blackbox.http_chunk.HttpChunkBlackboxTests.test_exact_request_size +^samba.tests.blackbox.http_chunk.samba.tests.blackbox.http_chunk.HttpChunkBlackboxTests.test_exceed_request_size + diff --git a/selftest/tests.py b/selftest/tests.py index 1efa8e79b09..76950f447eb 100644 --- a/selftest/tests.py +++ b/selftest/tests.py @@ -221,6 +221,7 @@ plantestsuite( cmdline('test_special_group.sh', '$PREFIX_ABS/provision')) planpythontestsuite("fileserver", "samba.tests.blackbox.http_content") +planpythontestsuite("fileserver", "samba.tests.blackbox.http_chunk") planpythontestsuite("none", "samba.tests.upgradeprovision") planpythontestsuite("none", "samba.tests.xattr") planpythontestsuite("none", "samba.tests.ntacls") -- 2.35.3 From 1720632775afbe719931ea52a03b8271f29c5fb1 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Mon, 25 Mar 2024 19:44:10 +0000 Subject: [PATCH 5/5] libcli/http: Handle http chunked transfer encoding Also removes the knownfail for the chunked transfer test Signed-off-by: Noel Power Reviewed-by: Andrew Bartlett BUG: https://bugzilla.samba.org/show_bug.cgi?id=15611 (cherry picked from commit 03240c91fb6ffcf5afe47c14a1ba7a8bc12f2348) --- libcli/http/http.c | 218 +++++++++++++++++++++- libcli/http/http_internal.h | 4 + python/samba/tests/blackbox/http_chunk.py | 2 +- selftest/knownfail.d/http_chunks | 5 - 4 files changed, 220 insertions(+), 9 deletions(-) delete mode 100644 selftest/knownfail.d/http_chunks diff --git a/libcli/http/http.c b/libcli/http/http.c index 00bbde534ea..95ca3fad9cb 100644 --- a/libcli/http/http.c +++ b/libcli/http/http.c @@ -45,6 +45,12 @@ static int http_response_needs_body(struct http_request *req) char c; unsigned long long v; + cmp = strcasecmp(h->key, "Transfer-Encoding"); + if (cmp == 0) { + cmp = strcasecmp(h->value, "chunked"); + return 2; + } + cmp = strcasecmp(h->key, "Content-Length"); if (cmp != 0) { continue; @@ -66,6 +72,11 @@ static int http_response_needs_body(struct http_request *req) return 0; } +struct http_chunk +{ + struct http_chunk *prev, *next; + DATA_BLOB blob; +}; struct http_read_response_state { enum http_parser_state parser_state; @@ -73,6 +84,7 @@ struct http_read_response_state { uint64_t max_content_length; DATA_BLOB buffer; struct http_request *response; + struct http_chunk *chunks; }; /** @@ -119,6 +131,11 @@ static enum http_read_status http_parse_headers(struct http_read_response_state ret = http_response_needs_body(state->response); switch (ret) { + case 2: + DEBUG(11, ("%s: need to process chunks... %d\n", __func__, + state->response->response_code)); + state->parser_state = HTTP_READING_CHUNK_SIZE; + break; case 1: if (state->response->remaining_content_length <= state->max_content_length) { DEBUG(11, ("%s: Start of read body\n", __func__)); @@ -162,6 +179,141 @@ error: return status; } +static bool http_response_process_chunks(struct http_read_response_state *state) +{ + struct http_chunk *chunk = NULL; + struct http_request *resp = state->response; + + for (chunk = state->chunks; chunk; chunk = chunk->next) { + DBG_DEBUG("processing chunk of size %zi\n", + chunk->blob.length); + if (resp->body.data == NULL) { + resp->body = chunk->blob; + chunk->blob = data_blob_null; + talloc_steal(resp, resp->body.data); + continue; + } + + resp->body.data = + talloc_realloc(resp, + resp->body.data, + uint8_t, + resp->body.length + chunk->blob.length); + if (!resp->body.data) { + return false; + } + memcpy(resp->body.data + resp->body.length, + chunk->blob.data, + chunk->blob.length); + + resp->body.length += chunk->blob.length; + + TALLOC_FREE(chunk->blob.data); + chunk->blob = data_blob_null; + } + return true; +} + +static enum http_read_status http_read_chunk_term(struct http_read_response_state *state) +{ + enum http_read_status status = HTTP_ALL_DATA_READ; + char *ptr = NULL; + char *line = NULL; + + /* Sanity checks */ + if (!state || !state->response) { + DBG_ERR("%s: Invalid Parameter\n", __func__); + return HTTP_DATA_CORRUPTED; + } + + line = talloc_strndup(state, (char *)state->buffer.data, state->buffer.length); + if (!line) { + DBG_ERR("%s: Memory error\n", __func__); + return HTTP_DATA_CORRUPTED; + } + ptr = strstr(line, "\r\n"); + if (ptr == NULL) { + TALLOC_FREE(line); + return HTTP_MORE_DATA_EXPECTED; + } + + if (strncmp(line, "\r\n", 2) == 0) { + /* chunk terminator */ + if (state->parser_state == HTTP_READING_FINAL_CHUNK_TERM) { + if (http_response_process_chunks(state) == false) { + status = HTTP_DATA_CORRUPTED; + goto out; + } + state->parser_state = HTTP_READING_DONE; + } else { + state->parser_state = HTTP_READING_CHUNK_SIZE; + } + status = HTTP_ALL_DATA_READ; + goto out; + } + + status = HTTP_DATA_CORRUPTED; +out: + TALLOC_FREE(line); + return status; +} + +static enum http_read_status http_read_chunk_size(struct http_read_response_state *state) +{ + enum http_read_status status = HTTP_ALL_DATA_READ; + char *ptr = NULL; + char *line = NULL; + char *value = NULL; + int n = 0; + unsigned long long v; + + /* Sanity checks */ + if (!state || !state->response) { + DBG_ERR("%s: Invalid Parameter\n", __func__); + return HTTP_DATA_CORRUPTED; + } + + line = talloc_strndup(state, (char *)state->buffer.data, state->buffer.length); + if (!line) { + DBG_ERR("%s: Memory error\n", __func__); + return HTTP_DATA_CORRUPTED; + } + ptr = strstr(line, "\r\n"); + if (ptr == NULL) { + TALLOC_FREE(line); + return HTTP_MORE_DATA_EXPECTED; + } + + n = sscanf(line, "%m[^\r\n]\r\n", &value); + if (n != 1) { + DBG_ERR("%s: Error parsing chunk size '%s'\n", __func__, line); + status = HTTP_DATA_CORRUPTED; + goto out; + } + + DBG_DEBUG("Got chunk size string %s\n", value); + n = sscanf(value, "%llx", &v); + if (n != 1) { + DBG_ERR("%s: Error parsing chunk size '%s'\n", __func__, line); + status = HTTP_DATA_CORRUPTED; + goto out; + } + DBG_DEBUG("Got chunk size %llu 0x%llx\n", v, v); + if (v == 0) { + state->parser_state = HTTP_READING_FINAL_CHUNK_TERM; + } else { + state->parser_state = HTTP_READING_CHUNK; + } + state->response->remaining_content_length = v; + status = HTTP_ALL_DATA_READ; +out: + if (value) { + free(value); + } + TALLOC_FREE(line); + return status; +} + /** * Parses the first line of a HTTP response */ @@ -301,6 +453,55 @@ static enum http_read_status http_read_body(struct http_read_response_state *sta return HTTP_ALL_DATA_READ; } +static enum http_read_status http_read_chunk(struct http_read_response_state *state) +{ + struct http_request *resp = state->response; + struct http_chunk *chunk = NULL; + size_t total = 0; + size_t prev = 0; + + if (state->buffer.length < resp->remaining_content_length) { + return HTTP_MORE_DATA_EXPECTED; + } + + for (chunk = state->chunks; chunk; chunk = chunk->next) { + total += chunk->blob.length; + } + + prev = total; + total = total + state->buffer.length; + if (total < prev) { + DBG_ERR("adding chunklen %zu to buf len %zu " + "will overflow\n", + state->buffer.length, + prev); + return HTTP_DATA_CORRUPTED; + } + if (total > state->max_content_length) { + DBG_DEBUG("size %zu exceeds " + "max content len %"PRIu64" skipping body\n", + total, + state->max_content_length); + state->parser_state = HTTP_READING_DONE; + goto out; + } + + /* chunk read */ + chunk = talloc_zero(state, struct http_chunk); + if (chunk == NULL) { + DBG_ERR("%s: Memory error\n", __func__); + return HTTP_DATA_CORRUPTED; + } + chunk->blob = state->buffer; + talloc_steal(chunk, chunk->blob.data); + DLIST_ADD_END(state->chunks, chunk); + state->parser_state = HTTP_READING_CHUNK_TERM; +out: + state->buffer = data_blob_null; + resp->remaining_content_length = 0; + return HTTP_ALL_DATA_READ; +} + static enum http_read_status http_read_trailer(struct http_read_response_state *state) { enum http_read_status status = HTTP_DATA_CORRUPTED; @@ -323,6 +524,16 @@ static enum http_read_status http_parse_buffer(struct http_read_response_state * case HTTP_READING_BODY: return http_read_body(state); break; + case HTTP_READING_FINAL_CHUNK_TERM: + case HTTP_READING_CHUNK_TERM: + return http_read_chunk_term(state); + break; + case HTTP_READING_CHUNK_SIZE: + return http_read_chunk_size(state); + break; + case HTTP_READING_CHUNK: + return http_read_chunk(state); + break; case HTTP_READING_TRAILER: return http_read_trailer(state); break; @@ -530,7 +741,8 @@ static int http_read_response_next_vector(struct tstream_context *stream, case HTTP_MORE_DATA_EXPECTED: { size_t toread = 1; size_t total; - if (state->parser_state == HTTP_READING_BODY) { + if (state->parser_state == HTTP_READING_BODY || + state->parser_state == HTTP_READING_CHUNK) { struct http_request *resp = state->response; toread = resp->remaining_content_length - state->buffer.length; @@ -549,7 +761,7 @@ static int http_read_response_next_vector(struct tstream_context *stream, /* * test if content-length message exceeds the * specified max_content_length - * Note: This check wont be hit at the moment + * Note: This check won't be hit at the moment * due to an existing check in parse_headers * which will skip the body. Check is here * for completeness and to cater for future @@ -558,7 +770,7 @@ static int http_read_response_next_vector(struct tstream_context *stream, if (state->parser_state == HTTP_READING_BODY) { if (total > state->max_content_length) { DBG_ERR("content size %zu exceeds " - "max content len %zu\n", + "max content len %"PRIu64"\n", total, state->max_content_length); return -1; diff --git a/libcli/http/http_internal.h b/libcli/http/http_internal.h index ec17f7e2850..786ace62d84 100644 --- a/libcli/http/http_internal.h +++ b/libcli/http/http_internal.h @@ -28,6 +28,10 @@ enum http_parser_state { HTTP_READING_BODY, HTTP_READING_TRAILER, HTTP_READING_DONE, + HTTP_READING_CHUNK_SIZE, + HTTP_READING_CHUNK, + HTTP_READING_CHUNK_TERM, + HTTP_READING_FINAL_CHUNK_TERM, }; enum http_read_status { diff --git a/python/samba/tests/blackbox/http_chunk.py b/python/samba/tests/blackbox/http_chunk.py index 1b5cc174788..175c60d98a2 100644 --- a/python/samba/tests/blackbox/http_chunk.py +++ b/python/samba/tests/blackbox/http_chunk.py @@ -81,7 +81,7 @@ class HttpChunkBlackboxTests(BlackboxTestCase): try: msg = "one_chunk" resp = self.check_output("%s -U%% -I%s --uri %s" % (COMMAND, os.getenv("SERVER_IP", "localhost"), msg)) - self.assertEqual(msg, resp.decode('utf-8')) + self.assertEqual(msg,resp.decode('utf-8')) except BlackboxProcessError as e: print("Failed with: %s" % e) self.fail(str(e)) diff --git a/selftest/knownfail.d/http_chunks b/selftest/knownfail.d/http_chunks deleted file mode 100644 index e58e002abdb..00000000000 --- a/selftest/knownfail.d/http_chunks +++ /dev/null @@ -1,5 +0,0 @@ -^samba.tests.blackbox.http_chunk.samba.tests.blackbox.http_chunk.HttpChunkBlackboxTests.test_multi_chunks -^samba.tests.blackbox.http_chunk.samba.tests.blackbox.http_chunk.HttpChunkBlackboxTests.test_single_chunk -^samba.tests.blackbox.http_chunk.samba.tests.blackbox.http_chunk.HttpChunkBlackboxTests.test_exact_request_size -^samba.tests.blackbox.http_chunk.samba.tests.blackbox.http_chunk.HttpChunkBlackboxTests.test_exceed_request_size - -- 2.35.3