Botan 3.14.1
Crypto and TLS for C&
http_util.cpp
Go to the documentation of this file.
1/*
2* HTTP 1.0 client
3* (C) 2013,2016,2026 Jack Lloyd
4* 2017 René Korthaus, Rohde & Schwarz Cybersecurity
5*
6* Botan is released under the Simplified BSD License (see license.txt)
7*/
8
9#include <botan/internal/http_util.h>
10
11#include <botan/mem_ops.h>
12#include <botan/uri.h>
13#include <botan/internal/charset.h>
14#include <botan/internal/fmt.h>
15#include <botan/internal/mem_utils.h>
16#include <botan/internal/parsing.h>
17#include <botan/internal/socket.h>
18#include <limits>
19#include <sstream>
20
21namespace Botan::HTTP {
22
23namespace {
24
25constexpr size_t MaxHeaderBytes = 16 * 1024;
26
27// RFC 9110 5.5: "field-content = field-vchar [ 1*( SP / HTAB / field-vchar ) field-vchar ]"
28void check_field_value_chars(std::string_view what, std::string_view value) {
29 for(const char c : value) {
30 const auto b = static_cast<uint8_t>(c);
31 if(b != '\t' && (b < 0x20 || b == 0x7F)) {
32 throw HTTP_Error(fmt("Invalid control character in HTTP {}", what));
33 }
34 }
35}
36
37struct Parsed_Head {
38 unsigned int status_code;
39 std::string status_message;
40 Headers headers;
41};
42
43Parsed_Head parse_status_and_headers(std::string_view block) {
44 const auto first_eol = block.find("\r\n");
45 const auto status_line_end = (first_eol == std::string_view::npos) ? block.size() : first_eol;
46 if(status_line_end == 0) {
47 throw HTTP_Error("No status line");
48 }
49
50 // RFC 9112 4: reason-phrase = 1*( HTAB / SP / VCHAR / obs-text )
51 check_field_value_chars("status line", block.substr(0, status_line_end));
52
53 std::stringstream ss{std::string(block.substr(0, status_line_end))};
54 std::string http_version;
55 unsigned int status_code = 0;
56 ss >> http_version >> status_code;
57 std::string status_message;
58 std::getline(ss, status_message);
59 if(!status_message.empty() && status_message.front() == ' ') {
60 status_message.erase(0, 1);
61 }
62
63 if(!ss || !http_version.starts_with("HTTP/")) {
64 throw HTTP_Error("Not an HTTP response");
65 }
66
67 // RFC 9110 Section 15: "All valid status codes are within the range of 100 to 599, inclusive."
68 if(status_code < 100 || status_code > 599) {
69 throw HTTP_Error(fmt("Invalid HTTP status code {}", status_code));
70 }
71
72 // RFC 9110 5.6.2 tchar
73 constexpr auto is_tchar = CharacterValidityTable::alpha_numeric_plus("!#$%&'*+-.^_`|~");
74 // RFC 9110 5.6.3 OWS = *( SP / HTAB )
75 constexpr auto is_ows = [](char c) { return c == ' ' || c == '\t'; };
76
77 Headers headers;
78 size_t pos = (first_eol == std::string_view::npos) ? block.size() : first_eol + 2;
79 while(pos < block.size()) {
80 const auto eol = block.find("\r\n", pos);
81 const auto line_end = (eol == std::string_view::npos) ? block.size() : eol;
82 const auto line = block.substr(pos, line_end - pos);
83
84 // RFC 9110 5.5: field-line = field-name ":" OWS field-value OWS
85 const auto sep = line.find(':');
86 if(sep == std::string_view::npos || sep == 0) {
87 throw HTTP_Error(fmt("Invalid HTTP header '{}'", line));
88 }
89
90 const auto name = line.substr(0, sep);
91 if(!std::all_of(name.begin(), name.end(), is_tchar)) {
92 throw HTTP_Error(fmt("Invalid HTTP header name '{}'", name));
93 }
94
95 auto value = line.substr(sep + 1);
96 while(!value.empty() && is_ows(value.front())) {
97 value.remove_prefix(1);
98 }
99 while(!value.empty() && is_ows(value.back())) {
100 value.remove_suffix(1);
101 }
102
103 // RFC 9110 5.5: "a recipient of CR, LF, or NUL within a field value MUST
104 // either reject the message or replace each of those characters with SP"
105 check_field_value_chars("header value", value);
106
107 auto [it, inserted] = headers.emplace(std::string(name), std::string(value));
108 if(!inserted) {
109 throw HTTP_Error(fmt("Duplicate HTTP header '{}'", it->first));
110 }
111
112 if(eol == std::string_view::npos) {
113 break;
114 }
115 pos = eol + 2;
116 }
117
118 return {status_code, std::move(status_message), std::move(headers)};
119}
120
121/*
122* Post-header validation shared by the streaming reader and the in-memory
123* parser. Rejects Transfer-Encoding outright (we only speak HTTP/1.0) and
124* enforces Content-Length against max_body_size. Returns the parsed
125* Content-Length on success, if present.
126*/
127std::optional<size_t> validate_response_headers(const Headers& headers, size_t max_body_size) {
128 // RFC 9112 6.1: "A server MUST NOT send a response containing Transfer-Encoding
129 // unless the corresponding request indicates HTTP/1.1 (or later minor revisions)."
130 if(headers.contains("Transfer-Encoding")) {
131 throw HTTP_Error("Server sent Transfer-Encoding header in response to HTTP/1.0 request");
132 }
133
134 std::optional<size_t> content_length;
135 if(const auto it = headers.find("Content-Length"); it != headers.end()) {
136 // RFC 9110 8.6: Content-Length = 1*DIGIT
137 if(const auto cl = parse_sz(it->second)) {
138 content_length = cl;
139 } else {
140 throw HTTP_Error(fmt("Invalid Content-Length value '{}'", it->second));
141 }
142 }
143
144 if(content_length && *content_length > max_body_size) {
145 throw HTTP_Error(fmt("Content-Length {} exceeds maximum body size {}", *content_length, max_body_size));
146 }
147
148 return content_length;
149}
150
151/*
152* Connect to a host, write the request, then delegate to
153* read_response_from_socket. Body- and header-size caps are enforced
154* there; this just owns the socket lifecycle.
155*/
156Response http_transact(std::string_view hostname,
157 std::string_view service,
158 std::string_view message,
159 std::chrono::milliseconds timeout,
160 size_t max_body_size) {
161 std::unique_ptr<OS::Socket> socket;
162 try {
163 socket = OS::open_socket(hostname, service, timeout);
164 if(!socket) {
165 throw Not_Implemented("No socket support enabled in build");
166 }
167 } catch(std::exception& e) {
168 throw HTTP_Error(fmt("HTTP connection to {} failed: {}", hostname, e.what()));
169 }
170
171 socket->write(as_span_of_bytes(message));
172 return read_response_from_socket(*socket, timeout, max_body_size);
173}
174
175void check_no_crlf_nul(std::string_view field, std::string_view value) {
176 for(const char c : value) {
177 if(c == '\r' || c == '\n' || c == '\0') {
178 throw HTTP_Error(fmt("Invalid character in HTTP {}", field));
179 }
180 }
181}
182
183/*
184* Resolve a Location header value against the request URI per RFC 9110 10.2.2.
185* Handles two cases: an absolute URI, or a path-absolute reference (begins
186* with '/' but not '//') which is composed against the request URI's scheme
187* and authority. Other relative forms (network-path "//host/p", protocol-
188* relative, dot-segments) are rejected.
189*/
190std::optional<URI> resolve_location(const URI& base, std::string_view location) {
191 if(auto absolute = URI::from_string(location)) {
192 return absolute;
193 }
194 if(location.starts_with('/') && !location.starts_with("//")) {
195 const auto raw_authority = base.raw_authority();
196 if(!raw_authority.has_value()) {
197 return std::nullopt;
198 }
199 const std::string composed = base.scheme() + "://" + std::string(*raw_authority) + std::string(location);
200 return URI::from_string(composed);
201 }
202 return std::nullopt;
203}
204
205} // namespace
206
207Response read_response_from_socket(OS::Socket& socket, std::chrono::milliseconds timeout, size_t max_body_size) {
208 const auto start_time = std::chrono::steady_clock::now();
209 const auto deadline_exceeded = [&] { return std::chrono::steady_clock::now() - start_time > timeout; };
210
211 if(deadline_exceeded()) {
212 throw HTTP_Error("Timeout before reading response");
213 }
214
215 std::string buf;
216 std::vector<uint8_t> chunk(DefaultBufferSize);
217 size_t header_end = std::string::npos;
218
219 while(header_end == std::string::npos) {
220 const size_t got = socket.read(chunk.data(), chunk.size());
221 if(got == 0) {
222 throw HTTP_Error("Server closed connection before headers complete");
223 }
224 if(deadline_exceeded()) {
225 throw HTTP_Error("Timeout while reading headers");
226 }
227 buf.append(cast_uint8_ptr_to_char(chunk.data()), got);
228 header_end = buf.find("\r\n\r\n");
229 if(header_end == std::string::npos && buf.size() > MaxHeaderBytes) {
230 throw HTTP_Error("HTTP headers exceed maximum size");
231 }
232 }
233
234 // Same cap re-checked once the terminator is found, since the terminator
235 // can arrive in the chunk that crosses the limit.
236 if(header_end > MaxHeaderBytes) {
237 throw HTTP_Error("HTTP headers exceed maximum size");
238 }
239
240 auto parsed = parse_status_and_headers(std::string_view(buf).substr(0, header_end));
241 const auto content_length = validate_response_headers(parsed.headers, max_body_size);
242
243 const size_t body_cap = std::min(max_body_size, content_length.value_or(std::numeric_limits<size_t>::max()));
244
245 std::vector<uint8_t> body;
246 if(content_length) {
247 // Already checked against max_body_size by validate_response_headers
248 body.reserve(*content_length);
249 }
250 const size_t body_start = header_end + 4;
251 if(body_start < buf.size()) {
252 const size_t spill = buf.size() - body_start;
253 if(spill > body_cap) {
254 throw HTTP_Error("Response body exceeds maximum size");
255 }
256 body.insert(body.end(),
257 reinterpret_cast<const uint8_t*>(buf.data() + body_start),
258 reinterpret_cast<const uint8_t*>(buf.data() + buf.size()));
259 }
260
261 while(!content_length || body.size() < *content_length) {
262 const size_t got = socket.read(chunk.data(), chunk.size());
263 if(got == 0) {
264 break;
265 }
266 if(deadline_exceeded()) {
267 throw HTTP_Error("Timeout while reading body");
268 }
269 if(body.size() + got > body_cap) {
270 throw HTTP_Error("Response body exceeds maximum size");
271 }
272 body.insert(body.end(), chunk.data(), chunk.data() + got);
273 }
274
275 if(content_length && body.size() != *content_length) {
276 throw HTTP_Error(fmt("Content-Length disagreement, header says {} got {}", *content_length, body.size()));
277 }
278
279 return Response(parsed.status_code, std::move(parsed.status_message), std::move(body), std::move(parsed.headers));
280}
281
282std::string url_encode(std::string_view in) {
283 constexpr auto needs_url_encoding = CharacterValidityTable::alpha_numeric_plus("-_.~").invert();
284 constexpr std::string_view hex_digits = "0123456789ABCDEF";
285
286 std::string out;
287 out.reserve(in.size());
288 for(const char c : in) {
289 if(needs_url_encoding(c)) {
290 const auto byte = static_cast<uint8_t>(c);
291 out += '%';
292 out += hex_digits[byte >> 4];
293 out += hex_digits[byte & 0x0F];
294 } else {
295 out += c;
296 }
297 }
298 return out;
299}
300
301std::ostream& operator<<(std::ostream& o, const Response& resp) {
302 o << "HTTP " << resp.status_code() << " " << resp.status_message() << "\n";
303 for(const auto& h : resp.headers()) {
304 o << "Header '" << h.first << "' = '" << h.second << "'\n";
305 }
306 o << "Body " << std::to_string(resp.body().size()) << " bytes:\n";
307 o.write(cast_uint8_ptr_to_char(resp.body().data()), resp.body().size());
308 return o;
309}
310
311namespace {
312
313Response http_sync_until(const http_exch_fn& http_transact,
314 std::string_view verb,
315 const URI& uri,
316 std::string_view content_type,
317 const std::vector<uint8_t>& body,
318 const RequestLimits& limits,
319 std::chrono::steady_clock::time_point deadline) {
320 if(uri.scheme() != "http") {
321 throw HTTP_Error(fmt("Cannot initiate HTTP request to URI with scheme of '{}'", uri.scheme()));
322 }
323
324 const auto& authority = uri.authority();
325 if(!authority.has_value()) {
326 throw HTTP_Error("Cannot initiate HTTP request to URI without authority");
327 }
328
329 check_no_crlf_nul("verb", verb);
330 check_no_crlf_nul("content type", content_type);
331
332 const std::string hostname = authority->host_to_string();
333 const auto port = authority->port();
334 const std::string service = port.has_value() ? std::to_string(*port) : uri.scheme();
335
336 // RFC 9112 3.2.1: request-target origin-form is "absolute-path [ '?' query ]".
337 // If the URI has an empty path, the client MUST send "/". Fragment is
338 // excluded from the request-target per RFC 9110 7.1.
339 std::string loc = uri.path().empty() ? "/" : uri.path();
340 if(const auto& q = uri.query()) {
341 loc += '?';
342 loc += *q;
343 }
344
345 const std::string host_header = [&]() -> std::string {
346 const std::string h = (authority->host_kind() == URI::HostKind::IPv6) ? "[" + hostname + "]" : hostname;
347 return port.has_value() ? h + ":" + std::to_string(*port) : h;
348 }();
349
350 std::ostringstream outbuf;
351
352 outbuf << verb << " " << loc << " HTTP/1.0\r\n";
353 outbuf << "Host: " << host_header << "\r\n";
354
355 if(verb == "GET") {
356 outbuf << "Accept: */*\r\n";
357 outbuf << "Cache-Control: no-cache\r\n";
358 } else if(verb == "POST") {
359 outbuf << "Content-Length: " << body.size() << "\r\n";
360 }
361
362 if(!content_type.empty()) {
363 outbuf << "Content-Type: " << content_type << "\r\n";
364 }
365 outbuf << "Connection: close\r\n\r\n";
366 outbuf.write(cast_uint8_ptr_to_char(body.data()), body.size());
367
368 const auto remaining = deadline - std::chrono::steady_clock::now();
369 if(remaining <= std::chrono::steady_clock::duration::zero()) {
370 throw HTTP_Error("Timeout before sending request");
371 }
372
373 // Round up so a positive sub-millisecond remainder is not truncated to zero
374 const auto remaining_ms = std::chrono::ceil<std::chrono::milliseconds>(remaining);
375
376 Response resp = http_transact(hostname, service, outbuf.str(), remaining_ms, limits.max_body_size());
377
378 const auto sc = resp.status_code();
379 const bool is_redirect = (sc == 301 || sc == 302 || sc == 303 || sc == 307 || sc == 308);
380 if(is_redirect) {
381 const auto loc_it = resp.headers().find("Location");
382 if(loc_it != resp.headers().end()) {
383 if(limits.max_redirects() == 0) {
384 throw HTTP_Error("HTTP redirection count exceeded");
385 }
386 auto redir = resolve_location(uri, loc_it->second);
387 if(!redir) {
388 throw HTTP_Error("HTTP redirected to invalid URL");
389 }
390 RequestLimits next = limits;
391 next.set_max_redirects(limits.max_redirects() - 1);
392
393 // 303 (RFC 9110 15.4.4) re-issues as GET; 301/302/307/308 preserve the
394 // original method and content. The POST->GET downgrade allowed for
395 // 301/302 by RFC 9110 15.4.2/3 exists for browser form-submission
396 // legacy and would silently drop the request body, which is wrong here.
397 //
398 // The recursion goes through the same http_exch_fn so a test seam (or
399 // any caller wrapping the network layer) sees every hop.
400 if(sc == 303) {
401 return http_sync_until(http_transact, "GET", *redir, "", std::vector<uint8_t>(), next, deadline);
402 } else {
403 return http_sync_until(http_transact, verb, *redir, content_type, body, next, deadline);
404 }
405 }
406 }
407
408 return resp;
409}
410
411} // namespace
412
413Response http_sync(const http_exch_fn& http_transact,
414 std::string_view verb,
415 const URI& uri,
416 std::string_view content_type,
417 const std::vector<uint8_t>& body,
418 const RequestLimits& limits) {
419 const auto deadline = std::chrono::steady_clock::now() + limits.timeout();
420 return http_sync_until(http_transact, verb, uri, content_type, body, limits, deadline);
421}
422
423Response http_sync(std::string_view verb,
424 const URI& uri,
425 std::string_view content_type,
426 const std::vector<uint8_t>& body,
427 const RequestLimits& limits) {
428 return http_sync(http_transact, verb, uri, content_type, body, limits);
429}
430
431Response GET_sync(const URI& uri, const RequestLimits& limits) {
432 return http_sync("GET", uri, "", std::vector<uint8_t>(), limits);
433}
434
436 std::string_view content_type,
437 const std::vector<uint8_t>& body,
438 const RequestLimits& limits) {
439 return http_sync("POST", uri, content_type, body, limits);
440}
441
442} // namespace Botan::HTTP
static constexpr CharacterValidityTable alpha_numeric_plus(std::string_view extras)
Definition charset.h:114
constexpr CharacterValidityTable invert() const
Definition charset.h:130
std::chrono::milliseconds timeout() const
Definition http_util.h:106
const std::vector< uint8_t > & body() const
Definition http_util.h:74
const Headers & headers() const
Definition http_util.h:76
unsigned int status_code() const
Definition http_util.h:72
std::string status_message() const
Definition http_util.h:78
virtual size_t read(uint8_t buf[], size_t len)=0
static std::optional< URI > from_string(std::string_view raw)
Definition uri.cpp:163
const std::string & scheme() const
Definition uri.h:127
const std::optional< std::string > & query() const
Definition uri.h:161
const std::optional< Authority > & authority() const
Definition uri.h:132
std::optional< std::string_view > raw_authority() const
Definition uri.cpp:129
const std::string & path() const
Definition uri.h:154
const char * name
Definition ffi_tls.h:59
std::string url_encode(std::string_view in)
std::function< Response(std::string_view, std::string_view, std::string_view, std::chrono::milliseconds, size_t)> http_exch_fn
Definition http_util.h:136
Response GET_sync(const URI &uri, const RequestLimits &limits)
std::map< std::string, std::string, Case_Insensitive_Less > Headers
Definition http_util.h:60
Response http_sync(const http_exch_fn &http_transact, std::string_view verb, const URI &uri, std::string_view content_type, const std::vector< uint8_t > &body, const RequestLimits &limits)
Response read_response_from_socket(OS::Socket &socket, std::chrono::milliseconds timeout, size_t max_body_size)
Response POST_sync(const URI &uri, std::string_view content_type, const std::vector< uint8_t > &body, const RequestLimits &limits)
std::ostream & operator<<(std::ostream &o, const Response &resp)
std::unique_ptr< Socket > BOTAN_TEST_API open_socket(std::string_view hostname, std::string_view service, std::chrono::milliseconds timeout)
Definition socket.cpp:491
std::span< const uint8_t > as_span_of_bytes(const char *s, size_t len)
Definition mem_utils.h:59
std::string fmt(std::string_view format, const T &... args)
Definition fmt.h:53
std::optional< size_t > parse_sz(std::string_view input, bool require_canonical)
Definition parsing.cpp:71
const char * cast_uint8_ptr_to_char(const uint8_t *b)
Definition mem_ops.h:323
constexpr size_t DefaultBufferSize
Definition types.h:150