25constexpr size_t MaxHeaderBytes = 16 * 1024;
28void check_field_value_chars(std::string_view what, std::string_view value) {
29 for(
const char c : value) {
30 const auto b =
static_cast<uint8_t
>(c);
31 if(b !=
'\t' && (b < 0x20 || b == 0x7F)) {
32 throw HTTP_Error(
fmt(
"Invalid control character in HTTP {}", what));
38 unsigned int status_code;
39 std::string status_message;
43Parsed_Head parse_status_and_headers(std::string_view block) {
44 const auto first_eol = block.find(
"\r\n");
45 const auto status_line_end = (first_eol == std::string_view::npos) ? block.size() : first_eol;
46 if(status_line_end == 0) {
51 check_field_value_chars(
"status line", block.substr(0, status_line_end));
53 std::stringstream ss{std::string(block.substr(0, status_line_end))};
54 std::string http_version;
55 unsigned int status_code = 0;
56 ss >> http_version >> status_code;
57 std::string status_message;
58 std::getline(ss, status_message);
59 if(!status_message.empty() && status_message.front() ==
' ') {
60 status_message.erase(0, 1);
63 if(!ss || !http_version.starts_with(
"HTTP/")) {
68 if(status_code < 100 || status_code > 599) {
69 throw HTTP_Error(
fmt(
"Invalid HTTP status code {}", status_code));
75 constexpr auto is_ows = [](
char c) {
return c ==
' ' || c ==
'\t'; };
78 size_t pos = (first_eol == std::string_view::npos) ? block.size() : first_eol + 2;
79 while(pos < block.size()) {
80 const auto eol = block.find(
"\r\n", pos);
81 const auto line_end = (eol == std::string_view::npos) ? block.size() : eol;
82 const auto line = block.substr(pos, line_end - pos);
85 const auto sep = line.find(
':');
86 if(sep == std::string_view::npos || sep == 0) {
90 const auto name = line.substr(0, sep);
91 if(!std::all_of(
name.begin(),
name.end(), is_tchar)) {
95 auto value = line.substr(sep + 1);
96 while(!value.empty() && is_ows(value.front())) {
97 value.remove_prefix(1);
99 while(!value.empty() && is_ows(value.back())) {
100 value.remove_suffix(1);
105 check_field_value_chars(
"header value", value);
107 auto [it, inserted] = headers.emplace(std::string(
name), std::string(value));
109 throw HTTP_Error(
fmt(
"Duplicate HTTP header '{}'", it->first));
112 if(eol == std::string_view::npos) {
118 return {status_code, std::move(status_message), std::move(headers)};
127std::optional<size_t> validate_response_headers(
const Headers& headers,
size_t max_body_size) {
130 if(headers.contains(
"Transfer-Encoding")) {
131 throw HTTP_Error(
"Server sent Transfer-Encoding header in response to HTTP/1.0 request");
134 std::optional<size_t> content_length;
135 if(
const auto it = headers.find(
"Content-Length"); it != headers.end()) {
137 if(
const auto cl =
parse_sz(it->second)) {
140 throw HTTP_Error(
fmt(
"Invalid Content-Length value '{}'", it->second));
144 if(content_length && *content_length > max_body_size) {
145 throw HTTP_Error(
fmt(
"Content-Length {} exceeds maximum body size {}", *content_length, max_body_size));
148 return content_length;
156Response http_transact(std::string_view hostname,
157 std::string_view service,
158 std::string_view message,
159 std::chrono::milliseconds timeout,
160 size_t max_body_size) {
161 std::unique_ptr<OS::Socket> socket;
167 }
catch(std::exception& e) {
168 throw HTTP_Error(
fmt(
"HTTP connection to {} failed: {}", hostname, e.what()));
175void check_no_crlf_nul(std::string_view field, std::string_view value) {
176 for(
const char c : value) {
177 if(c ==
'\r' || c ==
'\n' || c ==
'\0') {
178 throw HTTP_Error(
fmt(
"Invalid character in HTTP {}", field));
190std::optional<URI> resolve_location(
const URI& base, std::string_view location) {
194 if(location.starts_with(
'/') && !location.starts_with(
"//")) {
196 if(!raw_authority.has_value()) {
199 const std::string composed = base.
scheme() +
"://" + std::string(*raw_authority) + std::string(location);
208 const auto start_time = std::chrono::steady_clock::now();
209 const auto deadline_exceeded = [&] {
return std::chrono::steady_clock::now() - start_time > timeout; };
211 if(deadline_exceeded()) {
212 throw HTTP_Error(
"Timeout before reading response");
217 size_t header_end = std::string::npos;
219 while(header_end == std::string::npos) {
220 const size_t got = socket.
read(chunk.data(), chunk.size());
222 throw HTTP_Error(
"Server closed connection before headers complete");
224 if(deadline_exceeded()) {
225 throw HTTP_Error(
"Timeout while reading headers");
228 header_end = buf.find(
"\r\n\r\n");
229 if(header_end == std::string::npos && buf.size() > MaxHeaderBytes) {
230 throw HTTP_Error(
"HTTP headers exceed maximum size");
236 if(header_end > MaxHeaderBytes) {
237 throw HTTP_Error(
"HTTP headers exceed maximum size");
240 auto parsed = parse_status_and_headers(std::string_view(buf).substr(0, header_end));
241 const auto content_length = validate_response_headers(parsed.headers, max_body_size);
243 const size_t body_cap = std::min(max_body_size, content_length.value_or(std::numeric_limits<size_t>::max()));
245 std::vector<uint8_t> body;
248 body.reserve(*content_length);
250 const size_t body_start = header_end + 4;
251 if(body_start < buf.size()) {
252 const size_t spill = buf.size() - body_start;
253 if(spill > body_cap) {
254 throw HTTP_Error(
"Response body exceeds maximum size");
256 body.insert(body.end(),
257 reinterpret_cast<const uint8_t*
>(buf.data() + body_start),
258 reinterpret_cast<const uint8_t*
>(buf.data() + buf.size()));
261 while(!content_length || body.size() < *content_length) {
262 const size_t got = socket.
read(chunk.data(), chunk.size());
266 if(deadline_exceeded()) {
267 throw HTTP_Error(
"Timeout while reading body");
269 if(body.size() + got > body_cap) {
270 throw HTTP_Error(
"Response body exceeds maximum size");
272 body.insert(body.end(), chunk.data(), chunk.data() + got);
275 if(content_length && body.size() != *content_length) {
276 throw HTTP_Error(
fmt(
"Content-Length disagreement, header says {} got {}", *content_length, body.size()));
279 return Response(parsed.status_code, std::move(parsed.status_message), std::move(body), std::move(parsed.headers));
284 constexpr std::string_view hex_digits =
"0123456789ABCDEF";
287 out.reserve(in.size());
288 for(
const char c : in) {
289 if(needs_url_encoding(c)) {
290 const auto byte =
static_cast<uint8_t
>(c);
292 out += hex_digits[
byte >> 4];
293 out += hex_digits[
byte & 0x0F];
303 for(
const auto& h : resp.
headers()) {
304 o <<
"Header '" << h.first <<
"' = '" << h.second <<
"'\n";
306 o <<
"Body " << std::to_string(resp.
body().size()) <<
" bytes:\n";
313Response http_sync_until(
const http_exch_fn& http_transact,
314 std::string_view verb,
316 std::string_view content_type,
317 const std::vector<uint8_t>& body,
318 const RequestLimits& limits,
319 std::chrono::steady_clock::time_point deadline) {
320 if(uri.
scheme() !=
"http") {
321 throw HTTP_Error(
fmt(
"Cannot initiate HTTP request to URI with scheme of '{}'", uri.
scheme()));
325 if(!authority.has_value()) {
326 throw HTTP_Error(
"Cannot initiate HTTP request to URI without authority");
329 check_no_crlf_nul(
"verb", verb);
330 check_no_crlf_nul(
"content type", content_type);
332 const std::string hostname = authority->host_to_string();
333 const auto port = authority->port();
334 const std::string service = port.has_value() ? std::to_string(*port) : uri.scheme();
339 std::string loc = uri.
path().empty() ?
"/" : uri.
path();
340 if(
const auto& q = uri.
query()) {
345 const std::string host_header = [&]() -> std::string {
346 const std::string h = (authority->host_kind() == URI::HostKind::IPv6) ?
"[" + hostname +
"]" : hostname;
347 return port.has_value() ? h +
":" + std::to_string(*port) : h;
350 std::ostringstream outbuf;
352 outbuf << verb <<
" " << loc <<
" HTTP/1.0\r\n";
353 outbuf <<
"Host: " << host_header <<
"\r\n";
356 outbuf <<
"Accept: */*\r\n";
357 outbuf <<
"Cache-Control: no-cache\r\n";
358 }
else if(verb ==
"POST") {
359 outbuf <<
"Content-Length: " << body.size() <<
"\r\n";
362 if(!content_type.empty()) {
363 outbuf <<
"Content-Type: " << content_type <<
"\r\n";
365 outbuf <<
"Connection: close\r\n\r\n";
368 const auto remaining = deadline - std::chrono::steady_clock::now();
369 if(remaining <= std::chrono::steady_clock::duration::zero()) {
370 throw HTTP_Error(
"Timeout before sending request");
374 const auto remaining_ms = std::chrono::ceil<std::chrono::milliseconds>(remaining);
376 Response resp = http_transact(hostname, service, outbuf.str(), remaining_ms, limits.max_body_size());
379 const bool is_redirect = (sc == 301 || sc == 302 || sc == 303 || sc == 307 || sc == 308);
381 const auto loc_it = resp.headers().find(
"Location");
382 if(loc_it != resp.headers().end()) {
383 if(limits.max_redirects() == 0) {
384 throw HTTP_Error(
"HTTP redirection count exceeded");
386 auto redir = resolve_location(uri, loc_it->second);
388 throw HTTP_Error(
"HTTP redirected to invalid URL");
390 RequestLimits next = limits;
391 next.set_max_redirects(limits.max_redirects() - 1);
401 return http_sync_until(http_transact,
"GET", *redir,
"", std::vector<uint8_t>(), next, deadline);
403 return http_sync_until(http_transact, verb, *redir, content_type, body, next, deadline);
414 std::string_view verb,
416 std::string_view content_type,
417 const std::vector<uint8_t>& body,
419 const auto deadline = std::chrono::steady_clock::now() + limits.
timeout();
420 return http_sync_until(http_transact, verb, uri, content_type, body, limits, deadline);
425 std::string_view content_type,
426 const std::vector<uint8_t>& body,
428 return http_sync(http_transact, verb, uri, content_type, body, limits);
432 return http_sync(
"GET", uri,
"", std::vector<uint8_t>(), limits);
436 std::string_view content_type,
437 const std::vector<uint8_t>& body,
439 return http_sync(
"POST", uri, content_type, body, limits);