catching up (trying to) wiht CSpot

This commit is contained in:
Philippe G
2022-01-06 18:46:57 -08:00
parent 491d0d260d
commit 9af4cd5b23
57 changed files with 2165 additions and 343 deletions

View File

@@ -5,14 +5,31 @@
using namespace bell;
struct HTTPClient::HTTPResponse *HTTPClient::execute(const struct HTTPRequest &request) {
auto *response = new HTTPResponse();
auto *url = request.url.c_str();
HTTPClient::executeImpl(request, url, response);
return response;
void HTTPClient::HTTPResponse::close() {
socket = nullptr;
if (buf)
free(buf);
buf = nullptr;
bufPtr = nullptr;
}
HTTPClient::HTTPResponse::~HTTPResponse() {
socket = nullptr;
if (buf)
free(buf);
}
void HTTPClient::executeImpl(const struct HTTPRequest &request, const char *url, struct HTTPResponse *&response) {
HTTPResponse_t HTTPClient::execute(const struct HTTPRequest &request) {
auto response = std::make_unique<HTTPResponse>();
response->dumpFs = request.dumpFs;
response->dumpRawFs = request.dumpRawFs;
return HTTPClient::executeImpl(request, std::move(response));
}
HTTPResponse_t HTTPClient::executeImpl(const struct HTTPRequest &request, HTTPResponse_t response) {
const char *url = request.url.c_str();
if (response->isRedirect) {
url = response->location.c_str();
}
bool https = url[4] == 's';
uint16_t port = https ? 443 : 80;
auto *hostname = url + (https ? 8 : 7);
@@ -45,9 +62,9 @@ void HTTPClient::executeImpl(const struct HTTPRequest &request, const char *url,
stream << path << " HTTP/1.1" << endl;
stream << "Host: " << hostnameStr << ":" << port << endl;
stream << "Accept: */*" << endl;
if (!request.body.empty()) {
if (request.body != nullptr) {
stream << "Content-Type: " << request.contentType << endl;
stream << "Content-Length: " << request.body.size() << endl;
stream << "Content-Length: " << strlen(request.body) << endl;
}
for (const auto &header : request.headers) {
stream << header.first << ": " << header.second << endl;
@@ -60,9 +77,7 @@ void HTTPClient::executeImpl(const struct HTTPRequest &request, const char *url,
if (len != data.size()) {
response->close();
BELL_LOG(error, "http", "Writing failed: wrote %d of %d bytes", len, data.size());
free(response);
response = nullptr;
return;
return nullptr;
}
response->readHeaders();
@@ -70,8 +85,9 @@ void HTTPClient::executeImpl(const struct HTTPRequest &request, const char *url,
if (response->isRedirect && (request.maxRedirects < 0 || response->redirectCount < request.maxRedirects)) {
response->redirectCount++;
response->close(); // close the previous socket
HTTPClient::executeImpl(request, response->location.c_str(), response);
return HTTPClient::executeImpl(request, std::move(response));
}
return response;
}
bool HTTPClient::readHeader(const char *&header, const char *name) {
@@ -87,8 +103,9 @@ bool HTTPClient::readHeader(const char *&header, const char *name) {
size_t HTTPClient::HTTPResponse::readRaw(char *dst) {
size_t len = this->socket->read((uint8_t *)dst, BUF_SIZE);
if (dumpRawFs)
dumpRawFs->write(dst, (long)len);
// BELL_LOG(debug, "http", "Read %d bytes", len);
this->bodyRead += len; // after reading headers this gets overwritten
dst[len] = '\0';
return len;
}
@@ -128,7 +145,6 @@ void HTTPClient::HTTPResponse::readHeaders() {
if (lineEnd + 2 < this->buf + len) {
this->bufPtr = lineEnd + 2;
this->bufRemaining = len - (this->bufPtr - this->buf);
this->bodyRead = this->bufRemaining;
this->isStreaming =
!this->isComplete && !this->contentLength && (len < BUF_SIZE || this->socket->poll() == 0);
}
@@ -151,7 +167,7 @@ void HTTPClient::HTTPResponse::readHeaders() {
this->isRedirect = true;
this->location = std::string(header);
} else {
char *colonPtr = (char*) strchr(header, ':');
auto *colonPtr = strchr((char *)header, ':');
if (colonPtr) {
auto *valuePtr = colonPtr + 1;
while (*valuePtr == ' ')
@@ -166,10 +182,10 @@ void HTTPClient::HTTPResponse::readHeaders() {
lineBuf.clear();
line = lineEnd + 2; // skip \r\n
} while (true);
} while (!complete);
} while (!complete && len); // if len == 0, the connection is closed
}
bool HTTPClient::HTTPResponse::skip(size_t len, bool dontRead) {
bool HTTPClient::HTTPResponse::skipRaw(size_t len, bool dontRead) {
size_t skip = 0;
if (len > bufRemaining) {
skip = len - bufRemaining;
@@ -184,7 +200,7 @@ bool HTTPClient::HTTPResponse::skip(size_t len, bool dontRead) {
}
bufRemaining = this->readRaw(this->buf);
if (!bufRemaining)
return false; // no more data - shouldn't happen for valid responses
return false; // if len == 0, the connection is closed
bufPtr = this->buf + skip;
bufRemaining -= skip;
if (!contentLength && bufRemaining < BUF_SIZE) {
@@ -195,16 +211,15 @@ bool HTTPClient::HTTPResponse::skip(size_t len, bool dontRead) {
return true;
}
size_t HTTPClient::HTTPResponse::read(char *dst, size_t toRead) {
size_t HTTPClient::HTTPResponse::read(char *dst, size_t toRead, bool wait) {
if (isComplete) {
// end of chunked stream was found OR complete body was read
dst[0] = '\0';
return 0;
}
auto *dstStart = dst;
auto *dstStart = dst ? dst : nullptr;
size_t read = 0;
while (toRead) { // this loop ends after original toRead
skip(0); // ensure the buffer contains data, wait if necessary
skipRaw(0); // ensure the buffer contains data, wait if necessary
if (isChunked && !chunkRemaining) {
if (*bufPtr == '0') { // all chunks were read *and emitted*
isComplete = true;
@@ -213,7 +228,7 @@ size_t HTTPClient::HTTPResponse::read(char *dst, size_t toRead) {
auto *endPtr = bufPtr;
if (strchr(bufPtr, '\r') == nullptr) { // buf doesn't contain complete chunk size
auto size = std::string(bufPtr, bufPtr + bufRemaining); // take the rest of the buffer
if (!skip(bufRemaining)) // skip the rest, read another buf
if (!skipRaw(bufRemaining)) // skip the rest, read another buf
break; // -> no more data
endPtr = strchr(bufPtr, '\r'); // find the end of the actual number
if (endPtr == nullptr) // something's wrong
@@ -223,41 +238,51 @@ size_t HTTPClient::HTTPResponse::read(char *dst, size_t toRead) {
} else {
chunkRemaining = strtol(bufPtr, &endPtr, 16); // read the hex size
}
if (!skip(endPtr - bufPtr + 2)) // skip the size and \r\n
break; // -> no more data, break out of main loop
if (!skipRaw(endPtr - bufPtr + 2)) // skip the size and \r\n
break; // -> no more data, break out of main loop
} else if (contentLength && !chunkRemaining) {
chunkRemaining = contentLength;
}
while (chunkRemaining && toRead) {
size_t count = std::min(toRead, std::min(bufRemaining, chunkRemaining));
strncpy(dst, bufPtr, count);
dst += count; // move the dst pointer
if (dst) {
memcpy(dst, bufPtr, count);
dst += count; // move the dst pointer
}
read += count; // increment read counter
bodyRead += count; // increment total response size
chunkRemaining -= count; // decrease chunk remaining size
toRead -= count; // decrease local remaining size
if (!skip(count)) { // eat some buffer
if (!skipRaw(count)) { // eat some buffer
toRead = 0; // -> no more data, break out of main loop
break;
}
if (isChunked && !chunkRemaining && !skip(2, isStreaming)) // skip the \r\n for chunked encoding
toRead = 0; // -> no more data, break out of main loop
if (isChunked && !chunkRemaining) { // bufPtr is on the end of chunk
if (!skipRaw(2, isStreaming)) // skip the \r\n for chunked encoding
toRead = 0; // -> no more data, break out of main loop
if (bufRemaining > 1 && bufPtr[0] == '0' && bufPtr[1] == '\r') // this is the last chunk
isComplete = true;
}
}
if (isStreaming && !bufRemaining) { // stream with no buffer available, just yield the current chunk
if (isStreaming && !bufRemaining && !wait) { // stream with no buffer available, just yield the current chunk
break;
}
}
if (!isChunked && contentLength && !chunkRemaining)
isComplete = true;
isComplete = true; // entire response was read
if (dumpFs && dstStart)
dumpFs->write(dstStart, (long)read);
// BELL_LOG(debug, "http", "Read %d of %d bytes", bodyRead, contentLength);
dstStart[read] = '\0';
return read;
}
std::string HTTPClient::HTTPResponse::readToString() {
if (this->contentLength) {
std::string result(this->contentLength, '\0');
this->read(result.data(), this->contentLength);
auto *data = result.data();
auto len = this->read(data, this->contentLength);
data[len] = '\0';
this->close();
return result;
}
@@ -266,6 +291,7 @@ std::string HTTPClient::HTTPResponse::readToString() {
size_t len;
do {
len = this->read(buffer, BUF_SIZE);
buffer[len] = '\0';
result += std::string(buffer);
} while (len);
this->close();