This commit is contained in:
fwastring 2025-10-13 10:38:50 +02:00
commit ab9a0bd4e2
183 changed files with 20701 additions and 0 deletions

98
src/http/cookiefile.cpp Normal file
View file

@ -0,0 +1,98 @@
// Copyright (c) 2019 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
#include <fstream>
#include <iterator>
#include "cookiefile.h"
#include "../http/http_common.h"
CookieFile::~CookieFile() {
if (isDirty())
writeCookiesFile();
}
void CookieFile::readCookiesFile() {
if (filename.empty())
return;
std::ifstream cookieFile(filename);
if (!cookieFile) {
// Failed, but who cares?
log() << "cannot read cookie file " << filename;
return;
}
log() << "reading cookie file " << filename;
std::string domain;
std::string line;
auto it = cookies().end();
while (std::getline(cookieFile, line)) {
if (line.size() < 10) {
continue;
} else if (line[0] == '*') {
if (line.substr(0, 5) == "**** " &&
line.substr(line.size()-5) == " ****") {
domain = line.substr(5, line.size()-10);
it = cookies().find(domain);
} else {
err_log() << "bad cookie file";
domain.clear();
}
continue;
} else if (domain.empty()) {
continue;
} else {
// name, value, domain, expires, path, secure, httponly
std::vector<std::string> vec = HttpCommon::split(line, ";");
if (vec.size() != Field::field_len)
continue;
if (it == cookies().end()) {
std::vector<std::string> v(Cache::cache_len);
auto p = cookies().insert(std::make_pair(domain, v));
it = p.first;
}
auto &v = it->second;
std::move(vec.begin(), vec.end(), back_inserter(v));
}
}
clearDirty();
}
void CookieFile::writeCookiesFile() {
if (filename.empty()) {
log() << "missing name of cookie file";
return;
}
std::ofstream cookieFile(filename, std::ofstream::trunc);
for (auto p : cookies()) {
const std::string &domain = p.first;
const std::vector<std::string> &v = p.second;
bool said_domain = false;
std::size_t i=Cache::cache_len;
while (i+Field::field_len <= v.size()) {
if (v[i+Field::expires].empty() ||
isExpired(v[i+Field::expires])) {
// Session cookie or expired, don't persist.
i += Field::field_len;
continue;
} else if (!said_domain) {
cookieFile << "**** " << domain << " ****\n";
said_domain = true;
}
cookieFile << v[i++];
for (std::size_t j=1; j<Field::field_len; ++j)
cookieFile << ';' << v[i++];
cookieFile << '\n';
}
}
cookieFile.close();
if (cookieFile) {
clearDirty();
} else {
// Failed, but who cares?
log() << "cannot write cookie file";
}
}

37
src/http/cookiefile.h Normal file
View file

@ -0,0 +1,37 @@
// Copyright (c) 2019 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
#pragma once
#include <map>
#include "../http/cookiemanager.h"
class CookieFile : public CookieManager {
public:
CookieFile(const std::string &cookie_filename = "") :
CookieManager("CookieFile"),
filename(cookie_filename) {
readCookiesFile();
}
// By default, we try to save cookies in the destructor. However, if
// the save operation fails, all updates are lost. If you can't afford to
// lose cookies, call save() and check the return value.
~CookieFile() override;
// Write to disk, return false on failure.
bool save() override {
if (isDirty())
writeCookiesFile();
return isDirty();
}
// Default move constructor is OK despite us having a destructor:
CookieFile(CookieFile &&) = default;
private:
void readCookiesFile();
void writeCookiesFile();
std::string filename;
};

288
src/http/cookiemanager.cpp Normal file
View file

@ -0,0 +1,288 @@
// Copyright (c) 2019 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
#include "cookiemanager.h"
#include "http_common.h"
#include <algorithm>
#include <stdexcept>
#include <iterator>
CookieManager::~CookieManager() {
}
bool CookieManager::save() {
return false;
}
// timeval contains only digits 0-9, doesn't start with 0.
bool CookieManager::isExpired(const std::string &timeval) {
if (timeval.empty())
return false; // No expiry - a session cookie.
if (time_t d = time(nullptr) - t_now) {
t_now += d;
s_now = std::to_string(t_now);
}
return !less(s_now, timeval);
}
// line is the remainder of a header line starting with "Set-Cookie: ", e.g.
// previous_isp=23; expires=Wed, 07-Jun-2017 11:34:59 GMT; path=/;
// domain=.bredbandskollen.se
void CookieManager::setCookie(const std::string &line,
std::string domain, std::string uri) {
std::vector<std::string> cookie_av = cookieSplit(line);
if (cookie_av.empty())
return;
const std::string &cdomain = cookie_av[Field::domain];
// Disallow third-party cookies and cookies for top-level domains.
// TODO: should disallow all public suffixes, e.g. ".co.uk".
if (cdomain.find('.') != std::string::npos &&
HttpCommon::isSubdomain(domain, cdomain))
domain = cdomain;
auto it = store.find(domain);
if (it == store.end()) {
std::vector<std::string> v(Cache::cache_len);
auto p = store.insert(std::make_pair(domain, v));
it = p.first;
} else {
// Invalidate cache, also for subdomains.
it->second[Cache::cookie_header].clear();
for (auto p : store)
if (HttpCommon::isSubdomain(p.first, domain))
p.second[Cache::cookie_header].clear();
}
auto &v = it->second;
bool expired = isExpired(cookie_av[Field::expires]);
if (!expired)
if (cookie_av[Field::path].empty() || cookie_av[Field::path][0] != '/')
cookie_av[Field::path] = HttpCommon::uriPath(uri); // default-path
// Check if the new cookie replaces an existing
for (size_t i=Cache::cache_len; i<v.size(); i+=Field::field_len) {
if (v[i+Field::name] == cookie_av[Field::name] &&
v[i+Field::path] == cookie_av[Field::path]) {
if (expired) {
v.erase(v.begin() + static_cast<long>(i),
v.begin() + static_cast<long>(i+Field::field_len));
} else {
std::move(cookie_av.begin(), cookie_av.end(),
v.begin() + static_cast<long>(i));
dirty = true;
cookie_av.clear();
return;
}
break;
}
}
if (!expired) {
std::move(cookie_av.begin(), cookie_av.end(),
back_inserter(v));
dirty = true;
}
}
// line is the remainder of a header line starting with "Set-Cookie: ", e.g.
// previous_isp=23; expires=Wed, 07-Jun-2017 11:34:59 GMT; path=/;
// domain=.bredbandskollen.se
// Return vector with Field::field_len elements if OK:
// name, value, domain, expires, path, secure, httponly
// Return empty vector if bad.
std::vector<std::string> CookieManager::cookieSplit(const std::string &line) {
std::vector<std::string> v = HttpCommon::split(line, "; "), cookie_av;
for (auto av : v) {
std::string lcname, value, avlc;
auto eqpos = av.find('=');
if (!cookie_av.empty()) {
// Make a lower-case copy:
std::transform(av.begin(), av.end(), back_inserter(avlc), tolower);
HttpCommon::trimWSP(lcname = avlc.substr(0, eqpos));
if (eqpos != std::string::npos)
HttpCommon::trimWSP(value = av.substr(eqpos+1));
} else {
if (eqpos == std::string::npos)
break; // Bad line
}
if (cookie_av.empty()) {
cookie_av.resize(Field::field_len);
HttpCommon::trimWSP(cookie_av[Field::name] = av.substr(0, eqpos));
HttpCommon::trimWSP(cookie_av[Field::value] = av.substr(eqpos+1));
} else if (eqpos == 6 && lcname == "domain") {
++eqpos;
if (av[eqpos] == '.')
++eqpos; // Ignore leading dot.
cookie_av[Field::domain] = avlc.substr(eqpos);
} else if (eqpos == 8 && lcname == "_expires") {
// Our internal format, unix timestamp value.
++eqpos;
if (av.find_first_not_of("0123456789", eqpos) == std::string::npos)
cookie_av[Field::expires] = av.substr(eqpos);
else // Corrupt cookie file?
cookie_av[Field::expires] = "0";
} else if (eqpos == 7 && lcname == "expires") {
// max-age has precedence, so ignore if already set.
if (cookie_av[Field::expires].empty()) {
time_t val = HttpCommon::parseDateRfc1123(av.substr(eqpos+1));
if (val >= 0) {
cookie_av[Field::expires] = std::to_string(val);
} else if (val == -1) {
// Ignore bad date
} else {
// Expired.
cookie_av[Field::expires] = "0";
}
}
} else if (eqpos == 4 && lcname == "path") {
cookie_av[Field::path] = av.substr(eqpos+1);
} else if (eqpos == 7 && lcname == "max-age") {
++eqpos;
if (av.size() == eqpos) {
// Ignore?
} else if (av[eqpos] == '-') {
if (av.find_first_not_of("0123456789", ++eqpos)
== std::string::npos)
cookie_av[Field::expires] = "0"; // Mark as expired
// Else ignore!
} else if (av.find_first_not_of("0123456789", eqpos) ==
std::string::npos) {
time_t res, maxv = std::numeric_limits<time_t>::max();
try {
long secs = std::stol(av.substr(eqpos));
res = time(nullptr);
if (secs >= maxv-res)
res = maxv;
else
res += secs;
} catch (std::out_of_range &) {
// Too large
res = maxv;
}
cookie_av[Field::expires] = std::to_string(res);
}
} else if (avlc == "secure") {
cookie_av[Field::secure] = avlc;
} else if (avlc == "httponly") {
cookie_av[Field::secure] = avlc;
} else {
// Unknown attribute; ignore.
}
}
return cookie_av;
}
std::string CookieManager::httpHeaderLine(const std::string &domain,
const std::string &uri) {
auto cit = store.find(domain);
if (cit == store.end()) {
// Make room for cache!
std::vector<std::string> v(Cache::cache_len);
auto p = store.insert(std::make_pair(domain, v));
cit = p.first;
} else {
// Check if cached value exists and hasn't expired:
const std::vector<std::string> &v = cit->second;
if (!v.empty() && !v[Cache::cookie_header].empty() &&
!isExpired(v[Cache::expiry])) {
// Cached value exists, no part of it has expired.
if (v[Cache::cookie_header].size() == 1)
return "";
else
return v[Cache::cookie_header];
}
}
// We'll set this to true if we can't cache the result, e.g.
// if it depends on path:
bool dont_cache = false;
// Find all subdomains for which we have cookies:
std::vector<std::map<std::string, std::vector<std::string>>::iterator>
subdomains;
std::string subdomain = domain;
auto it = cit;
while (true) {
if (it != store.end())
subdomains.push_back(it);
auto pos = subdomain.find('.');
if (pos == std::string::npos)
break; // We don't accept top level domains.
subdomain.erase(0, pos+1);
it = store.find(subdomain);
}
// Get all cookie values, starting from least significant subdomain:
std::string cache_expiry;
std::map<std::string, std::string> cookieVal;
while (!subdomains.empty()) {
// Reevaluate:
auto &v = subdomains.back()->second;
for (size_t i=Cache::cache_len; i<v.size(); ) {
const std::string &expiry = v[i+Field::expires];
if (!expiry.empty() &&
(cache_expiry.empty() || less(expiry, cache_expiry))) {
if (isExpired(expiry)) {
v.erase(v.begin()+static_cast<long>(i),
v.begin()+static_cast<long>(i+Field::field_len));
continue;
} else
cache_expiry = expiry;
}
if (v[i+Field::path] != "/") {
dont_cache = true;
if (HttpCommon::isWithinPath(uri, v[i+Field::path]))
cookieVal[v[i+Field::name]] = v[i+Field::value];
} else
cookieVal[v[i+Field::name]] = v[i+Field::value];
i += Field::field_len;
}
subdomains.pop_back();
}
auto p = cookieVal.begin();
if (p == cookieVal.end()) {
cit->second[Cache::cookie_header] = "-";
cit->second[Cache::expiry].clear();
return "";
}
std::string cval = "Cookie: " + p->first + "=" + p->second;
while (++p != cookieVal.end()) {
cval += ("; " + p->first + "=" + p->second);
}
cval += "\r\n";
if (!dont_cache) {
cit->second[Cache::cookie_header] = cval;
cit->second[Cache::expiry] = cache_expiry;
}
return cval;
}
void CookieManager::eraseCookies(const std::string &domain) {
auto cit = store.find(domain);
if (cit == store.end())
return;
store.erase(cit);
dirty = true;
}
std::string CookieManager::getCookieVal(const std::string &name,
std::string domain) {
while (true) {
auto cit = store.find(domain);
if (cit != store.end()) {
const auto &v = cit->second;
for (unsigned i=Cache::cache_len; i<v.size(); i+=Field::field_len)
if (v[i+Field::name] == name && v[i+Field::path] == "/")
return v[i+Field::value];
}
auto pos = domain.find('.');
if (pos == std::string::npos)
return std::string();
domain.erase(0, pos+1);
}
}

95
src/http/cookiemanager.h Normal file
View file

@ -0,0 +1,95 @@
// Copyright (c) 2019 The Swedish Internet Foundation
// Written by Göran Andersson <goran@init.se>
// This class stores cookies in memory only.
// Create a subclass if you want persistent cookies.
#pragma once
#include <map>
#include <vector>
#include "../framework/logger.h"
class CookieManager : public Logger {
public:
CookieManager(const std::string &name = "CookieManager") :
Logger(name) {
}
virtual ~CookieManager();
void setCookie(const std::string &line, std::string domain,
std::string uri);
// Return empty string if no cookies exist, or a HTTP header line
// "Cookie: name1=val1; name2=val2\r\n"
std::string httpHeaderLine(const std::string &domain,
const std::string &uri);
// Return cookie value for domain and name; return empty if not found.
std::string getCookieVal(const std::string &name, std::string domain);
// Return true if no cookies have been set.
bool empty() const {
return store.empty();
}
void eraseCookies(const std::string &domain);
// Store persistently, return true on success
virtual bool save();
protected:
bool isDirty() const {
return dirty;
}
void clearDirty() {
dirty = false;
}
// timeval contains only digits 0-9, doesn't start with 0.
bool isExpired(const std::string &timeval);
// both must be non-empty, only digits, no leading zeroes.
static bool less(const std::string &tv1, const std::string &tv2) {
if (tv1.size() < tv2.size())
return true;
if (tv1.size() > tv2.size())
return false;
return tv1 < tv2;
}
// Access to all cookies, to serialise/deserialise.
// The key is the domain for which the cookie is valid (also subdomains).
// The value is a vector with elements as follows:
// The first Cache::cache_len elements are used to cache values and must be
// ignored. The remaining elements are 0 or more groups of Field::field_len
// elements; each group describes the properties of a single cookie.
std::map<std::string, std::vector<std::string> > &cookies() {
return store;
}
enum Cache {
cookie_header, expiry, cache_len
};
enum Field {
name, value, domain, expires, path, secure, httponly, field_len
};
private:
// Map domain to cookie strings:
// First element of each store vector is a cache for httpHeaderLine():
// - empty string means nothing is cached
// - string of length 1 means there are no cookies for the domain
// - string of length > 1 is the cookie header line
// - NOTE! We only cache for path=/
// Second element is cache expiry time, empty for no expiry.
std::map<std::string, std::vector<std::string> > store;
// TODO: Store time of creation / last use?
// last use may ignore cache.
static std::vector<std::string> cookieSplit(const std::string &line);
TimePoint expire_date;
bool dirty = false;
time_t t_now = 0;
std::string s_now = "0";
};

173
src/http/http_common.cpp Normal file
View file

@ -0,0 +1,173 @@
// Copyright (c) 2018 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
#include "http_common.h"
#include <stdlib.h>
#include <time.h>
bool HttpCommon::parseHeaders(const std::string &header, std::string &r,
std::multimap<std::string, std::string> &res) {
std::string::size_type start, end=0;
while (true) {
start = end;
end = header.find("\r\n", start);
if (end == std::string::npos)
return false;
std::string line = header.substr(start, end-start);
if (line.empty()) {
break;
} else {
// Remove trailing spaces
std::size_t n = line.find_last_not_of(' ');
if (n != std::string::npos)
line.resize(n+1);
}
end += std::strlen("\r\n");
if (!start) {
r = line;
continue;
}
std::string::size_type separator = line.find(": ");
if (separator == std::string::npos)
return false;
std::string attribute = line.substr(0, separator);
for (std::string::size_type i=0; i<attribute.size(); ++i)
if (attribute[i] >= 'A' && attribute[i] <= 'Z')
attribute[i] += ('a' - 'A');
res.insert(std::make_pair(attribute, line.substr(separator+2)));
}
return true;
}
namespace {
std::map<std::string, const char *> mime_db = {
{ "txt", "text/plain; charset=utf-8" },
{ "html", "text/html; charset=utf-8" },
{ "css", "text/css" },
{ "js", "text/javascript" },
{ "pdf", "application/pdf" },
{ "xml", "application/xml" },
{ "json", "application/json" },
};
}
const char *HttpCommon::mime_type(const std::string &file_name) {
auto pos = file_name.find_last_of('.');
if (pos == std::string::npos)
pos = 0;
else
++pos;
auto p = mime_db.find(file_name.substr(pos));
if (p != mime_db.end())
return p->second;
return "application/octet-stream";
}
namespace {
inline int toInt(char p1, char p2) {
if (!isdigit(p1) || !isdigit(p2))
return -1;
return 10*(p1-'0') + (p2-'0');
}
}
// Return -1 on failure.
time_t HttpCommon::parseDateRfc1123(const std::string &date) {
time_t ret = -1;
// Date format:
// Wed, 07-Jun-2017 11:34:59 GMT
// 012345678901234567890123456789
if (date.size() != 29)
return ret;
static const std::string wdays("Sun, Mon, Tue, Wed, Thu, Fri, Sat, ");
auto wday_pos = wdays.find(date.substr(0, 5));
if (wday_pos == std::string::npos || wday_pos%5)
return ret;
static const std::string months("-Jan-Feb-Mar-Apr-May-Jun-Jul-Aug-Sep-Oct-Nov-Dec-");
auto mon_pos = months.find(date.substr(7, 5));
if (mon_pos == std::string::npos || mon_pos%4)
return ret;
if (date[16] != ' ' || date[19] != ':' || date[22] != ':' ||
date.substr(25) != " GMT")
return ret;
struct tm timespec;
timespec.tm_sec = toInt(date[23], date[24]);
timespec.tm_min = toInt(date[20], date[21]);
timespec.tm_hour = toInt(date[17], date[18]);
int mday = toInt(date[5], date[6]);
timespec.tm_mday = mday;
if (timespec.tm_sec < 0 || timespec.tm_sec > 59 ||
timespec.tm_min < 0 || timespec.tm_min > 59 ||
timespec.tm_hour < 0 || timespec.tm_hour > 23 || mday < 0)
return ret;
timespec.tm_mon = static_cast<int>(mon_pos / 4);
timespec.tm_year = toInt(date[14], date[15]);
int yy = toInt(date[12], date[13]) - 19;
if (timespec.tm_year < 0 || yy < 0)
return ret;
timespec.tm_year += yy*100;
//timespec.tm_wday = 0;
//timespec.tm_yday = 0;
timespec.tm_isdst = -1;
// mktime depends on current timezone, so we must
// temporarily reset timezone.
char *tz = getenv("TZ");
#ifdef _WIN32
_putenv("TZ=UTC");
_tzset();
ret = mktime(&timespec);
if (tz) {
char buf[255];
snprintf(buf, sizeof(buf), "TZ=%s", tz);
_putenv(buf);
} else {
_putenv("TZ=");
}
_tzset();
#else
setenv("TZ", "", 1);
tzset();
ret = mktime(&timespec);
if (tz)
setenv("TZ", tz, 1);
else
unsetenv("TZ");
tzset();
if (timespec.tm_wday*5 != static_cast<int>(wday_pos)
|| timespec.tm_mday != mday)
return -1;
#endif
return ret;
}
void HttpCommon::trimWSP(std::string &s) {
auto pos = s.find_first_not_of(" \t\r\n\v");
if (pos && pos != std::string::npos)
s.erase(0, pos);
pos = s.find_last_not_of(" \t\r\n\v");
if (pos != std::string::npos)
s.erase(++pos);
}
std::vector<std::string> HttpCommon::split(const std::string &s,
const std::string &sep) {
std::vector<std::string> v;
std::string::size_type pos = 0;
while (true) {
auto epos = s.find(sep, pos);
v.push_back(s.substr(pos, epos-pos));
if (epos == std::string::npos)
break;
pos = epos + sep.size();
}
return v;
}

62
src/http/http_common.h Normal file
View file

@ -0,0 +1,62 @@
// Copyright (c) 2018 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
#pragma once
#include <string>
#include <cstring>
#include <ctime>
#include <vector>
#include <map>
class HttpCommon {
public:
// Return true on success.
static bool parseHeaders(const std::string &header, std::string &r,
std::multimap<std::string, std::string> &res);
// Guess mime type, e.g. "text/html", from url/filename
static const char *mime_type(const std::string &file_name);
// Return -1 on failure.
static time_t parseDateRfc1123(const std::string &date);
// Remove leading and trailing whitespace:
static void trimWSP(std::string &s);
// Split s into fields sepatared by sep:
static std::vector<std::string> split(const std::string &s,
const std::string &sep);
static bool isSubdomain(const std::string &subdomain,
const std::string &domain) {
auto s = domain.size();
return (subdomain.size() > s &&
subdomain[subdomain.size()-s-1] == '.' &&
subdomain.substr(subdomain.size()-s) == domain);
}
static bool isWithinPath(const std::string &uri,
const std::string &path) {
if (path.empty())
return (!uri.empty() && uri[0] == '/');
if (path != uri.substr(0, path.size()))
return false;
if (path.size() < uri.size()) {
return path[path.size()-1] == '/' ||
uri[path.size()] == '/';
} else if (path.size() == uri.size()) {
return true; // Identical.
} else {
return false;
}
}
// Return the path part of an uri.
static std::string uriPath(std::string uri) {
auto qpos = uri.find('?');
if (qpos != std::string::npos)
uri.erase(qpos);
qpos = uri.rfind('/');
if (qpos != std::string::npos)
uri.erase(qpos+1);
return uri;
}
};

View file

@ -0,0 +1,397 @@
// Copyright (c) 2018 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
#ifdef _WIN32
#define NOMINMAX
#endif
#include <iomanip>
#include <cctype>
#include <stdexcept>
#include "httpclientconnection.h"
#include "httpclienttask.h"
#include "http_common.h"
#include "sha1.h"
HttpClientConnection::HttpClientConnection(const std::string &label,
HttpClientTask *task,
const std::string &hostname,
uint16_t port,
const std::string &http_hostname,
uint16_t iptype,
struct addrinfo *local_addr) :
HttpConnection(label, task, hostname, port, iptype, local_addr),
real_hostname(http_hostname.empty() ? hostname : http_hostname),
owner_task(task) {
if (!http_hostname.empty())
proto_hostname = "http://" + http_hostname;
}
PollState HttpClientConnection::connected() {
owner_task->newRequest(this);
if (current_request == "pass")
return PollState::NONE;
if (current_request.empty())
return PollState::KEEPALIVE;
//dbg_log() << "HttpClientConnection::connected() request: " << current_request;
asyncSendData(current_request);
if (bytes_left_to_post)
return PollState::WRITE;
else
return PollState::READ;
}
bool HttpClientConnection::parseChunkedEncodingLength(char *&buf, size_t &len) {
// Between chunks, there is \r\n, a hex number, and \r\n again.
// The number gives the size of next chunk.
// The last size is 0, and then an extra \r\n to terminate the request.
size_t oldlen = chunked_info.size();
chunked_info += std::string(buf, len<20 ? len : 20);
if (final_chunk) {
len = 0;
// We're waiting for the trailing \r\n.
if (chunked_info == "\r\n") {
// OK, all done!
response_is_chunked = false;
return true;
}
return (chunked_info == "\r");
}
if (chunked_info.size() < 5) {
// Wait for more, keeping the fragment in chunked_info
len = 0;
return true;
}
if (chunked_info.substr(0, 2) != "\r\n")
return false;
size_t endOfSizePos = chunked_info.find("\r\n", 2);
if (endOfSizePos == std::string::npos) {
// Haven't got terminating \r\n yet.
// Wait for more if len < 20, otherwise fail.
len = 0;
return chunked_info.size() < 20;
}
size_t to_skip = endOfSizePos + 2 - oldlen;
buf += to_skip;
len -= to_skip;
try {
std::size_t count;
std::string num = "0x"+ chunked_info.substr(2);
bytes_left_to_read = std::stoul(num, &count, 16);
if (count != endOfSizePos)
return false;
} catch (...) {
return false;
}
if (!bytes_left_to_read) {
final_chunk = true;
}
chunked_info.clear();
return true;
}
PollState HttpClientConnection::readData(char *buf, size_t len) {
if (dbgIsOn()) {
log() << socket() << "HttpClientConnection <" << std::string(buf, len) << ">";
}
if (!response_header_complete) {
std::string::size_type old_length = response_header.size();
std::string::size_type to_append = len;
if (old_length + to_append > max_header_length)
to_append = max_header_length - old_length;
response_header.append(buf, to_append);
std::string::size_type pos = response_header.find("\r\n\r\n");
if (pos == std::string::npos) {
if (response_header.size() >= max_header_length)
return PollState::KILL;
else
return PollState::READ;
}
response_header.resize(pos + strlen("\r\n\r\n"));
if (!parseResponseHeaders())
return PollState::CLOSE;
len -= (response_header.size() - old_length);
if (len)
buf += (response_header.size() - old_length);
else if (!response_header_complete) {
response_header.clear();
http_response_headers.clear();
return PollState::READ; // e.g. after PROXY CONNECT
}
}
if (is_websocket)
return wsReadData(buf, len);
if (dbgIsOn()) {
dbg_log() << socket() << " HEADER OK, PAYLOAD <"
<< std::string(buf, len) << "> LEFT: "
<< bytes_left_to_read << " CH: " << response_is_chunked;
}
// Reading payload
if (len) {
if (response_is_chunked && !bytes_left_to_read) {
if (!parseChunkedEncodingLength(buf, len)) {
err_log() << "content encoding error";
return PollState::KILL;
}
}
if (bytes_left_to_read) {
size_t to_read = std::min(len, bytes_left_to_read);
if (buffer_response)
buffer.append(buf, to_read);
else
owner_task->payload(this, buf, to_read);
bytes_left_to_read -= to_read;
len -= to_read;
buf += to_read;
}
}
if (response_is_chunked) {
if (len > 0)
return readData(buf, len);
else
return PollState::READ;
}
if (bytes_left_to_read > 0)
return PollState::READ;
bool to_continue = owner_task->requestComplete(this);
if (len) {
// Error
err_log() << "got " << len << " bytes after request complete";
return PollState::KILL;
}
if (to_continue) {
current_request.clear();
current_uri.clear();
response_header.clear();
response_header_complete = false;
response_http_status = 0;
buffer.clear();
http_response_headers.clear();
return connected(); // New request or close.
} else {
return PollState::KEEPALIVE;
}
}
bool HttpClientConnection::parseResponseHeaders() {
if (response_http_status) {
err_log() << "internal error: headers already parsed";
return false;
}
response_header_complete = true;
std::string response_first_line;
if (!HttpCommon::parseHeaders(response_header, response_first_line,
http_response_headers)) {
log() << "Bad HTTP headers: " << response_header;
return false;
}
{
// HTTP/1.1 200 OK
unsigned int major, minor, status;
int pos = sscanf(response_first_line.c_str(), "HTTP/%u.%u %u",
&major, &minor, &status);
if (pos != 3 || !status)
return false;
set_http_version(major, minor);
response_http_status = status;
}
for (auto p = http_response_headers.lower_bound("set-cookie");
p!=http_response_headers.upper_bound("set-cookie"); ++p)
owner_task->setCookie(p->second, current_uri);
if (!websocket_rkey.empty()) {
if (websocket_rkey == "CONNECT") {
// Doing HTTP CONNECT through a proxy
log() << "PROXY CONNECT status " << response_http_status;
websocket_rkey.clear();
proto_hostname.clear(); // No more proxy calls
if (response_http_status < 200 || response_http_status >= 300)
return false;
response_header_complete = false;
response_http_status = 0;
current_request.clear();
ws_get(current_uri);
asyncSendData(current_request);
return true;
}
auto p = http_response_headers.find("sec-websocket-accept");
if (p == http_response_headers.end() || p->second != websocket_rkey) {
err_log() << "wrong Sec-WebSocket-Accept";
return false;
}
is_websocket = true;
response_is_chunked = false;
bytes_left_to_read = 0;
return owner_task->websocketUpgrade(this);
}
try {
auto p = http_response_headers.find("transfer-encoding");
if (p != http_response_headers.end() && p->second == "chunked") {
response_is_chunked = true;
chunked_info = "\r\n";
final_chunk = false;
bytes_left_to_read = 0;
} else {
p = http_response_headers.find("content-length");
if (p == http_response_headers.end()) {
err_log() << "cannot find Content-Length";
return false;
}
response_is_chunked = false;
bytes_left_to_read = std::stoul(p->second);
}
} catch (std::exception &err) {
err_log() << "Exception when parsing HTTP response: " << err.what();
return false;
}
return owner_task->headerComplete(this);
}
PollState HttpClientConnection::writeData() {
if (is_websocket)
return wsWriteData();
bytes_left_to_post -= owner_task->doPost(this, bytes_left_to_post);
if (bytes_left_to_post)
return PollState::WRITE;
else
return PollState::READ;
}
std::string HttpClientConnection::urlencode(const std::string &val) {
std::ostringstream res;
res.fill('0');
res << std::hex;
for (auto i=val.cbegin(); i!=val.cend(); ++i) {
std::string::value_type c = *i;
if (std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') {
res << c;
} else {
res << '%' << std::setw(2)
<< std::uppercase << int(static_cast<unsigned char>(c))
<< std::nouppercase;
}
}
return res.str();
}
void HttpClientConnection::
addUrlPars(std::string &url, const std::map<std::string, std::string> &pars) {
auto p = pars.begin();
if (p == pars.end())
return;
if (url.find('?') == std::string::npos)
url += "?";
else
url += "&";
((url += p->first) += "=") += urlencode(p->second);
while (++p != pars.end())
(((url += "&") += p->first) += "=") += urlencode(p->second);
}
void HttpClientConnection::get(const std::string &url) {
if (!idle()) {
err_log() << "previous request not complete: " << current_request;
return;
}
current_uri = url;
current_request = "GET " + proto_hostname +
url + " HTTP/1.1\r\nHost: " + real_hostname +
"\r\n" + owner_task->httpHeaderLines(url) + "\r\n";
}
void HttpClientConnection::ws_get(const std::string &url) {
if (!idle()) {
err_log() << "previous request not complete: " << current_request;
return;
}
if (!proto_hostname.empty()) {
current_uri = url;
websocket_rkey = "CONNECT";
current_request = "CONNECT " + real_hostname + " HTTP/1.1\r\n"
"Hostname: " + real_hostname + "\r\n\r\n";
return;
}
// The first 24 chars will be filled with random base64 chars,
// the rest is the websocket uuid.
static char dst[] {
"01234567890123456789====258EAFA5-E914-47DA-95CA-C5AB0DC85B11" };
// 16 byte random data, convert to 24 base64 chars:
int32_t rnd[4] = { rand(), rand(), rand(), rand() };
const unsigned char *src = reinterpret_cast<const unsigned char *>(rnd);
base64_encode(src, sizeof(rnd), dst);
current_uri = url;
current_request = "GET " + proto_hostname +
url + " HTTP/1.1\r\nHost: " + real_hostname +
"\r\nSec-WebSocket-Key: " + std::string(dst, 24) +
"\r\nSec-WebSocket-Version: 13\r\nUpgrade: websocket"
"\r\nConnection: Upgrade"
"\r\n" + owner_task->httpHeaderLines(url) + "\r\n";
static char rkey[] = "012345678901234567890123456=";
static SHA1 sha1(rkey);
sha1.update(dst);
// The server's response is supposed to contain this string:
websocket_rkey = std::string(rkey, 28);
}
void HttpClientConnection::post(const std::string &url,
const std::string &data) {
if (!idle()) {
err_log() << "previous request not complete: " << current_request;
return;
}
current_uri = url;
current_request = "POST " + proto_hostname + url + " HTTP/1.1\r\nHost: " +
real_hostname + "\r\n" + owner_task->httpHeaderLines(url) +
"Content-Length: " + std::to_string(data.size()) + "\r\n\r\n" + data;
}
void HttpClientConnection::post(const std::string &url, size_t len) {
if (!idle()) {
err_log() << "previous request not complete: " << current_request;
return;
}
current_uri = url;
current_request = "POST " + proto_hostname + url + " HTTP/1.1\r\nHost: " +
real_hostname + "\r\n" + owner_task->httpHeaderLines(url) +
"Content-Length: " + std::to_string(len) + "\r\n\r\n";
bytes_left_to_post = len;
}
std::string HttpClientConnection::cacheLabel() {
return owner_task->cacheLabel();
}
void HttpClientConnection::setOwner(Task *new_owner) {
owner_task = dynamic_cast<HttpClientTask *>(new_owner);
if (!owner_task)
throw std::logic_error("expected HttClientTask");
Socket::setOwner(new_owner);
}

View file

@ -0,0 +1,160 @@
// Copyright (c) 2018 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
#pragma once
#include <map>
#include <sstream>
#include "httpconnection.h"
class HttpClientTask;
/// \brief
/// HTTP/1.1 client protocol.
///
/// This class implements a small subset of the HTTP/1.1 client protocol.
/// A connection will be made to the given hostname/port.
/// When the connection is ready for a new HTTP request, the owner
/// task's method newRequest(HttpClientConnection *conn) will be called.
/// To use this class, you must derive from HttpClientTask and implement the
/// newRequest method to perform HTTP requests over the HttpClientConnection.
///
/// A call to the owner task's headerComplete will be made when response
/// headers are read. If headerComplete returns false the connection
/// will be dropped.
///
/// Within headerComplete you may call conn->doStreamResponse()
/// if you want the respone to be "streamed" to you, i.e. to be notified
/// (by a call to the owner task's payload method) whenever a part of the
/// response arrives. The default is for this class to buffer the response
/// and only notify the ower task when the complete response has been received.
///
/// A call to the owner task's requestComplete will be made when the complete
/// response has been read. If requestComplete returns false the connection
/// will be dropped. Otherwise, the newRequest will be called again so that
/// a new HTTP request can be made.
/// Within requestComplete, you can call contents() to get the HTTP response
/// unless doStreamResponse() was called before.
class HttpClientConnection : public HttpConnection {
public:
// TODO: take a HttpHost as a parameter!
HttpClientConnection(const std::string &label, HttpClientTask *task,
const std::string &hostname, uint16_t port,
const std::string &http_hostname = std::string(),
uint16_t iptype = 0,
struct addrinfo *local_addr = nullptr);
PollState connected() final;
PollState readData(char *buf, size_t len) final;
/// For HTTP POST.
PollState writeData() final;
/// The owner task should call this in the header_complete
/// callback if it wants to get partial contents immediately when it
/// arrives. The default is for this class to buffer the response and
/// pass it to the owner task when the complete response has arrived.
void doStreamResponse() {
buffer_response = false;
}
/// The owner task may call the below functions in the header_complete
/// callback or the response_complete callback.
unsigned int httpStatus() const {
return response_http_status;
}
/// Return value of first Content-Type header, empty string on failure.
std::string contentType() const {
auto p = http_response_headers.find("content-type");
if (p == http_response_headers.end())
return std::string();
else
return p->second;
}
const std::string &rawHttpHeader() const {
return response_header;
}
const std::multimap<std::string, std::string> &responseHeaders() const {
return http_response_headers;
}
size_t remainingContentLength() const {
return bytes_left_to_read;
}
std::string contents() const {
return buffer;
}
/// May be called in the newRequest callback:
void get(const std::string &url);
/// May be called in the newRequest callback:
/// The owner task's doPost method will be called repeatedly until it
/// returns false, which means there is no more data to post.
void post(const std::string &url, size_t len);
// As above, but with first chunk of data available. Using this method
// may save one call to doPost(), i.e. one less send system call.
/*
void post(const std::string &url, size_t len,
const char *buf, size_t buffer_len, size_t lspace = 0);
*/
/// May be called in the newRequest callback:
/// If data is very large, you should use the above method instead.
void post(const std::string &url, const std::string &data);
/// May be called in the newRequest callback to open a websocket.
void ws_get(const std::string &url);
/// Call in the newRequest callback if you want to keep
/// the connection but not do any new request at this time.
void pass() {
current_request = "pass";
}
bool idle() const {
return current_request.empty() ||
current_request == "pass";
}
static std::string urlencode(const std::string &val);
static void addUrlPars(std::string &url,
const std::map<std::string, std::string> &pars);
std::string cacheLabel() override;
void setOwner(Task *new_owner) override;
private:
std::string real_hostname;
// A weak pointer to the owner task. Dangerous. However, the EventLoop will
// kill all connections (i.e. us) before killing the owner task.
HttpClientTask *owner_task;
std::string proto_hostname;
std::string current_request, current_uri;
// Set if this is a websocket connection:
std::string websocket_rkey;
// Used when reading response headers:
std::string response_header;
const std::string::size_type max_header_length = 20000;
bool response_header_complete = false;
// Set when response headers are parsed:
bool response_is_chunked;
bool final_chunk;
bool buffer_response = true;
size_t bytes_left_to_read = 0;
size_t bytes_left_to_post = 0;
unsigned int response_http_status = 0;
std::string chunked_info;
bool parseResponseHeaders();
std::multimap<std::string, std::string> http_response_headers;
bool parseChunkedEncodingLength(char *&buf, size_t &len);
};

View file

@ -0,0 +1,52 @@
#include <stdexcept>
#include "httpclienttask.h"
HttpClientTask::HttpClientTask(const std::string &name,
const HttpHost &httpserver) :
HttpTask(name),
peer(httpserver) {
if (!peer.proxyHost.empty()) {
real_server = peer.hostname + ":" + std::to_string(peer.port);
}
}
bool HttpClientTask::createNewConnection() {
HttpClientConnection *conn = real_server.empty() ?
new HttpClientConnection(label(), this, peer.hostname, peer.port, "",
peer.iptype, local_ip) :
new HttpClientConnection(label(), this,
peer.proxyHost, peer.proxyPort,
real_server, peer.iptype, local_ip);
#ifdef USE_GNUTLS
if (peer.is_tls)
conn->enableTLS();
#endif
return addConnection(conn);
}
bool HttpClientTask::headerComplete(HttpClientConnection *) {
return true;
}
size_t HttpClientTask::doPost(HttpClientConnection *, size_t ) {
throw std::runtime_error("doPost method not implemented");
}
bool HttpClientTask::websocketUpgrade(HttpClientConnection *) {
return true;
}
std::string HttpClientTask::local_address;
struct addrinfo *HttpClientTask::local_ip = nullptr;
bool HttpClientTask::setLocalAddress(const std::string &addr, uint16_t iptype) {
local_address = addr;
if (local_address.empty()) {
local_ip = nullptr;
return true;
}
Socket sobj("temp", nullptr, local_address, 0);
local_ip = sobj.getAddressInfo(iptype);
return (local_ip != nullptr);
}

122
src/http/httpclienttask.h Normal file
View file

@ -0,0 +1,122 @@
// Copyright (c) 2018 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
#pragma once
#include "httptask.h"
#include "cookiemanager.h"
#include "httpclientconnection.h"
/// API for HTTP clients.
class HttpClientTask : public HttpTask {
public:
HttpClientTask(const std::string &name, const HttpHost &httpserver);
/// Initiate next request, or ignore to close connection.
virtual void newRequest(HttpClientConnection *) {
}
/// \brief
/// Called when response headers are fully read and parsed,
/// except for websocket upgrades, where websocketUpgrade is called instead.
///
/// Override it to send a HTTP request to the server.
///
/// Return false to close the connection, true to continue.
/// If not overridden, it will always return true.
virtual bool headerComplete(HttpClientConnection *);
/// \brief
/// Called after succesful websocket upgrade.
///
/// Override it to start sending websocket messages to the server.
///
/// Return false to close the connection, true to continue.
///
/// If not overridden, it will always return true.
virtual bool websocketUpgrade(HttpClientConnection *);
/// Called when response has been fully read.
virtual bool requestComplete(HttpClientConnection *) = 0;
/// \brief
/// Data has arrived from the server.
///
/// Called whenever some part of the content arrives, *if* you
/// asked for it by calling doStreamResponse() on conn
/// in the HttpClientConnection::headerComplete callback.
virtual void payload(HttpClientConnection *, char *, size_t ) {
}
/// \brief
/// Send POST data, return number of bytes sent.
///
/// You _must_ override this method if you intend to stream post requests.
/// Typically, you simply do this:
///
/// return conn->sendData(my_buffer, std::min(my_buffer_len, len));
virtual size_t doPost(HttpClientConnection *conn, size_t len);
/// Return server's host name.
std::string serverHost() const {
return peer.hostname;
}
/// \brief
/// Extra header lines to be added to outgoing requests.
///
/// If you override this, return zero or more lines ending with "\r\n".
/// Unless you want to disable cookies, append the return value of
///
/// cookiemgr->httpHeaderLine(server, uri)
virtual std::string httpHeaderLines(const std::string &uri) const {
if (!peer.cmgr)
return user_agent_header;
return peer.cmgr->httpHeaderLine(peer.hostname, uri) +
user_agent_header;
}
void setCookie(const std::string &header_line, const std::string &uri) {
if (peer.cmgr)
peer.cmgr->setCookie(header_line, peer.hostname, uri);
}
virtual std::string cacheLabel() {
return peer.hostname + std::to_string(peer.port);
}
void setUserAgentString(const std::string &s) {
if (s.find_first_of("\r\n") == std::string::npos)
user_agent_header = "User-Agent: " + s + "\r\n";
}
/// \brief
/// Bind all subsequent client sockets to the given local ip address
///
/// Don't call simultaneously from more than one thread
static bool setLocalAddress(const std::string &addr, uint16_t iptype);
static const std::string &getLocalAddress() {
return local_address;
}
protected:
bool createNewConnection();
std::string proxyHost() const {
return peer.proxyHost;
}
uint32_t proxyPort() const {
return peer.proxyPort;
}
CookieManager *cookieMgr() const {
return peer.cmgr;
}
void setServer(const std::string hostname, uint16_t port_number = 80) {
peer.hostname = hostname;
peer.port = port_number;
}
private:
HttpHost peer;
std::string user_agent_header;
std::string real_server;
static std::string local_address;
static struct addrinfo *local_ip;
};

359
src/http/httpconnection.cpp Normal file
View file

@ -0,0 +1,359 @@
// Copyright (c) 2018 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
#ifdef _WIN32
#include <winsock2.h>
#else
#include <arpa/inet.h>
#endif
#include "httpconnection.h"
#include "sha1.h"
#include "webservertask.h"
void HttpConnection::send_ws_handshake(const std::string &key) {
static const std::string resp = "HTTP/1.1 101 Switching Protocols\r\n"
"Connection: Upgrade\r\n"
"Upgrade: websocket\r\n"
"Sec-WebSocket-Accept: ";
#ifdef USE_THREADS
thread_local
#endif
static char rkey[] = "012345678901234567890123456=";
#ifdef USE_THREADS
thread_local
#endif
static SHA1 sha1(rkey);
sha1.update((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").c_str());
asyncSendData(resp + rkey + "\r\n\r\n");
}
PollState HttpConnection::wsReadData(const char *buf, size_t len) {
//log() << "wsReadData L=" << len << ", RCV: " << receiving_message;
if (receiving_message) {
if (!stream_incoming)
return incoming_ws_data(buf, len);
size_t count;
if (len >= bytes_to_receive) {
len -= bytes_to_receive;
count = bytes_to_receive;
bytes_to_receive = 0;
receiving_message = false;
} else {
bytes_to_receive -= len;
count = len;
len = 0;
}
HttpTask *owner_task = dynamic_cast<HttpTask *>(owner());
if (!owner_task)
return PollState::CLOSE;
bool ret;
if (incoming_is_binary)
ret = owner_task->wsBinData(this, buf, count);
else
ret = owner_task->wsTextData(this, buf, count);
if (!ret)
return PollState::CLOSE;
if (!len)
return (sending_message ? PollState::READ_WRITE : PollState::READ);
buf += count;
}
if (buffer.empty())
return incoming_ws_header(buf, len);
std::string tmp = buffer;
tmp.append(buf, len);
buffer.clear();
return incoming_ws_header(tmp.data(), tmp.size());
}
PollState HttpConnection::incoming_ws_data(const char *buf, size_t len) {
// Unmask, append to buffer (up to bytes_to_receive)
size_t count;
if (len >= bytes_to_receive) {
count = bytes_to_receive;
len -= bytes_to_receive;
bytes_to_receive = 0;
} else {
count = len;
bytes_to_receive -= len;
len = 0;
}
if (incoming_is_masked) {
const unsigned char *msg =
reinterpret_cast<const unsigned char *>(buf);
buf += count;
while (count) {
unsigned char c = *msg++ ^ incoming_mask[buffer.size()%4];
buffer.push_back(static_cast<char>(c));
--count;
}
} else {
buffer.append(buf, count);
buf += count;
}
if (!bytes_to_receive) {
HttpTask *owner_task = dynamic_cast<HttpTask *>(owner());
if (!owner_task)
return PollState::CLOSE;
bool ret;
if (incoming_is_binary)
ret = owner_task->wsBinMessage(this, buffer);
else
ret = owner_task->wsTextMessage(this, buffer);
if (!ret)
return PollState::CLOSE;
buffer.clear();
receiving_message = false;
}
if (len)
return wsReadData(buf, len);
return (sending_message ? PollState::READ_WRITE : PollState::READ);
}
PollState HttpConnection::incoming_ws_header(const char *buf, size_t buffer_pos) {
if (buffer_pos < 2) {
buffer.append(buf, buffer_pos);
return (sending_message ? PollState::READ_WRITE : PollState::READ);
}
const unsigned char *msg =
reinterpret_cast<const unsigned char *>(buf);
size_t hdr_len;
size_t msg_len = msg[1] & 0x7f;
if (msg_len < 126) {
hdr_len = 2;
} else if (msg_len == 126) {
// 2 bytes message length
hdr_len = 4;
if (buffer_pos >= hdr_len)
msg_len = ntohs(*reinterpret_cast<const uint16_t *>(msg+2));
} else {
// msg_len == 127, 8 byte message length
hdr_len = 10;
/* Will not support message size > 4GB
msg_len = (((uint64_t) ntohl(*(uint32_t *) (msg+2))) << 32) +
ntohl(*(uint32_t *) (msg+6));
*/
if (buffer_pos >= hdr_len)
msg_len = ntohl(*reinterpret_cast<const uint32_t *>(msg+6));
}
incoming_is_masked = msg[1] & 0x80;
if (incoming_is_masked)
hdr_len += 4;
if (buffer_pos < hdr_len) {
buffer.append(buf, buffer_pos);
return (sending_message ? PollState::READ_WRITE : PollState::READ);
}
if (incoming_is_masked)
memcpy(incoming_mask, msg + hdr_len - 4, 4);
unsigned char opcode = msg[0] & 0xf;
bool is_final = (msg[0] & 0x80);
if (opcode)
current_opcode = is_final ? 0 : opcode;
else
opcode = current_opcode;
HttpTask *owner_task = dynamic_cast<HttpTask *>(owner());
if (!owner_task)
return PollState::CLOSE;
switch (opcode) {
case 1:
stream_incoming = false;
receiving_message = true;
incoming_is_binary = false;
bytes_to_receive = msg_len;
tot_to_receive = msg_len;
if (!owner_task->wsTextHeader(this, msg_len))
return PollState::CLOSE;
break;
case 2:
bytes_to_receive = msg_len;
tot_to_receive = msg_len;
stream_incoming = false;
receiving_message = true;
incoming_is_binary = true;
if (!owner_task->wsBinHeader(this, msg_len))
return PollState::CLOSE;
break;
case 8:
// Closed by client
log() << "ws closed by peer";
return PollState::CLOSE;
case 9:
// Ping
send_ws_pong();
break;
case 10:
// Pong, ignore.
break;
default:
err_log() << "Bad websocket opcode " << static_cast<int>(msg[0])
<< ' ' << static_cast<int>(msg[1]);
return PollState::CLOSE;
}
buffer_pos -= hdr_len;
if (buffer_pos)
return wsReadData(buf+hdr_len, buffer_pos);
return (sending_message ? PollState::READ_WRITE : PollState::READ);
}
void HttpConnection::send_ws_bin_header(size_t len) {
if (sending_message) {
log() << "Internal error: sending message while streaming";
closeMe();
return;
}
unsigned char hdr[11];
size_t pos = 0;
hdr[pos++] = 0x80 | 0x02; // Final fragment, type binary
if (len > 65535) {
hdr[pos++] = 127;
uint32_t l = 0;
memcpy(&hdr[pos], &l, 4);
pos += 4;
l = htonl(static_cast<uint32_t>(len));
memcpy(&hdr[pos], &l, 4);
pos += 4;
} else {
hdr[pos++] = 126;
uint16_t l = htons(static_cast<uint16_t>(len));
memcpy(&hdr[pos], &l, 2);
pos += 2;
}
asyncSendData(reinterpret_cast<const char *>(hdr), pos);
}
void HttpConnection::send_ws_txt_header(size_t len) {
if (sending_message) {
log() << "Internal error: sending message while streaming";
closeMe();
return;
}
// TODO: support for masking the message.
unsigned char hdr[11];
unsigned int pos = 0;
hdr[pos++] = 0x80 | 0x01; // Final fragment, type text
if (len < 126) {
hdr[pos++] = static_cast<unsigned char>(len);
} else if (len < 65536) {
hdr[pos++] = 126;
uint16_t l = htons(static_cast<uint16_t>(len));
memcpy(&hdr[pos], &l, 2);
pos += 2;
} else {
hdr[pos++] = 127;
uint32_t l = 0;
memcpy(&hdr[pos], &l, 4);
pos += 4;
l = htonl(static_cast<uint32_t>(len));
memcpy(&hdr[pos], &l, 4);
pos += 4;
}
asyncSendData(reinterpret_cast<char *>(hdr), pos);
}
void HttpConnection::sendWsMessage(const std::string &msg) {
size_t len = msg.size();
if (sending_message) {
log() << "Internal error: sending message while streaming";
closeMe();
return;
}
// TODO: support for masking the message.
unsigned char hdr[11];
unsigned int pos = 0;
hdr[pos++] = 0x80 | 0x01; // Final fragment, type text
if (len < 126) {
hdr[pos++] = static_cast<unsigned char>(len);
} else if (len < 65536) {
hdr[pos++] = 126;
uint16_t l = htons(static_cast<uint16_t>(len));
memcpy(&hdr[pos], &l, 2);
pos += 2;
} else {
hdr[pos++] = 127;
uint32_t l = 0;
memcpy(&hdr[pos], &l, 4);
pos += 4;
l = htonl(static_cast<uint32_t>(len));
memcpy(&hdr[pos], &l, 4);
pos += 4;
}
asyncSendData(std::string(reinterpret_cast<char *>(hdr), pos) + msg);
}
void HttpConnection::sendWsBinary(const char *buf, size_t len) {
send_ws_bin_header(len);
asyncSendData(buf, len);
}
void HttpConnection::sendWsClose(uint16_t code, std::string msg) {
if (msg.size() > 124)
msg.resize(124); // Will only allow a very short message.
unsigned char hdr[4];
hdr[0] = 0x80 | 0x08; // Final fragment, opcode CLOSE
// Will send 2 byte error code, then the msg:
hdr[1] = static_cast<unsigned char>(2 + msg.size());
// The error code:
uint16_t l = htons(code + 2);
memcpy(hdr+2, &l, 2);
asyncSendData(std::string(reinterpret_cast<char *>(hdr), 4) + msg);
closeMe();
}
void HttpConnection::send_ws_pong() {
unsigned char hdr[2];
hdr[0] = 0x80 | 0x0a; // Final fragment, opcode PONG
hdr[1] = 0; // No payload, length 0.
asyncSendData(reinterpret_cast<char *>(hdr), 2);
}
void HttpConnection::startWsStream(size_t len, bool is_binary) {
// TODO: also text
log() << "startWsStream " << len;
if (is_binary)
send_ws_bin_header(len);
else
send_ws_txt_header(len);
bytes_to_send = len;
tot_to_send = len;
sending_message = true;
output_is_binary = is_binary;
setWantToSend();
}
PollState HttpConnection::wsWriteData() {
if (!sending_message)
return PollState::READ;
HttpTask *owner_task = dynamic_cast<HttpTask *>(owner());
if (!owner_task)
return PollState::CLOSE;
size_t sent = owner_task->sendWsData(this);
if (sent > bytes_to_send)
return PollState::CLOSE;
bytes_to_send -= sent;
if (bytes_to_send)
return PollState::READ_WRITE;
sending_message = false;
return PollState::READ;
}
void HttpConnection::setOwner(Task *) {
}

112
src/http/httpconnection.h Normal file
View file

@ -0,0 +1,112 @@
// Copyright (c) 2018 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
// This class implements HTTP functionality common to server and client,
// mainly the websocket protocol.
#pragma once
#include "../framework/socketconnection.h"
class HttpConnection : public SocketConnection {
public:
// 10 for HTTP/1.0, 11 for 1.1, 20 for 2.0 etc.
unsigned int httpVersion() {
return http_version;
}
void sendWsMessage(const std::string &msg);
void sendWsBinary(const char *buf, size_t len);
// Initiate sending a very large message. The owner task's sendWsData will
// be called repetedly until all data has been sent.
void startWsBinStream(size_t len) {
startWsStream(len, true);
}
void startWsTxtStream(size_t len) {
startWsStream(len, false);
}
void abortWsStream() {
closeMe();
}
void sendWsClose(uint16_t code, std::string msg);
void setOwner(Task *new_owner) override;
// By default, incoming messages will be buffered and not delivered (through
// wsBinMessage or wsTextMessage) until the complete message has arrived.
// Call this from within the owner task's wsBinHeader or wsTextHeader
// callback to get the response "streamed", i.e. the response will be
// delivered in "chunks" to wsBinData/wsTextData as they arrive. Note that
// the data may be masked, and you will have to unmask it yourself.
void streamWsResponse() {
stream_incoming = true;
}
// Returns pointer to 4 byte incoming mask,
// or nullptr if the response isn't masked.
const unsigned char *responseMask() const {
if (incoming_is_masked)
return incoming_mask;
else
return nullptr;
}
size_t wsIncomingBytesLeft() const {
return bytes_to_receive;
}
size_t wsBytesReceived() const {
return tot_to_receive-bytes_to_receive;
}
size_t wsOutgoingBytesLeft() const {
return bytes_to_send;
}
size_t wsBytesSent() const {
return tot_to_send-bytes_to_send;
}
bool isWebsocket() const {
return is_websocket;
}
protected:
HttpConnection(const std::string &label, Task *owner,
const std::string &hostname, uint16_t port,
uint16_t iptype = 0, struct addrinfo *local_addr = nullptr) :
SocketConnection(label, owner, hostname, port, iptype, local_addr) {
}
HttpConnection(const std::string &label, Task *owner, int fd,
const char *ip, uint16_t port) :
SocketConnection(label, owner, fd, ip, port) {
}
void set_http_version(unsigned int major, unsigned int minor) {
http_version = 10U*major + minor;
}
void send_ws_handshake(const std::string &key);
void send_ws_bin_header(size_t len);
void send_ws_txt_header(size_t len);
void send_ws_pong();
PollState incoming_ws_data(const char *buf, size_t len);
PollState incoming_ws_header(const char *buf, size_t len);
PollState wsReadData(const char *buf, size_t len);
PollState wsWriteData();
// Incoming buffer
std::string buffer;
// Everything below is websocket stuff:
bool is_websocket = false;
private:
void startWsStream(size_t len, bool is_binary = true);
// For fragmented messages:
unsigned char current_opcode = 0;
bool receiving_message = false;
bool sending_message = false;
bool output_is_binary = false;
bool stream_incoming = false;
bool incoming_is_masked = false;
bool incoming_is_binary = false;
size_t bytes_to_send, tot_to_send;
size_t bytes_to_receive, tot_to_receive;
unsigned char incoming_mask[4];
unsigned int http_version = 11; // 11 for 1.1, 20 for 2.0 etc.
};

47
src/http/httphost.h Normal file
View file

@ -0,0 +1,47 @@
// Copyright (c) 2017 The Swedish Internet Foundation
// Written by Göran Andersson <goran@init.se>
#pragma once
#include <string>
#include <cstdint>
class CookieManager;
/// \brief
/// The host name and port number of a HTTP host.
///
/// May also contain a pointer to a cookie jar.
/// The same CookieManager should
/// be used for all objects with the same hostname.
class HttpHost {
public:
/// You create and own the cookie manager.
HttpHost(const std::string &hName = std::string(),
uint16_t sPort = 80,
const std::string &pHost = std::string(),
uint16_t pPort = 0,
CookieManager *cMgr = nullptr) :
hostname(hName), proxyHost(pHost),
port(sPort), proxyPort(pPort), cmgr(cMgr) {
#ifdef USE_GNUTLS
is_tls = (sPort == 443);
#endif
}
HttpHost(const char *hName, uint16_t sPort = 80) :
hostname(hName), port(sPort) {
#ifdef USE_GNUTLS
is_tls = (sPort == 443);
#endif
cmgr = nullptr;
}
std::string hostname;
std::string proxyHost;
uint16_t port;
uint16_t proxyPort;
CookieManager *cmgr;
uint16_t iptype = 0; // 4 for ipv4, 6 for ipv6, 0 for any.
#ifdef USE_GNUTLS
bool is_tls;
#endif
};

View file

@ -0,0 +1,220 @@
#include "httprequestengine.h"
void HttpRequestEngine::postJob(Task *task, const std::string &event_name,
const std::string &url,
const std::string &data) {
auto job = new HREJob({ task, nullptr, event_name, url, data });
incoming_jobs.push_back(job);
dbg_log() << "JOB " << url << " Q=" << incoming_jobs.size()
<< " I=" << idle_connections.size()
<< " C=" << no_connections;
checkQueue();
}
void HttpRequestEngine::checkQueue() {
if (incoming_jobs.empty())
return;
while (!idle_connections.empty()) {
auto p = idle_connections.begin();
SocketConnection *s = *p;
idle_connections.erase(p);
if (wakeUpConnection(s))
return;
}
dbg_log() << "No idle connection available, have " << no_connections
<< " max=" << max_connections;
if (no_connections < max_connections) {
// Increase number of simultaneous connections
++no_connections;
checkConnectionCount();
}
// We'll have to wait for an existing connection to become available.
}
double HttpRequestEngine::start() {
log() << "start()";
if (incoming_jobs.size() > min_connections) {
if (incoming_jobs.size() > max_connections)
no_connections = max_connections;
else
no_connections = static_cast<unsigned int>(incoming_jobs.size());
}
checkConnectionCount();
return tick;
}
double HttpRequestEngine::timerEvent() {
if (no_connections > min_connections) {
unsigned int n = static_cast<unsigned int>(active_jobs.size() +
incoming_jobs.size());
// Will only need n simultaneous connections
if (n < no_connections)
no_connections = std::max(n, min_connections);
}
// There may be failed jobs waiting to be restarted:
checkQueue();
return tick;
}
void HttpRequestEngine::newRequest(HttpClientConnection *conn) {
dbg_log() << "Ready for a new request, queue size=" << incoming_jobs.size();
auto p = active_jobs.find(conn);
if (p != active_jobs.end()) {
HREJob *job = p->second;
if (job->data.empty())
conn->get(job->url);
else
conn->post(job->url, job->data);
return;
}
while (!incoming_jobs.empty()) {
HREJob *job = incoming_jobs.front();
incoming_jobs.pop_front();
if (!job)
continue;
log() << "Job " << job->event_name << ": " << job->url;
job->connection = conn;
if (job->data.empty())
conn->get(job->url);
else
conn->post(job->url, job->data);
active_jobs[conn] = job;
return;
}
//dbg_log() << "nothing to do";
idle_connections.insert(conn);
conn->pass();
}
bool HttpRequestEngine::requestComplete(HttpClientConnection *conn) {
dbg_log() << "HttpRequestEngine::requestComplete " << conn->httpStatus()
<< " --> " << conn->contents();
{
auto p = active_jobs.find(conn);
if (p == active_jobs.end())
return true;
last_completed = conn;
current_job = p->second;
}
HREJob *job = current_job;
redo_job = nullptr;
if (!current_job->event_name.empty())
executeHandler(current_job->task, current_job->event_name);
if (redo_job == job) {
// User has asked for a new request, let conn stay in active_jobs.
redo_job = nullptr;
} else {
delete job;
active_jobs.erase(conn);
}
last_completed = nullptr;
return true;
}
void HttpRequestEngine::taskFinished(Task *task) {
dbg_log() << "Client task " << task->label() << " died.";
cancelRequestsByTask(task);
}
void HttpRequestEngine::cancelRequestsByTask(Task *task) {
// Cancel active jobs
dbg_log() << "Cancel active jobs";
std::set<HttpClientConnection *> to_remove;
for (auto p : active_jobs)
if (p.second->task == task) {
// Clear event_name to stop handler from being executed:
p.second->event_name.clear();
to_remove.insert(p.first);
}
for (auto &conn : to_remove)
cancelConnection(conn);
dbg_log() << "Cancel pending jobs";
// Cancel pending jobs
for (auto &job : incoming_jobs) {
if (job && job->task == task) {
delete job;
job = nullptr;
}
}
}
void HttpRequestEngine::connAdded(SocketConnection *c) {
++active_connections;
dbg_log() << "conn added: " << c << " now have " << active_connections;
}
void HttpRequestEngine::connRemoved(SocketConnection *c) {
--active_connections;
dbg_log() << "conn removed: " << c << " now have " << active_connections;
if (idle_connections.erase(c)) {
// An idle connection was closed, perhaps due to keep-alive timeout.
if (no_connections > min_connections)
--no_connections; // Don't restart the connection now.
return;
}
HttpClientConnection *conn = dynamic_cast<HttpClientConnection *>(c);
auto p = active_jobs.find(conn);
if (p == active_jobs.end())
return;
HREJob *job = p->second;
active_jobs.erase(p);
if (job->event_name.empty())
return;
// Notify user that request failed
last_completed = nullptr;
redo_job = nullptr;
current_job = job;
executeHandler(current_job->task, current_job->event_name);
if (redo_job == job) {
redo_job = nullptr;
// Probably better to wait a little while before restarting.
// Put it last in queue, and don't do checkQueue(); now.
incoming_jobs.push_back(job);
} else {
delete current_job;
}
}
void HttpRequestEngine::checkConnectionCount() {
if (!hasStarted() || terminated()) {
dbg_log() << "FAIL! HttpRequestEngine not active!";
return;
}
dbg_log() << "checkConnectionCount: act=" << active_connections
<< " want=" << no_connections
<< " max=" << max_connections
<< " Q=" << incoming_jobs.size()
<< " I=" << idle_connections.size();
for (unsigned int i = active_connections; i < no_connections; ++i) {
if (!createNewConnection())
log() << "couldn't add connection";
}
if (no_connections && !active_connections)
connectionLost();
}

View file

@ -0,0 +1,137 @@
// Copyright (c) 2018 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
#pragma once
#include <queue>
#include <vector>
#include <map>
#include <set>
#include <functional>
#include "httpclienttask.h"
struct HttpRequestEngineEvent {
const std::string &url;
unsigned int http_status;
const std::string &response;
};
class HttpRequestEngine : public HttpClientTask {
public:
HttpRequestEngine(const std::string &name, const HttpHost &host,
unsigned int min_conn = 0, unsigned int max_conn = 10,
double tick_duration = 0.5) :
HttpClientTask(name, host),
min_connections(std::min(min_conn, max_conn)),
max_connections(max_conn),
no_connections(min_connections),
active_connections(0),
tick(tick_duration > 0.0 ? tick_duration : 0.5) {
}
// Will call task's handleExecution when done, if task has told me to
// observe it. I.e. task must have done engine->startObserving(this);
// Also, if event_name is an empty string, handleExecution won't be called.
void getJob(Task *task, const std::string &event_name,
const std::string &url) {
postJob(task, event_name, url, "");
}
void postJob(Task *task, const std::string &event_name,
const std::string &url, const std::string &data);
// Return HTTP status of last completed request, 0 for failure.
// Should only be called in the handleExecution callback.
unsigned int httpStatus() const {
return last_completed ? last_completed->httpStatus() : 0;
}
// Return MIME type of last completed request, empty on failure.
// Should only be called in the handleExecution callback.
std::string contentType() const {
return last_completed ? last_completed->contentType() : "";
}
// Return the payload of the last completed request, empty on failure.
// Should only be called in the handleExecution callback.
const std::string contents() const {
return last_completed ? last_completed->contents() : "";
}
// Return the URL of the last completed request, empty on failure.
// Should only be called in the handleExecution callback.
const std::string &currentUrl() const {
static std::string dummy;
return current_job ? current_job->url : dummy;
}
// Perform the same request again.
// Should only be called in the handleExecution callback.
void redoJob() {
redo_job = current_job;
}
// Perform another GET request, but with updated url.
void redoJob(const std::string &url) {
if (current_job) {
current_job->url = url;
current_job->data.clear();
}
}
// Perform another POST request, but with updated url/data.
void redoJob(const std::string &url, const std::string &data) {
if (current_job) {
current_job->url = url;
current_job->data = data;
}
}
// If a task terminates, release ongoing jobs
void taskFinished(Task *t) override;
// Will be called if all connections have failed.
// Default is to try again after next tick.
virtual void connectionLost() {
dbg_log() << "connectionLost() not implemented";
}
private:
struct HREJob {
Task *task;
HttpClientConnection *connection;
std::string event_name, url, data;
};
double start() final;
double timerEvent() final;
// Cancel all request owned by task. Handlers won't be executed.
void cancelRequestsByTask(Task *task);
void newRequest(HttpClientConnection *conn) final;
bool requestComplete(HttpClientConnection *conn) final;
void connAdded(SocketConnection *) final;
void connRemoved(SocketConnection *) final;
void checkConnectionCount();
void checkQueue();
unsigned int min_connections, max_connections;
unsigned int no_connections, active_connections;
double tick;
HttpClientConnection *last_completed = nullptr;
HREJob *current_job;
HREJob *redo_job;
//size_t max_response_size = 10000000;
// Jobs waiting to be started
std::deque<HREJob *> incoming_jobs;
std::set<SocketConnection *> idle_connections;
std::map<HttpClientConnection *, HREJob *> active_jobs;
};

View file

@ -0,0 +1,468 @@
// Copyright (c) 2018 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
#ifdef _WIN32
#define NOMINMAX
#include <ws2tcpip.h>
#else
#include <arpa/inet.h>
#endif
#include <cstdlib>
#include <cstdio>
#include <cstring>
#include <cerrno>
#include <fcntl.h>
#ifdef _WIN32
#include <time.h>
#include <windows.h>
#include <winsock2.h>
#pragma comment(lib, "ws2_32.lib")
#else
#include <unistd.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <netdb.h>
#endif
#include <sys/stat.h>
#include <sstream>
#include <algorithm>
#include "sha1.h"
#include "httpserverconnection.h"
#include "http_common.h"
#include "webservertask.h"
HttpServerConnection::
HttpServerConnection(const std::string &label, WebServerTask *task,
int fd, const char *ip, uint16_t port) :
HttpConnection(label, task, fd, ip, port),
owner_task(task) {
}
void HttpServerConnection::setOwner(Task *new_owner) {
owner_task = dynamic_cast<WebServerTask *>(new_owner);
if (!owner_task)
throw std::logic_error("expected WebServerTask");
Socket::setOwner(new_owner);
}
PollState HttpServerConnection::got_post_data(const char *buf, size_t len) {
//log() << "got_post_data " << len << " bytes, " << remaining_post_data
// << " left\n" << std::string(buf, len);
size_t length = std::min(remaining_post_data, len);
remaining_post_data -= length;
if (remaining_post_data) {
if (!buffer_post_data)
if (!owner_task->partialPostData(this, buf, length))
return PollState::CLOSE;
} else {
if (buffer_post_data)
state = owner_task->lastPostData(this, buffer.c_str(),
buffer.size());
else
state = owner_task->lastPostData(this, buf, len);
if (len > length) {
err_log() << "too much post data sent";
return PollState::CLOSE;
}
if (state == HttpState::SENDING_RESPONSE)
return PollState::WRITE;
else if (state == HttpState::CLOSE)
return PollState::CLOSE;
}
return PollState::READ;
}
#ifdef USE_WEBROOT
bool HttpServerConnection::prepareFileResponse(const std::string &url) {
if (url.find("../") != std::string::npos)
return false;
std::string webroot = owner_task->webRoot();
if (webroot.empty())
return false;
static_file_fd = open( (webroot + url).c_str(), O_RDONLY );
if (static_file_fd < 0)
return false;
struct stat sbuf;
if (fstat(static_file_fd, &sbuf) < 0 || !S_ISREG(sbuf.st_mode)) {
close(static_file_fd);
return false;
}
remaining_bytes = static_cast<size_t>(sbuf.st_size);
sending_static_file = true;
return true;
}
HttpState HttpServerConnection::sendHttpFileResponse(const std::string &hdrs,
const std::string &mime) {
if (!sending_static_file || state != HttpState::WAITING_FOR_REQUEST) {
err_log() << "internal error, prepareFileResponse not called";
return HttpState::CLOSE;
}
sendHttpResponseHeader(hdrs, mime, remaining_bytes);
if (remaining_bytes)
remaining_bytes -= sendFileData(static_file_fd, remaining_bytes);
if (remaining_bytes)
return HttpState::SENDING_RESPONSE;
else
return HttpState::WAITING_FOR_REQUEST;
}
#endif
PollState HttpServerConnection::writeData() {
if (is_websocket)
return wsWriteData();
if (state == HttpState::SENDING_RESPONSE) {
// TODO: don't give conn to the owner, too much that can go wrong.
size_t sent;
#ifdef USE_WEBROOT
if (sending_static_file)
sent = sendFileData(static_file_fd, remaining_bytes);
else
#endif
sent = owner_task->sendResponseData(this, remaining_bytes);
remaining_bytes -= sent;
if (remaining_bytes)
return PollState::WRITE;
#ifdef USE_WEBROOT
if (sending_static_file) {
close(static_file_fd);
sending_static_file = false;
}
#endif
state = HttpState::WAITING_FOR_REQUEST;
return PollState::READ;
}
err_log() << "HttpServerConnection::writeData() called";
return PollState::CLOSE;
}
PollState HttpServerConnection::readData(char *buf, size_t len) {
if (is_websocket)
return wsReadData(buf, len);
if (state == HttpState::READING_POST_DATA && !buffer_post_data)
return got_post_data(buf, len);
buffer.append(buf, len);
return check_buffer();
}
PollState HttpServerConnection::check_buffer() {
if (state == HttpState::WAITING_FOR_REQUEST) {
size_t end_of_request = buffer.find("\r\n\r\n");
if (end_of_request == std::string::npos) {
if (buffer.size() > 20000) {
log() << "error: no end of request";
return PollState::CLOSE;
}
return PollState::READ;
}
//log() << "Got request: " << buffer;
current_request = buffer.substr(0, end_of_request+strlen("\r\n\r\n"));
buffer.erase(0, current_request.size());
if (!parse_request()) {
err_log() << "bad request";
return PollState::CLOSE;
}
if (current_method == "GET") {
auto p = http_request_headers.find("sec-websocket-key");
if (p == http_request_headers.end()) {
state = owner_task->newGetRequest(this, current_uri);
} else {
if (!owner_task->newWsRequest(this, current_uri))
return PollState::CLOSE;
state = HttpState::WEBSOCKET;
is_websocket = true;
send_ws_handshake(p->second);
if (notify_after_ws_handshake)
owner_task->wsHandshakeFinished(this, current_uri);
}
if (buffer.size()) {
log() << "unexpected data after GET request";
return PollState::CLOSE;
}
} else if (current_method == "POST") {
try {
auto p = http_request_headers.find("content-length");
if (p == http_request_headers.end()) {
err_log() << "cannot find Content-Length";
// Just assume a very large post:
remaining_post_data = std::string::npos;
} else {
remaining_post_data = std::stoul(p->second);
}
} catch (std::exception &) {
err_log() << "cannot parse Content-Length";
return PollState::CLOSE;
}
buffer_post_data = false;
state = owner_task->newPostRequest(this, current_uri);
} else if (current_method == "OPTIONS") {
state = owner_task->preflightRequest(this, current_uri);
} else {
return PollState::CLOSE;
}
}
if (state == HttpState::CLOSE)
return PollState::CLOSE;
if (state == HttpState::SENDING_RESPONSE)
return PollState::WRITE;
if (buffer.empty())
return PollState::READ;
if (state == HttpState::READING_POST_DATA) {
PollState ret = got_post_data(buffer.c_str(), buffer.size());
if (!buffer_post_data || !remaining_post_data)
buffer.clear();
return ret;
}
return check_buffer();
}
namespace {
inline int hexval(std::string::value_type c) {
if (c >= '0' && c <= '9')
return c-'0';
else if (c >= 'A' && c <= 'F')
return (c - 'A' + 10);
else if (c >= 'a' && c <= 'f')
return (c - 'a' + 10);
else
return -1000;
}
}
void HttpServerConnection::get_all_parameters(const char *qpos,
std::multimap<std::string,
std::string> &pars) {
const char *ppos = qpos; // Start of parameter name
while (*qpos) {
if (*qpos == '&') {
// Epmty value:
pars.insert(std::make_pair(std::string(ppos, qpos), std::string()));
++qpos;
ppos = qpos;
} else if (*qpos == '=') {
// Urldecode value:
std::string par_name = std::string(ppos, qpos);
++qpos; // Start of value
std::ostringstream res;
while (char c = *qpos) {
if (c == '&') {
++qpos;
break;
} else if (c == '+') {
++qpos;
c = ' ';
} else if (c == '%') {
// Should be two hex digits after %,
// but if not, ignore errors and just keep the %
if (*++qpos) {
char c1 = *qpos, c2;
if (*++qpos) {
c2 = *qpos;
if (c2) {
++qpos;
int n = hexval(c1)*16+hexval(c2);
if (n >= 0)
c = static_cast<char>(n);
}
}
}
} else {
++qpos;
}
res << c;
}
pars.insert(std::make_pair(par_name, res.str()));
ppos = qpos;
} else {
++qpos;
}
}
}
bool HttpServerConnection::parse_request() {
std::string first_line;
http_request_headers.clear();
if (!HttpCommon::parseHeaders(current_request, first_line,
http_request_headers))
return false;
size_t eom = first_line.find(' ');
if (eom == std::string::npos)
return false;
size_t eor = first_line.find(' ', eom+1);
if (eor == std::string::npos)
return false;
current_method = first_line.substr(0, eom);
current_uri = first_line.substr(eom+1, eor-eom-1);
{
// HTTP/1.1
unsigned int major, minor;
int pos = sscanf(first_line.substr(eor+1).c_str(), "HTTP/%u.%u",
&major, &minor);
if (pos != 2)
return false;
set_http_version(major, minor);
}
current_query_pars.clear();
size_t qpos = current_uri.find("?");
if (qpos != std::string::npos) {
current_query_string = current_uri.substr(qpos+1);
current_uri.resize(qpos);
get_all_parameters(current_query_string.c_str(), current_query_pars);
} else {
current_query_string.clear();
}
// uri might be given as http://some.domain/blah
if (current_uri.substr(0, 4) == "http") {
if (current_uri.substr(0, 7) == "http://")
qpos = current_uri.find('/', 7);
else if (current_uri.substr(0, 8) == "https://")
qpos = current_uri.find('/', 8);
else
return true;
if (qpos == std::string::npos)
current_uri = "/";
else
current_uri.erase(0, qpos);
}
return true;
}
size_t HttpServerConnection::
sendHttpResponse(const std::string &headers,
const std::string &mime,
const std::string &contents) {
std::ostringstream response;
response << headers << "Content-Length: " << contents.size()
<< "\r\nContent-Type: " << mime << "\r\n\r\n" << contents;
size_t len = response.str().size();
asyncSendData(response.str());
return len;
}
size_t HttpServerConnection::
sendHttpResponseHeader(const std::string &headers,
const std::string &mime,
size_t content_length) {
std::ostringstream response;
response << headers << "Content-Length: " << content_length
<< "\r\nContent-Type: " << mime << "\r\n\r\n";
size_t len = response.str().size();
asyncSendData(response.str());
remaining_bytes = content_length;
return len;
}
size_t HttpServerConnection::
sendChunkedResponseHeader(const std::string &headers,
const std::string &mime) {
sending_chunked_response = true;
std::ostringstream response;
response << headers << "Transfer-Encoding: chunked"
<< "\r\nContent-Type: " << mime << "\r\n\r\n";
size_t len = response.str().size();
asyncSendData(response.str());
return len;
}
size_t HttpServerConnection::sendChunk(const std::string &content) {
if (!sending_chunked_response) {
err_log("response encoding is not chunked");
return 0;
}
std::ostringstream chunk;
chunk << std::hex << content.size() << "\r\n" << content << "\r\n";
asyncSendData(chunk.str());
return chunk.str().size();
}
size_t HttpServerConnection::chunkedResponseComplete() {
if (!sending_chunked_response) {
err_log("response encoding is not chunked");
return 0;
}
std::string last_chunk = "0\r\n\r\n";
sending_chunked_response = false;
asyncSendData(last_chunk);
return last_chunk.size();
}
bool HttpServerConnection::hasQueryPar(const std::string &name) const {
return (current_query_pars.find(name) != current_query_pars.end());
}
std::string HttpServerConnection::getQueryVal(const std::string &name) const {
auto p = current_query_pars.find(name);
if (p != current_query_pars.end())
return p->second;
return std::string();
}
std::string HttpServerConnection::getHeaderVal(const std::string &name) const {
auto p = http_request_headers.find(name);
if (p != http_request_headers.end())
return p->second;
return std::string();
}
std::map<std::string, std::string>
HttpServerConnection::currentRequestCookies() const {
std::map<std::string, std::string> c;
auto r = getHeaderVals("cookie");
for (auto p=r.first; p!=r.second; ++p) {
std::string s = p->second;
while (!s.empty()) {
auto pos1 = s.find('=');
if (pos1 == std::string::npos)
break;
auto pos2 = s.find(';');
c[s.substr(0, pos1)] = s.substr(pos1+1, pos2-pos1-1);
if (pos2 == std::string::npos)
break;
s.erase(0, s.find_first_not_of(" \t\r", pos2+1));
}
}
return c;
}
std::string HttpServerConnection::cookieVal(const std::string &name) const {
if (name.find_first_of(" =;,\t") != std::string::npos)
return "";
auto r = getHeaderVals("cookie");
for (auto p=r.first; p!=r.second; ++p) {
auto pos = (" " + p->second).find(" " + name + "=");
if (pos != std::string::npos) {
auto first = p->second.find('=', pos);
auto last = p->second.find(';', pos);
if (last > first) {
return p->second.substr(first+1, last-first-1);
}
}
}
return "";
}
HttpServerConnection::~HttpServerConnection() {
dbg_log() << "client connection closed";
#ifdef USE_WEBROOT
if (sending_static_file)
close(static_file_fd);
#endif
}

View file

@ -0,0 +1,189 @@
// Copyright (c) 2018 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
#pragma once
#include <map>
#include "httpconnection.h"
class WebServerTask;
enum class HttpState {
CLOSE,
WAITING_FOR_REQUEST, SENDING_RESPONSE,
READING_POST_DATA, WEBSOCKET
};
/// \brief
/// HTTP/1.1 server protocol.
///
/// This class implements a small subset of the HTTP/1.1 server protocol.
/// When a new HTTP GET or POST request is made, the owner task's method
/// newGetRequest or newPostRequest will be called. If a websocket connection
/// is made, the owner task's method newWsRequest will be called.
///
/// To use this class, you must derive from WebServerTask and implement
/// at least one of newGetRequest, newPostRequest, or newWsRequest.
class HttpServerConnection : public HttpConnection {
public:
HttpServerConnection(const std::string &label, WebServerTask *task, int fd,
const char *ip, uint16_t port);
virtual ~HttpServerConnection() override;
PollState connected() final {
return PollState::READ;
}
PollState readData(char *buf, size_t len) final;
//PollState unexpectedData(char *, size_t ) override;
PollState writeData() override;
/// Return true if query string in current request has parameter name:
bool hasQueryPar(const std::string &name) const;
void eraseQueryPar(const std::string &name) {
current_query_pars.erase(name);
}
/// If query string in current request has parameter "name",
/// return the value of its first occurrence.
/// Otherwise return empty string.
std::string getQueryVal(const std::string &name) const;
const std::multimap<std::string, std::string> &currentQueryPars() const {
return current_query_pars;
}
std::map<std::string, std::string> currentRequestCookies() const;
/// Return value of cookie if it exists, otherwise empty string.
std::string cookieVal(const std::string &name) const;
/// If HTTP headers in current request has attribute "name",
/// return the value of its first occurrence.
/// Otherwise return empty string.
/// Note: use only lower-case letters in "name", e.g. "content-length".
std::string getHeaderVal(const std::string &name) const;
typedef std::multimap<std::string, std::string>::const_iterator Hit;
std::pair<Hit, Hit> getHeaderVals(const std::string &name) const {
return http_request_headers.equal_range(name);
}
/// Async send response. Return length of what's to be sent.
size_t sendHttpResponse(const std::string &headers,
const std::string &mime,
const std::string &contents);
/// Async send response headers. Return length of what's to be sent:
size_t sendHttpResponseHeader(const std::string &headers,
const std::string &mime,
size_t content_length);
/// Async send response headers with the "chunked" encoding.
/// Return length of what's to be sent.
size_t sendChunkedResponseHeader(const std::string &headers,
const std::string &mime);
/// Send chunk. May only be used if we sent chunked headers.
size_t sendChunk(const std::string &content);
/// To notify that all chunks have been sent.
size_t chunkedResponseComplete();
bool sendingChunkedResponse() const {
return sending_chunked_response;
}
#ifdef USE_WEBROOT
/// Can only be called in the newGetRequest method.
/// Return true if url is a file that can be sent; then you should call the
/// sendHttpFileResponse method; otherwise you'd want to send a 404 reply.
///
/// *Warning!* Call might be blocking!
bool prepareFileResponse(const std::string &url);
HttpState sendHttpFileResponse(const std::string &headers,
const std::string &mime);
#endif
const std::string &currentUri() const {
return current_uri;
}
const std::string currentFullUrl() const {
if (current_query_string.empty())
return current_uri;
return current_uri + '?' + current_query_string;
}
const std::string &currentQueryString() const {
return current_query_string;
}
/// Number of bytes left of the current HTTP POST.
size_t remainingPostData() const {
return remaining_post_data;
}
/// The owner task may call this in the newPostRequest handler and return
/// HttpState::READING_POST_DATA.
/// In that case, the owner task will not get partialPostData calls,
/// but only a lastPostData call when the user has posted all data.
///
/// Hint: Check first that remainingPostData() isn't too large.
void bufferPostData() {
buffer_post_data = true;
}
/// Call this if the owner is to be notified after handshake:
void notifyWsHandshake() {
notify_after_ws_handshake = true;
}
/// Override this to make sure we're not transfered to an owner Task that
/// is not a WebServerTask (or subclass).
void setOwner(Task *new_owner) override;
/// Current request (first line and headers) in raw form.
std::string currentRequest() const {
return current_request;
}
private:
// A weak pointer to the owner task. Dangerous. However, the EventLoop will
// kill all connections (i.e. us) before killing the owner task.
WebServerTask *owner_task;
HttpState state = HttpState::WAITING_FOR_REQUEST;
std::string current_request;
std::multimap<std::string, std::string> http_request_headers;
std::string current_method, current_uri;
std::string current_query_string;
std::multimap<std::string, std::string> current_query_pars;
static void get_all_parameters(const char *qpos,
std::multimap<std::string, std::string> &pars);
// Will be set when sending a response with known Content-Length:
size_t remaining_bytes;
#ifdef USE_WEBROOT
// Will be set if sending_static_file is true:
int static_file_fd;
// Will be true when sending a static file:
bool sending_static_file = false;
#endif
// Will be true when sending chunked response:
bool sending_chunked_response = false;
// If set to true, the owner will be notified after handshake:
bool notify_after_ws_handshake = false;
// Will be set in state READING_POST_DATA:
bool buffer_post_data;
size_t remaining_post_data;
bool parse_request();
PollState got_post_data(const char *buf, size_t len);
PollState check_buffer();
};

22
src/http/httptask.cpp Normal file
View file

@ -0,0 +1,22 @@
// Copyright (c) 2018 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
#include "httptask.h"
#include "httpconnection.h"
// New websocket text message
bool HttpTask::wsTextMessage(HttpConnection *,
const std::string &msg) {
log() << "ignoring websocket text message: " << msg;
return false;
}
bool HttpTask::wsBinMessage(HttpConnection *,
const std::string &msg) {
log() << "ignoring websocket bin message, length " << msg.size();
return false;
}
size_t HttpTask::sendWsData(HttpConnection *conn) {
conn->abortWsStream();
return 0;
}

75
src/http/httptask.h Normal file
View file

@ -0,0 +1,75 @@
// Copyright (c) 2018 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
#pragma once
#include "../framework/task.h"
#include "httphost.h"
class HttpConnection;
/// Common API for HTTP server and client tasks.
class HttpTask : public Task {
public:
HttpTask(const std::string &name) :
Task(name) {
}
/// Incoming websocket text message. Return false to kill connection.
virtual bool wsTextMessage(HttpConnection *,
const std::string &msg);
/// Incoming websocket binary message. Return false to kill connection.
virtual bool wsBinMessage(HttpConnection *,
const std::string &msg);
/// \brief
/// Called when headers of a binary message are read.
///
/// Return false to kill connection. Unless
/// you call streamWsResponse, the message will be buffered until complete,
/// and then delivered throughwsBinMessage.
/// If you call conn->streamWsResponse, the message will be streamed
/// through wsBinData.
virtual bool wsBinHeader(HttpConnection *, size_t ) {
return true;
}
/// \brief
/// Called when headers of a text message are read.
///
/// Return false to kill connection. Unless
/// you call streamWsResponse, the message will be buffered until complete,
/// and then delivered through wsTextMessage.
/// If you call conn->streamWsResponse, the message will be streamed
/// through wsBinData / wsTextData.
virtual bool wsTextHeader(HttpConnection *, size_t ) {
return true;
}
/// \brief
/// Incoming partial websocket binary message.
///
/// Return false to kill connection.
virtual bool wsBinData(HttpConnection *, const char *, size_t ) {
return false;
}
/// \brief
/// Incoming partial websocket text message.
///
/// Return false to kill connection.
virtual bool wsTextData(HttpConnection *, const char *, size_t ) {
return false;
}
/// If you have called conn->startWsBinStream, sendWsData will be called
/// repetedly (as fast as the network allows) until all data has been sent.
/// Override this to return the number of bytes you sent. You may return 0
/// if you have a temporary error, but it's really bad to keep returning 0.
/// So don't use this feature unless the data to send is readily available.
/// To give up, call conn->abortWsStream() and return 0.
virtual size_t sendWsData(HttpConnection *conn);
protected:
private:
};

25
src/http/mk.inc Normal file
View file

@ -0,0 +1,25 @@
ifeq ($(WEBROOT),1)
CXXFLAGS += -DUSE_WEBROOT
endif
SOURCES += \
$(DIRLEVEL)/http/sha1.cpp \
$(DIRLEVEL)/http/httptask.cpp \
$(DIRLEVEL)/http/httpclienttask.cpp \
$(DIRLEVEL)/http/singlerequest.cpp \
$(DIRLEVEL)/http/webservertask.cpp \
$(DIRLEVEL)/http/http_common.cpp \
$(DIRLEVEL)/http/httpconnection.cpp \
$(DIRLEVEL)/http/httpclientconnection.cpp \
$(DIRLEVEL)/http/cookiemanager.cpp \
$(DIRLEVEL)/http/httpserverconnection.cpp \
$(DIRLEVEL)/http/httprequestengine.cpp \
$(DIRLEVEL)/http/websocketbridge.cpp
OPT_SOURCES += \
$(DIRLEVEL)/http/cookiefile.cpp
# Note: we don't add $(DIRLEVEL)/http/cookiefile.cpp by default
# since it uses the file system, which not all clients may want.
include $(DIRLEVEL)/framework/mk.inc

306
src/http/sha1.cpp Normal file
View file

@ -0,0 +1,306 @@
// Copyright (c) 2018 The Swedish Internet Foundation
// Based on public domain code.
// Modified by Göran Andersson <initgoran@gmail.com>
#include <sstream>
#include <iomanip>
#include <fstream>
#include <string.h>
#include "sha1.h"
static char base64(unsigned int value_in) {
static const char* encoding = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
if (value_in > 63)
return '=';
return encoding[static_cast<int>(value_in)];
}
// encode len bytes starting at src, write to dst.
// number of bytes written will be 4 times the smallest integer >= len/3.
void base64_encode(const unsigned char *src, size_t len, char *destination) {
char *p = destination;
while (len >= 3) {
len -= 3;
*p++ = base64(src[0] >> 2);
*p++ = base64(static_cast<unsigned char>((src[0] & 0x3) << 4) + (src[1] >> 4));
*p++ = base64(static_cast<unsigned char>((src[1] & 0xf) << 2) + (src[2] >> 6));
*p++ = base64(src[2] & 0x3f);
src += 3;
}
switch (len) {
case 2:
*p++ = base64(src[0] >> 2);
*p++ = base64(static_cast<unsigned char>((src[0] & 0x3) << 4) + (src[1] >> 4));
*p++ = base64(static_cast<unsigned char>((src[1] & 0xf) << 2));
break;
case 1:
*p++ = base64(src[0] >> 2);
*p++ = base64(static_cast<unsigned char>((src[0] & 0x3) << 4));
break;
}
switch ((p-destination)%4) {
case 3:
*p++ = '=';
#ifdef __clang__
[[clang::fallthrough]];
#elif defined __GNUC__
#if __GNUC__ > 6
[[gnu::fallthrough]];
#endif
// -Wimplicit-fallthrough=0
#endif
case 2:
*p++ = '=';
#ifdef __clang__
[[clang::fallthrough]];
#elif defined __GNUC__
#if __GNUC__ > 6
[[gnu::fallthrough]];
#endif
// -Wimplicit-fallthrough=0
#endif
case 1:
*p++ = '=';
break;
}
}
namespace {
int _one = 1;
}
bool SHA1::is_big_endian = (*(reinterpret_cast<int8_t *>(&_one)) == 0);
/* Help macros */
#define SHA1_ROL(value, bits) (((value) << (bits)) | (((value) & 0xffffffff) >> (32 - (bits))))
#define SHA1_BLK(i) (block[i&15] = SHA1_ROL(block[(i+13)&15] ^ block[(i+8)&15] ^ block[(i+2)&15] ^ block[i&15],1))
/* (R0+R1), R2, R3, R4 are the different operations used in SHA1 */
#define SHA1_R0(v,w,x,y,z,i) z += ((w&(x^y))^y) + block[i] + 0x5a827999 + SHA1_ROL(v,5); w=SHA1_ROL(w,30)
#define SHA1_R1(v,w,x,y,z,i) z += ((w&(x^y))^y) + SHA1_BLK(i) + 0x5a827999 + SHA1_ROL(v,5); w=SHA1_ROL(w,30)
#define SHA1_R2(v,w,x,y,z,i) z += (w^x^y) + SHA1_BLK(i) + 0x6ed9eba1 + SHA1_ROL(v,5); w=SHA1_ROL(w,30)
#define SHA1_R3(v,w,x,y,z,i) z += (((w|x)&y)|(w&x)) + SHA1_BLK(i) + 0x8f1bbcdc + SHA1_ROL(v,5); w=SHA1_ROL(w,30)
#define SHA1_R4(v,w,x,y,z,i) z += (w^x^y) + SHA1_BLK(i) + 0xca62c1d6 + SHA1_ROL(v,5); w=SHA1_ROL(w,30)
SHA1::SHA1(char *buf) :
the_res(buf) {
}
void SHA1::update(const char *key) {
std::istringstream is(key);
/* SHA1 initialization constants */
digest[0] = 0x67452301;
digest[1] = 0xefcdab89;
digest[2] = 0x98badcfe;
digest[3] = 0x10325476;
digest[4] = 0xc3d2e1f0;
/* Reset counters */
transforms = 0;
std::string buffer;
std::string rest_of_buffer;
read(is, rest_of_buffer, static_cast<int>(BLOCK_BYTES) - static_cast<int>(buffer.size()));
buffer += rest_of_buffer;
while (is) {
uint32_t block[BLOCK_INTS];
buffer_to_block(buffer, block);
transform(block);
read(is, buffer, BLOCK_BYTES);
}
/* Total number of hashed bits */
uint64_t total_bits = (transforms*BLOCK_BYTES + buffer.size()) * 8;
/* Padding */
buffer += static_cast<char>(0x80);
unsigned int orig_size = static_cast<unsigned int>(buffer.size());
while (buffer.size() < BLOCK_BYTES) {
buffer += static_cast<char>(0x00);
}
uint32_t block[BLOCK_INTS];
buffer_to_block(buffer, block);
if (orig_size > BLOCK_BYTES - 8) {
transform(block);
for (unsigned int i = 0; i < BLOCK_INTS - 2; i++) {
block[i] = 0;
}
}
/* Append total_bits, split this uint64_t into two uint32_t */
block[BLOCK_INTS - 1] = static_cast<uint32_t>(total_bits);
block[BLOCK_INTS - 2] = (total_bits >> 32);
transform(block);
/* Base64 */
unsigned char *pp = reinterpret_cast<unsigned char *>(digest);
unsigned int pos = 0;
if (is_big_endian) {
base64_encode(pp, 20, the_res);
return;
}
// Byte order is 3, 2, 1, 0, 7, 6, 5, 4, 11, ...
the_res[pos++] = base64(pp[3] >> 2);
the_res[pos++] = base64(static_cast<unsigned char>((pp[3] & 0x3) << 4) + (pp[2] >> 4));
the_res[pos++] = base64(static_cast<unsigned char>((pp[2] & 0xf) << 2) + (pp[1] >> 6));
the_res[pos++] = base64(pp[1] & 0x3f);
the_res[pos++] = base64(pp[0] >> 2);
the_res[pos++] = base64(static_cast<unsigned char>((pp[0] & 0x3) << 4) + (pp[7] >> 4));
the_res[pos++] = base64(static_cast<unsigned char>((pp[7] & 0xf) << 2) + (pp[6] >> 6));
the_res[pos++] = base64(pp[6] & 0x3f);
the_res[pos++] = base64(pp[5] >> 2);
the_res[pos++] = base64(static_cast<unsigned char>((pp[5] & 0x3) << 4) + (pp[4] >> 4));
the_res[pos++] = base64(static_cast<unsigned char>((pp[4] & 0xf) << 2) + (pp[11] >> 6));
the_res[pos++] = base64(pp[11] & 0x3f);
the_res[pos++] = base64(pp[10] >> 2);
the_res[pos++] = base64(static_cast<unsigned char>((pp[10] & 0x3) << 4) + (pp[9] >> 4));
the_res[pos++] = base64(static_cast<unsigned char>((pp[9] & 0xf) << 2) + (pp[8] >> 6));
the_res[pos++] = base64(pp[8] & 0x3f);
the_res[pos++] = base64(pp[15] >> 2);
the_res[pos++] = base64(static_cast<unsigned char>((pp[15] & 0x3) << 4) + (pp[14] >> 4));
the_res[pos++] = base64(static_cast<unsigned char>((pp[14] & 0xf) << 2) + (pp[13] >> 6));
the_res[pos++] = base64(pp[13] & 0x3f);
the_res[pos++] = base64(pp[12] >> 2);
the_res[pos++] = base64(static_cast<unsigned char>((pp[12] & 0x3) << 4) + (pp[19] >> 4));
the_res[pos++] = base64(static_cast<unsigned char>((pp[19] & 0xf) << 2) + (pp[18] >> 6));
the_res[pos++] = base64(pp[18] & 0x3f);
the_res[pos++] = base64(pp[17] >> 2);
the_res[pos++] = base64(static_cast<unsigned char>((pp[17] & 0x3) << 4) + (pp[16] >> 4));
the_res[pos++] = base64(static_cast<unsigned char>((pp[16] & 0xf) << 2));
}
/*
* Hash a single 512-bit block. This is the core of the algorithm.
*/
void SHA1::transform(uint32_t block[BLOCK_BYTES])
{
/* Copy digest[] to working vars */
uint32_t a = digest[0];
uint32_t b = digest[1];
uint32_t c = digest[2];
uint32_t d = digest[3];
uint32_t e = digest[4];
/* 4 rounds of 20 operations each. Loop unrolled. */
SHA1_R0(a,b,c,d,e, 0);
SHA1_R0(e,a,b,c,d, 1);
SHA1_R0(d,e,a,b,c, 2);
SHA1_R0(c,d,e,a,b, 3);
SHA1_R0(b,c,d,e,a, 4);
SHA1_R0(a,b,c,d,e, 5);
SHA1_R0(e,a,b,c,d, 6);
SHA1_R0(d,e,a,b,c, 7);
SHA1_R0(c,d,e,a,b, 8);
SHA1_R0(b,c,d,e,a, 9);
SHA1_R0(a,b,c,d,e,10);
SHA1_R0(e,a,b,c,d,11);
SHA1_R0(d,e,a,b,c,12);
SHA1_R0(c,d,e,a,b,13);
SHA1_R0(b,c,d,e,a,14);
SHA1_R0(a,b,c,d,e,15);
SHA1_R1(e,a,b,c,d,16);
SHA1_R1(d,e,a,b,c,17);
SHA1_R1(c,d,e,a,b,18);
SHA1_R1(b,c,d,e,a,19);
SHA1_R2(a,b,c,d,e,20);
SHA1_R2(e,a,b,c,d,21);
SHA1_R2(d,e,a,b,c,22);
SHA1_R2(c,d,e,a,b,23);
SHA1_R2(b,c,d,e,a,24);
SHA1_R2(a,b,c,d,e,25);
SHA1_R2(e,a,b,c,d,26);
SHA1_R2(d,e,a,b,c,27);
SHA1_R2(c,d,e,a,b,28);
SHA1_R2(b,c,d,e,a,29);
SHA1_R2(a,b,c,d,e,30);
SHA1_R2(e,a,b,c,d,31);
SHA1_R2(d,e,a,b,c,32);
SHA1_R2(c,d,e,a,b,33);
SHA1_R2(b,c,d,e,a,34);
SHA1_R2(a,b,c,d,e,35);
SHA1_R2(e,a,b,c,d,36);
SHA1_R2(d,e,a,b,c,37);
SHA1_R2(c,d,e,a,b,38);
SHA1_R2(b,c,d,e,a,39);
SHA1_R3(a,b,c,d,e,40);
SHA1_R3(e,a,b,c,d,41);
SHA1_R3(d,e,a,b,c,42);
SHA1_R3(c,d,e,a,b,43);
SHA1_R3(b,c,d,e,a,44);
SHA1_R3(a,b,c,d,e,45);
SHA1_R3(e,a,b,c,d,46);
SHA1_R3(d,e,a,b,c,47);
SHA1_R3(c,d,e,a,b,48);
SHA1_R3(b,c,d,e,a,49);
SHA1_R3(a,b,c,d,e,50);
SHA1_R3(e,a,b,c,d,51);
SHA1_R3(d,e,a,b,c,52);
SHA1_R3(c,d,e,a,b,53);
SHA1_R3(b,c,d,e,a,54);
SHA1_R3(a,b,c,d,e,55);
SHA1_R3(e,a,b,c,d,56);
SHA1_R3(d,e,a,b,c,57);
SHA1_R3(c,d,e,a,b,58);
SHA1_R3(b,c,d,e,a,59);
SHA1_R4(a,b,c,d,e,60);
SHA1_R4(e,a,b,c,d,61);
SHA1_R4(d,e,a,b,c,62);
SHA1_R4(c,d,e,a,b,63);
SHA1_R4(b,c,d,e,a,64);
SHA1_R4(a,b,c,d,e,65);
SHA1_R4(e,a,b,c,d,66);
SHA1_R4(d,e,a,b,c,67);
SHA1_R4(c,d,e,a,b,68);
SHA1_R4(b,c,d,e,a,69);
SHA1_R4(a,b,c,d,e,70);
SHA1_R4(e,a,b,c,d,71);
SHA1_R4(d,e,a,b,c,72);
SHA1_R4(c,d,e,a,b,73);
SHA1_R4(b,c,d,e,a,74);
SHA1_R4(a,b,c,d,e,75);
SHA1_R4(e,a,b,c,d,76);
SHA1_R4(d,e,a,b,c,77);
SHA1_R4(c,d,e,a,b,78);
SHA1_R4(b,c,d,e,a,79);
/* Add the working vars back into digest[] */
digest[0] += a;
digest[1] += b;
digest[2] += c;
digest[3] += d;
digest[4] += e;
/* Count the number of transformations */
transforms++;
}
void SHA1::buffer_to_block(const std::string &buffer, uint32_t block[BLOCK_BYTES])
{
/* Convert the std::string (byte buffer) to a uint32_t array (MSB) */
for (unsigned int i = 0; i < BLOCK_INTS; i++) {
block[i] = (buffer[4*i+3] & 0xff)
| static_cast<uint32_t>(buffer[4*i+2] & 0xff)<<8
| static_cast<uint32_t>(buffer[4*i+1] & 0xff)<<16
| static_cast<uint32_t>(buffer[4*i+0] & 0xff)<<24;
}
}
void SHA1::read(std::istream &is, std::string &s, int max) {
char sbuf[BLOCK_BYTES];
is.read(sbuf, max);
s.assign(sbuf, static_cast<size_t>(is.gcount()));
}

36
src/http/sha1.h Normal file
View file

@ -0,0 +1,36 @@
// Copyright (c) 2017 The Swedish Internet Foundation
// Based on public domain code.
// Modified by Göran Andersson <goran@init.se>
#pragma once
#include <iostream>
#include <string>
#include <cstdint>
void base64_encode(const unsigned char *src, size_t len, char *destination);
class SHA1 {
public:
// You provide a buffer, at least 27 bytes large.
SHA1(char *buf);
// Result (base64 encoded) will be stored in buf, no nul termination.
void update(const char *key);
private:
static bool is_big_endian;
char *the_res;
static const uint32_t DIGEST_INTS = 5; // number of 32bit integers per SHA1 digest
static const unsigned int BLOCK_INTS = 16; // number of 32bit integers per SHA1 block
static const unsigned int BLOCK_BYTES = BLOCK_INTS * 4;
uint32_t digest[DIGEST_INTS];
uint64_t transforms;
void transform(uint32_t block[BLOCK_BYTES]);
static void buffer_to_block(const std::string &buffer, uint32_t block[BLOCK_BYTES]);
static void read(std::istream &is, std::string &s, int max);
};

View file

@ -0,0 +1,19 @@
#include "singlerequest.h"
double SingleRequest::start() {
if (!createNewConnection())
setError("SingleRequest Failure");
return _timeout;
}
double SingleRequest::timerEvent() {
if (!terminated())
setError("SingleRequest Timeout");
return 0.0;
}
bool SingleRequest::requestComplete(HttpClientConnection *conn) {
setResult(conn->contents());
_status = conn->httpStatus();
return false;
}

49
src/http/singlerequest.h Normal file
View file

@ -0,0 +1,49 @@
// Copyright (c) 2018 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
#pragma once
#include "httpclienttask.h"
class SingleRequest : public HttpClientTask {
public:
// GET url
SingleRequest(const std::string &name, const HttpHost &host,
const std::string &url, double timeout = 0.0) :
HttpClientTask(name, host), _url(url), _timeout(timeout) {
}
// POST post_data to url.
SingleRequest(const std::string &name, const HttpHost &host,
const std::string &url, const std::string &post_data,
double timeout = 0.0) :
HttpClientTask(name, host), _url(url),
_post_data(post_data), _timeout(timeout) {
}
double start() override;
double timerEvent() override;
void newRequest(HttpClientConnection *conn) override {
if (_post_data.empty())
conn->get(_url);
else
conn->post(_url, _post_data);
}
bool requestComplete(HttpClientConnection *conn) override;
void connRemoved(SocketConnection *) override {
if (!terminated())
setError("SingleRequest Failure");
}
unsigned int httpStatus() const {
return _status;
}
private:
std::string _url;
std::string _post_data;
double _timeout;
unsigned int _status = 0;
};

122
src/http/webservertask.cpp Normal file
View file

@ -0,0 +1,122 @@
// Copyright (c) 2018 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
#include <sstream>
#include <fstream>
#include "../framework/serversocket.h"
#include "webservertask.h"
#include "httpserverconnection.h"
WebServerTask::WebServerTask(const std::string &name, const TaskConfig &cfg) :
HttpTask(name),
the_config(cfg),
server_name(name) {
setFixedHeaders("");
}
double WebServerTask::start() {
log() << "WebServerTask::start()";
parseListen(the_config, "Web server");
webroot = the_config.value("webroot");
return 0;
}
std::string WebServerTask::setCookie(const std::string &name,
const std::string &val,
long expiry,
const std::string &path,
std::string domain) const {
std::string line = "Set-Cookie: " + name + "=";
if (expiry >= 0)
line += val;
if (!expiry) // Session cookie
return line + "\r\n";
if (domain.empty()) {
domain = the_config.value("cookie_domain");
if (domain.empty())
return "";
}
std::string date = dateString2(expiry < 0 ? 1 : time(nullptr) + expiry);
// Set-Cookie: previous_isp=23; expires=Wed, 07-Jun-2017 11:34:59 GMT;
// path=/; domain=.bredbandskollen.se
return line + "; expires=" + date + " GMT; path=" + path +
"; domain=" + domain + "\r\n";
}
SocketConnection *WebServerTask::
newClient(int fd, const char *ip, uint16_t port, ServerSocket *) {
return new HttpServerConnection("Web client", this, fd, ip, port);
}
HttpState WebServerTask::newPostRequest(HttpServerConnection *,
const std::string &uri) {
log() << "POST: " << uri;
return HttpState::READING_POST_DATA;
}
// Return true to continue, false to close the connection.
bool WebServerTask::partialPostData(HttpServerConnection *,
const char *, size_t ) {
return true;
}
HttpState WebServerTask::lastPostData(HttpServerConnection *,
const char *, size_t ) {
return HttpState::CLOSE;
}
bool WebServerTask::newWsRequest(HttpServerConnection *,
const std::string &uri) {
log() << "WS url: " << uri;
return true;
}
void WebServerTask::wsHandshakeFinished(HttpServerConnection *,
const std::string &) {
}
// Origin: http://www.mydomain.com:9001
// Origin: https://www.mydomain.com
// domain should be "mydomain.com" or "www.mydomain.com"
std::string WebServerTask::corsHeader(HttpServerConnection *conn,
const std::string &domain) {
if (domain.empty())
return "Access-Control-Allow-Origin: *\r\n";
std::string origin = conn->getHeaderVal("origin");
auto pos = origin.find(domain);
if (pos == std::string::npos)
return std::string();
if (pos && origin[pos-1] != '/' && origin[pos-1] != '.')
return std::string();
if (origin.size() != pos + domain.size() &&
origin[pos + domain.size()] != ':')
return std::string();
return "Access-Control-Allow-Origin: " + origin + "\r\n";
}
#ifdef USE_GNUTLS
#include "../framework/socketreceiver.h"
void WebServerTask::newWorkerChannel(SocketReceiver *srv, unsigned int chan) {
std::string cfg = the_config.value("channel" + std::to_string(chan));
if (cfg.empty())
return;
std::istringstream s(cfg);
std::string tls, cert, key, password;
s >> tls >> cert >> key >> password;
if (tls != "tls")
return;
if (key.empty() || !tlsSetKey(srv, cert, key, password)) {
err_log() << "cannot load TLS certificate " << cert;
setError("cannot use TLS certificate");
}
}
#endif

141
src/http/webservertask.h Normal file
View file

@ -0,0 +1,141 @@
// Copyright (c) 2018 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
#pragma once
#include <stdexcept>
#include <map>
#include "httptask.h"
#include "httpserverconnection.h"
#ifdef USE_GNUTLS
class SocketReceiver;
#endif
/// API for HTTP servers.
class WebServerTask : public HttpTask {
public:
WebServerTask(const std::string &name, const TaskConfig &cfg=TaskConfig());
double start() override;
std::string headers(const std::string &code) {
return "HTTP/1.1 " + code + "\r\n" + fixed_response_headers;
}
/// \brief
/// Return a string that may be inserted into the HTTP response headers.
///
/// Let expiry be 0 for a session cookie, < 0 to delete cookie, otherwise
/// expiry should be the cookie's max-time in seconds.
/// If domain is empty, the cookie_domain config parameter will be used.
/// Example:
///
/// conn->sendHttpResponse(headers("200 OK",
/// setCookie("lang", "en", 86400)),
// "text/plain", data)
std::string setCookie(const std::string &name, const std::string &val,
long expiry = 0, const std::string &path = "/",
std::string domain = "") const;
/// When you overload this, call sendHttpResponse and return
/// WAITING_FOR_REQUEST (or CLOSE), or
/// call sendHttpResponseHeader and return SENDING_RESPONSE.
/// In the latter case, your overloaded sendResponseData method will be
/// called regularly until all is sent.
virtual HttpState newGetRequest(HttpServerConnection *,
const std::string &) {
return HttpState::CLOSE;
}
/// Overload this if you really have to take control over how the response
/// is sent. Send up to bytes_left bytes through conn, return _exactly_ the
/// number of bytes written. To get it correct, you should just do
///
/// return conn->sendData(...)
virtual size_t sendResponseData(HttpServerConnection * /* conn */,
size_t /* bytes_left */) {
// Essentially, your overloaded code should do this:
// my_buffer = ...
// size_t sent = conn->sendData(my_buffer,
// std::min(bytes_left, sizeof my_buffer));
// return sent;
throw std::logic_error("sendResponseData not implemented");
}
virtual HttpState newPostRequest(HttpServerConnection *conn,
const std::string &uri);
/// \brief
/// Retrieve part of a post message from client.
///
/// Return true to continue, false to close the connection.
virtual bool partialPostData(HttpServerConnection *conn,
const char *buffer, size_t len);
/// \brief
/// Retrieve last part of a post message from client.
virtual HttpState lastPostData(HttpServerConnection *conn,
const char *buffer, size_t len);
/// \brief
/// Request from client for a websocket upgrade.
///
/// Return true to accept, false to close.
virtual bool newWsRequest(HttpServerConnection *conn,
const std::string &uri);
/// If you want to send data immediately after a websocket connection
/// has been established (i.e. before the client has asked for
/// anything), call conn->notifyWsHandshake() in the newWsRequest method;
/// then the below method will be called after the handshake:
virtual void wsHandshakeFinished(HttpServerConnection *conn,
const std::string &uri);
/// Override this to implement a response to a preflight (OPTIONS) request.
virtual HttpState preflightRequest(HttpServerConnection * /* conn */,
const std::string & /* uri */) {
return HttpState::CLOSE;
}
void connAdded(SocketConnection *conn) override {
log() << "new connection id=" << conn->id();
}
void connRemoved(SocketConnection *conn) override {
log() << "dropped connection " << conn->id();
}
/// \brief
/// One or more HTTP response header lines that always should be sent.
///
/// Terminate each line with \r\n.
void setFixedHeaders(const std::string &hdr) {
fixed_response_headers = "Server: " + server_name + "\r\n" + hdr;
}
SocketConnection *newClient(int fd, const char *ip, uint16_t port,
ServerSocket *) final;
/// Return Access-Control-Allow-Origin header if the conn's current request
/// originates from domain (or a subdomain to it), otherwise return "".
static std::string corsHeader(HttpServerConnection *conn,
const std::string &domain);
std::string webRoot() const {
return webroot;
}
void setWebRoot(const std::string &path) {
webroot = path;
}
#ifdef USE_GNUTLS
void newWorkerChannel(SocketReceiver *srv, unsigned int chan) override;
#endif
protected:
TaskConfig the_config;
private:
std::string server_name, fixed_response_headers;
std::string webroot;
};

View file

@ -0,0 +1,239 @@
// Copyright (c) 2018 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
#include <string>
#include "../json11/json11.hpp"
#include "websocketbridge.h"
#include "../http/httpclientconnection.h"
#if defined(__APPLE__)
#include <CoreFoundation/CFBundle.h>
#if !TARGET_OS_IPHONE
#include <ApplicationServices/ApplicationServices.h>
#endif
#endif
class WSBlistener : public WebServerTask {
public:
WSBlistener(Task *bridge, const TaskConfig &cfg);
bool newWsRequest(HttpServerConnection *conn,
const std::string &uri) override;
bool wsTextMessage(HttpConnection *conn,
const std::string &msg) override;
void sendMsgToClient(const std::string &msg) {
dbg_log() << "Send to client: " << msg;
if (client)
client->sendWsMessage(msg);
}
void connAdded(SocketConnection *s) override;
void serverAdded(ServerSocket *s) override;
void connRemoved(SocketConnection *s) override;
void serverRemoved(ServerSocket *s) override;
uint16_t listenPort() const {
if (server)
return server->port();
return 0;
}
#if !TARGET_OS_IPHONE
void run_browser(const std::string &url);
#endif
void processFinished(int pid, int wstatus) override;
// Returns empty string unless ready:
std::string url() const {
return connect_url;
}
// Return true if client is connected:
bool clientConnected() const {
return client != nullptr;
}
private:
void notify_url() {
std::cerr << "\nOPEN URL: " << connect_url << std::endl;
}
Task *parent;
int external_browser_pid = 0;
std::string connect_url;
HttpServerConnection *client = nullptr;
ServerSocket *server = nullptr;
TaskConfig config;
};
WebsocketBridge::WebsocketBridge(Task *agent, const TaskConfig &cfg) :
BridgeTask(agent),
listen_task(new WSBlistener(this, cfg)) {
killChildTaskWhenFinished();
}
WebsocketBridge::~WebsocketBridge() {
}
double WebsocketBridge::start() {
dbg_log() << "WebsocketBridge::start()";
double x = BridgeTask::start();
addNewTask(listen_task, this);
listen_port.store(listen_task->listenPort());
listen_task->startObserving(this);
dbg_log() << "Listening on port " << listen_port.load();
return x;
}
void WebsocketBridge::handleExecution(Task *sender, const std::string &msg) {
log() << "Got msg: " << msg;
if (sender == listen_task)
sendMsgToAgent(msg);
}
void WebsocketBridge::taskFinished(Task *task) {
if (task == listen_task) {
log() << "Listen task dead";
// We will not be able to talk to the client anymore.
listen_task = nullptr;
setResult("");
} else {
log() << "Agent task dead";
BridgeTask::taskFinished(task);
}
}
std::string WebsocketBridge::url() const {
if (!listen_task || !listen_port.load())
return std::string();
return listen_task->url();
}
std::string WebsocketBridge::port() const {
if (!listen_task || !listen_port.load())
return std::string();
return std::to_string(listen_port.load());
}
bool WebsocketBridge::clientConnected() const {
return (listen_task && listen_task->clientConnected());
}
WSBlistener::WSBlistener(Task *bridge, const TaskConfig &cfg) :
WebServerTask("WSB", "listen " + cfg.value("listen") +
" " + cfg.value("listen_addr")),
parent(bridge),
config(cfg) {
}
void WebsocketBridge::sendMsgToClient(const std::string &msg) {
log() << "Send msg " << msg;
if (listen_task)
listen_task->sendMsgToClient(msg);
}
#if !TARGET_OS_IPHONE
void WSBlistener::run_browser(const std::string &url) {
#if defined(__APPLE__)
CFURLRef url2 = CFURLCreateWithBytes(nullptr, (UInt8 *)url.c_str(),
url.size(), kCFStringEncodingASCII,
nullptr);
LSOpenCFURLRef(url2, 0);
CFRelease(url2);
#elif defined(_WIN32)
ShellExecuteA(NULL, "open", url.c_str(), NULL, NULL, SW_SHOWNORMAL);
#else
const char *const argv[] = { "xdg-open", url.c_str(), nullptr };
external_browser_pid = runProcess(argv);
#endif
}
#endif
void WSBlistener::processFinished(int pid, int wstatus) {
log() << "Process " << pid << " finished, status: " << wstatus;
if (pid == external_browser_pid && wstatus)
notify_url();
}
bool WSBlistener::newWsRequest(HttpServerConnection *conn,
const std::string &uri) {
if (client) {
log() << "error: already have a client";
return false;
}
if (uri != "/wsbridge") {
log() << "bad uri: " << uri;
return false;
}
if (config.value("listen_pw") != conn->getQueryVal("pwd")) {
log() << "bad password";
return false;
}
client = conn;
// Accept only one client:
if (server)
server->stopListening();
return true;
}
bool WSBlistener::wsTextMessage(HttpConnection *,
const std::string &msg) {
log() << "Got msg " << msg;
executeHandler(parent, msg);
return true;
}
void WSBlistener::connAdded(SocketConnection *) {
log() << "WebsocketBridge client connected";
}
void WSBlistener::serverAdded(ServerSocket *socket) {
log() << "WebsocketBridge listening on port " << socket->port();
server = socket;
std::map<std::string, std::string> pars;
if (config.value("browser") == "3") {
connect_url = "http://webview.bredbandskollen.se/";
pars["bridge"] = "ws";
pars["wsport"] = std::to_string(socket->port());
pars["backend"] = "frontend-beta.bredbandskollen.se";
pars["env"] = "generic";
if (!config.value("listen_pw").empty())
pars["wspwd"] = config.value("listen_pw");
} else {
connect_url = config.value("url");
pars["bridge"] = "ws";
pars["port"] = std::to_string(socket->port());
if (socket->hostname() != "127.0.0.1")
pars["ip"] = socket->hostname();
if (!config.value("listen_pw").empty())
pars["pwd"] = config.value("listen_pw");
}
HttpClientConnection::addUrlPars(connect_url, pars);
log() << "Browser: " << connect_url;
#if !TARGET_OS_IPHONE
if (config.value("browser") == "1") {
run_browser(connect_url);
} else if (config.value("browser") == "0") {
notify_url();
}
#endif
}
void WSBlistener::connRemoved(SocketConnection *s) {
if (dynamic_cast<HttpServerConnection *>(s) == client) {
log() << "WebsocketBridge client disconnected";
client = nullptr;
setResult("");
}
}
void WSBlistener::serverRemoved(ServerSocket *socket) {
log() << "WebsocketBridge listening port closed: " << socket->port();
server = nullptr;
}

View file

@ -0,0 +1,65 @@
// Copyright (c) 2018 The Swedish Internet Foundation
// Written by Göran Andersson <initgoran@gmail.com>
#pragma once
#include <deque>
#include <atomic>
#include "../framework/bridgetask.h"
#include "../http/webservertask.h"
#include "../framework/serversocket.h"
class WSBlistener;
// Open a web server, wait for one (and only one) client to open a websocket
// connection to the server. The websocket connection will then be used to
// pass text messages between the agent and the client.
// The client must connect to the URL /wsbridge
//
// Config parameter - ValIT
//
// listen - port number for the web server (0 to use a random port)
// listen_addr - IP number to listen on, e.g. 127.0.0.1 (or 0.0.0.0 for any)
// listen_pw - (optional) password; cilent must supply the password as value
// of URL parameter pwd when connecting to /wsbridge
// browser - (optional) value 0, 1, 2, or 3. If 0, write an URL to stderr;
// user may open the URL in a browser to load a web interface.
// If 1, an attempt to fire up the web interface in a new browser
// tab will be made. If that fails, write the URL to stderr.
// If 2, only generate the URL to the web interface. The client
// program must open the URL in a web view.
// If 3, generate the URL to the next generation web interface. The
// client program must open the URL in a web view. The web interface
// uses HTTPS, but must connect using an unencrypted web socket.
// Note that most webview components will refuse to do so.
// url - (optional) domain name to the server containing the legacy web
// interface; ignored if value of "browser" is "3".
class WebsocketBridge : public BridgeTask {
public:
WebsocketBridge(Task *agent, const TaskConfig &cfg);
virtual ~WebsocketBridge() override;
double start() override;
void sendMsgToClient(const std::string &msg) override;
void handleExecution(Task *sender, const std::string &msg) override;
void taskFinished(Task *task) override;
// Returns empty string unless ready. Thread safe.
std::string url() const;
std::string port() const;
// Return true if client is connected:
bool clientConnected() const;
private:
WSBlistener *listen_task = nullptr;
std::atomic<uint16_t> listen_port;
};