cgi.d arsd update

1 files changed, 426 insertions, 133 deletions

diff --git a/misc/ext_lib/src/arsd/cgi.d b/misc/ext_lib/src/arsd/cgi.d
index 1afd32b..a0249ee 100644
--- a/misc/ext_lib/src/arsd/cgi.d
+++ b/misc/ext_lib/src/arsd/cgi.d
@@ -150,7 +150,7 @@ void main() {
 	
 	For CGI, `dmd yourfile.d cgi.d` then put the executable in your cgi-bin directory.
 
-	For FastCGI: `dmd yourfile.d cgi.d -version=fastcgi` and run it. spawn-fcgi helps on nginx. You can put the file in the directory for Apache. On IIS, run it with a port on the command line.
+	For FastCGI: `dmd yourfile.d cgi.d -version=fastcgi` and run it. spawn-fcgi helps on nginx. You can put the file in the directory for Apache. On IIS, run it with a port on the command line (this causes it to call FCGX_OpenSocket, which can work on nginx too).
 
 	For SCGI: `dmd yourfile.d cgi.d -version=scgi` and run the executable, providing a port number on the command line.
 
@@ -333,12 +333,17 @@ void main() {
 
 	Copyright:
 
-	cgi.d copyright 2008-2019, Adam D. Ruppe. Provided under the Boost Software License.
+	cgi.d copyright 2008-2021, Adam D. Ruppe. Provided under the Boost Software License.
 
-	Yes, this file is almost ten years old, and yes, it is still actively maintained and used.
+	Yes, this file is old, and yes, it is still actively maintained and used.
 +/
 module arsd.cgi;
 
+version(Demo)
+unittest {
+
+}
+
 static import std.file;
 
 // for a single thread, linear request thing, use:
@@ -352,6 +357,8 @@ version(Posix) {
 	} else {
 		version(GNU) {
 			// GDC doesn't support static foreach so I had to cheat on it :(
+		} else version(FreeBSD) {
+			// I never implemented the fancy stuff there either
 		} else {
 			version=with_breaking_cgi_features;
 			version=with_sendfd;
@@ -376,7 +383,8 @@ void cloexec(Socket s) {
 
 version(embedded_httpd_hybrid) {
 	version=embedded_httpd_threads;
-	version=cgi_use_fork;
+	version(cgi_no_fork) {} else
+		version=cgi_use_fork;
 	version=cgi_use_fiber;
 }
 
@@ -440,9 +448,6 @@ enum long defaultMaxContentLength = 5_000_000;
 // somehow in here and dom.d.
 
 
-// FIXME: 100 Continue in the nph section? Probably belongs on the
-// httpd class though.
-
 // these are public so you can mixin GenericMain.
 // FIXME: use a function level import instead!
 public import std.string;
@@ -845,7 +850,7 @@ class Cgi {
 		this.requestUri = requestUri;
 		this.pathInfo = pathInfo;
 		this.queryString = queryString;
-		this.postJson = null;
+		this.postBody = null;
 	}
 
 	private {
@@ -1067,7 +1072,7 @@ class Cgi {
 				filesArray = assumeUnique(pps._files);
 				files = keepLastOf(filesArray);
 				post = keepLastOf(postArray);
-				this.postJson = pps.postJson;
+				this.postBody = pps.postBody;
 				cleanUpPostDataState();
 			}
 
@@ -1112,7 +1117,7 @@ class Cgi {
 			string boundary;
 			string localBoundary; // the ones used at the end or something lol
 			bool isMultipart;
-			bool isJson;
+			bool needsSavedBody;
 
 			ulong expectedLength;
 			ulong contentConsumed;
@@ -1124,7 +1129,7 @@ class Cgi {
 			string[] thisOnesHeaders;
 			immutable(ubyte)[] thisOnesData;
 
-			string postJson;
+			string postBody;
 
 			UploadedFile piece;
 			bool isFile = false;
@@ -1233,23 +1238,22 @@ class Cgi {
 		// but it seems to me that this only happens when it is urlencoded.
 		if(pps.contentType == "application/x-www-form-urlencoded" || pps.contentType == "") {
 			pps.isMultipart = false;
+			pps.needsSavedBody = false;
 		} else if(pps.contentType == "multipart/form-data") {
 			pps.isMultipart = true;
 			enforce(pps.boundary.length, "no boundary");
-		} else if(pps.contentType == "text/plain") {
-			pps.isMultipart = false;
-			pps.isJson = true; // FIXME: hack, it isn't actually this
-		} else if(pps.contentType == "text/xml") { // FIXME: what if we used this as a fallback?
+		} else if(pps.contentType == "text/xml") { // FIXME: could this be special and load the post params
+			// save the body so the application can handle it
 			pps.isMultipart = false;
-			pps.isJson = true; // FIXME: hack, it isn't actually this
-		} else if(pps.contentType == "application/json") {
-			pps.isJson = true;
+			pps.needsSavedBody = true;
+		} else if(pps.contentType == "application/json") { // FIXME: this could prolly try to load post params too
+			// save the body so the application can handle it
+			pps.needsSavedBody = true;
 			pps.isMultipart = false;
-		//} else if(pps.contentType == "application/json") {
-			//pps.isJson = true;
 		} else {
-			// FIXME: should set a http error code too
-			throw new Exception("unknown request content type: " ~ pps.contentType);
+			// the rest is 100% handled by the application. just save the body and send it to them
+			pps.needsSavedBody = true;
+			pps.isMultipart = false;
 		}
 	}
 
@@ -1574,8 +1578,8 @@ class Cgi {
 
 			// simple handling, but it works... until someone bombs us with gigabytes of crap at least...
 			if(pps.buffer.length == pps.expectedLength) {
-				if(pps.isJson)
-					pps.postJson = cast(string) pps.buffer;
+				if(pps.needsSavedBody)
+					pps.postBody = cast(string) pps.buffer;
 				else
 					pps._post = decodeVariables(cast(string) pps.buffer, "&", &allPostNamesInOrder, &allPostValuesInOrder);
 				version(preserveData)
@@ -1747,6 +1751,7 @@ class Cgi {
 				parts.popFront();
 				requestUri = parts.front;
 
+				// FIXME:  the requestUri could be an absolute path!!! should I rename it or something?
 				scriptName = requestUri[0 .. pathInfoStarts];
 
 				auto question = requestUri.indexOf("?");
@@ -1836,6 +1841,16 @@ class Cgi {
 				}
 				else if (name == "cookie") {
 					cookie ~= value;
+				} else if(name == "expect") {
+					if(value == "100-continue") {
+						// FIXME we should probably give user code a chance
+						// to process and reject but that needs to be virtual,
+						// perhaps part of the CGI redesign.
+
+						// FIXME: if size is > max content length it should
+						// also fail at this point.
+						_rawDataOutput(cast(ubyte[]) "HTTP/1.1 100 Continue\r\n\r\n");
+					}
 				}
 				// else
 				// ignore it
@@ -1876,7 +1891,7 @@ class Cgi {
 			filesArray = assumeUnique(pps._files);
 			files = keepLastOf(filesArray);
 			post = keepLastOf(postArray);
-			postJson = pps.postJson;
+			postBody = pps.postBody;
 			cleanUpPostDataState();
 		}
 
@@ -1995,13 +2010,22 @@ class Cgi {
 		return scriptName;
 	}
 
-	/// Sets the HTTP status of the response. For example, "404 File Not Found" or "500 Internal Server Error".
-	/// It assumes "200 OK", and automatically changes to "302 Found" if you call setResponseLocation().
-	/// Note setResponseStatus() must be called *before* you write() any data to the output.
+	/++
+		Sets the HTTP status of the response. For example, "404 File Not Found" or "500 Internal Server Error".
+		It assumes "200 OK", and automatically changes to "302 Found" if you call setResponseLocation().
+		Note setResponseStatus() must be called *before* you write() any data to the output.
+
+		History:
+			The `int` overload was added on January 11, 2021.
+	+/
 	void setResponseStatus(string status) {
 		assert(!outputtedResponseData);
 		responseStatus = status;
 	}
+	/// ditto
+	void setResponseStatus(int statusCode) {
+		setResponseStatus(getHttpCodeText(statusCode));
+	}
 	private string responseStatus = null;
 
 	/// Returns true if it is still possible to output headers
@@ -2081,11 +2105,26 @@ class Cgi {
 	private bool publicCaching = false;
 	*/
 
-	/// Sets an HTTP cookie, automatically encoding the data to the correct string.
-	/// expiresIn is how many milliseconds in the future the cookie will expire.
-	/// TIP: to make a cookie accessible from subdomains, set the domain to .yourdomain.com.
-	/// Note setCookie() must be called *before* you write() any data to the output.
-	void setCookie(string name, string data, long expiresIn = 0, string path = null, string domain = null, bool httpOnly = false, bool secure = false) {
+	/++
+		History:
+			Added January 11, 2021
+	+/
+	enum SameSitePolicy {
+		Lax,
+		Strict,
+		None
+	}
+
+	/++
+		Sets an HTTP cookie, automatically encoding the data to the correct string.
+		expiresIn is how many milliseconds in the future the cookie will expire.
+		TIP: to make a cookie accessible from subdomains, set the domain to .yourdomain.com.
+		Note setCookie() must be called *before* you write() any data to the output.
+
+		History:
+			Parameter `sameSitePolicy` was added on January 11, 2021.
+	+/
+	void setCookie(string name, string data, long expiresIn = 0, string path = null, string domain = null, bool httpOnly = false, bool secure = false, SameSitePolicy sameSitePolicy = SameSitePolicy.Lax) {
 		assert(!outputtedResponseData);
 		string cookie = std.uri.encodeComponent(name) ~ "=";
 		cookie ~= std.uri.encodeComponent(data);
@@ -2100,6 +2139,18 @@ class Cgi {
 			cookie ~= "; Secure";
 		if(httpOnly == true )
 			cookie ~= "; HttpOnly";
+		final switch(sameSitePolicy) {
+			case SameSitePolicy.Lax:
+				cookie ~= "; SameSite=Lax";
+			break;
+			case SameSitePolicy.Strict:
+				cookie ~= "; SameSite=Strict";
+			break;
+			case SameSitePolicy.None:
+				cookie ~= "; SameSite=None";
+				assert(secure); // cookie spec requires this now, see: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite
+			break;
+		}
 
 		if(auto idx = name in cookieIndexes) {
 			responseCookies[*idx] = cookie;
@@ -2302,6 +2353,18 @@ class Cgi {
 				// maybeAutoClose can be false though to avoid this (important if you call from inside close()!
 	}
 
+	/++
+		Convenience method to set content type to json and write the string as the complete response.
+
+		History:
+			Added January 16, 2020
+	+/
+	void writeJson(string json) {
+		this.setResponseContentType("application/json");
+		this.write(json, true);
+	}
+
+	/// Flushes the pending buffer, leaving the connection open so you can send more.
 	void flush() {
 		if(rawDataOutput is null)
 			stdout.flush();
@@ -2447,7 +2510,8 @@ class Cgi {
 	version(preserveData) // note: this can eat lots of memory; don't use unless you're sure you need it.
 	immutable(ubyte)[] originalPostData;
 
-	public immutable string postJson;
+	public immutable string postBody;
+	alias postJson = postBody; // old name
 
 	/* Internal state flags */
 	private bool outputtedResponseData;
@@ -2797,7 +2861,20 @@ struct Uri {
 				authority = authority[idx2 + 1 .. $];
 			}
 
-			idx2 = authority.indexOf(":");
+			if(authority.length && authority[0] == '[') {
+				// ipv6 address special casing
+				idx2 = authority.indexOf(']');
+				if(idx2 != -1) {
+					auto end = authority[idx2 + 1 .. $];
+					if(end.length && end[0] == ':')
+						idx2 = idx2 + 1;
+					else
+						idx2 = -1;
+				}
+			} else {
+				idx2 = authority.indexOf(":");
+			}
+
 			if(idx2 == -1) {
 				port = 0; // 0 means not specified; we should use the default for the scheme
 				host = authority;
@@ -2946,6 +3023,21 @@ struct Uri {
 		uri = Uri("?lol#foo");
 		assert(uri.fragment == "foo");
 		assert(uri.query == "lol");
+
+		uri = Uri("http://127.0.0.1/");
+		assert(uri.host == "127.0.0.1");
+		assert(uri.port == 0);
+
+		uri = Uri("http://127.0.0.1:123/");
+		assert(uri.host == "127.0.0.1");
+		assert(uri.port == 123);
+
+		uri = Uri("http://[ff:ff::0]/");
+		assert(uri.host == "[ff:ff::0]");
+
+		uri = Uri("http://[ff:ff::0]:123/");
+		assert(uri.host == "[ff:ff::0]");
+		assert(uri.port == 123);
 	}
 
 	// This can sometimes be a big pain in the butt for me, so lots of copy/paste here to cover
@@ -3231,7 +3323,7 @@ mixin template CustomCgiMain(CustomCgi, alias fun, long maxContentLength = defau
 }
 
 version(embedded_httpd_processes)
-	int processPoolSize = 8;
+	__gshared int processPoolSize = 8;
 
 // Returns true if run. You should exit the program after that.
 bool tryAddonServers(string[] args) {
@@ -3240,10 +3332,16 @@ bool tryAddonServers(string[] args) {
 		switch(args[1]) {
 			case "--websocket-server":
 				version(with_addon_servers)
-					runWebsocketServer();
+					websocketServers[args[2]](args[3 .. $]);
 				else
 					printf("Add-on servers not compiled in.\n");
 				return true;
+			case "--websocket-servers":
+				import core.demangle;
+				version(with_addon_servers_connections)
+				foreach(k, v; websocketServers)
+					writeln(k, "\t", demangle(k));
+				return true;
 			case "--session-server":
 				version(with_addon_servers)
 					runSessionServer();
@@ -3300,7 +3398,7 @@ bool trySimulatedRequest(alias fun, CustomCgi = Cgi)(string[] args) if(is(Custom
 +/
 struct RequestServer {
 	///
-	string listeningHost;
+	string listeningHost = defaultListeningHost();
 	///
 	ushort listeningPort = defaultListeningPort();
 
@@ -3319,6 +3417,8 @@ struct RequestServer {
 	void configureFromCommandLine(string[] args) {
 		bool foundPort = false;
 		bool foundHost = false;
+		bool foundUid = false;
+		bool foundGid = false;
 		foreach(arg; args) {
 			if(foundPort) {
 				listeningPort = to!ushort(arg);
@@ -3328,15 +3428,29 @@ struct RequestServer {
 				listeningHost = arg;
 				foundHost = false;
 			}
+			if(foundUid) {
+				privDropUserId = to!int(arg);
+				foundUid = false;
+			}
+			if(foundGid) {
+				privDropGroupId = to!int(arg);
+				foundGid = false;
+			}
 			if(arg == "--listening-host" || arg == "-h" || arg == "/listening-host")
 				foundHost = true;
 			else if(arg == "--port" || arg == "-p" || arg == "/port" || arg == "--listening-port")
 				foundPort = true;
+			else if(arg == "--uid")
+				foundUid = true;
+			else if(arg == "--gid")
+				foundGid = true;
 		}
 	}
 
+	// FIXME: the privDropUserId/group id need to be set in here instead of global
+
 	/++
-		Serves a single request on this thread, with an embedded server, then stops. Designed for cases like embedded oauth responders
+		Serves a single HTTP request on this thread, with an embedded server, then stops. Designed for cases like embedded oauth responders
 
 		History:
 			Added Oct 10, 2020.
@@ -3348,7 +3462,7 @@ struct RequestServer {
 			RequestServer server = RequestServer("127.0.0.1", 6789);
 			string oauthCode;
 			string oauthScope;
-			server.serveOnce!((cgi) {
+			server.serveHttpOnce!((cgi) {
 				oauthCode = cgi.request("code");
 				oauthScope = cgi.request("scope");
 				cgi.write("Thank you, please return to the application.");
@@ -3357,7 +3471,7 @@ struct RequestServer {
 		}
 		---
 	+/
-	void serveOnce(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)() {
+	void serveHttpOnce(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)() {
 		import std.socket;
 
 		bool tcp;
@@ -3387,12 +3501,10 @@ struct RequestServer {
 			serveEmbeddedHttpdProcesses!(fun, CustomCgi)(this);
 		} else
 		version(embedded_httpd_threads) {
-			auto manager = new ListeningConnectionManager(listeningHost, listeningPort, &doThreadHttpConnection!(CustomCgi, fun));
-			manager.listen();
+			serveEmbeddedHttp!(fun, CustomCgi, maxContentLength)();
 		} else
 		version(scgi) {
-			auto manager = new ListeningConnectionManager(listeningHost, listeningPort, &doThreadScgiConnection!(CustomCgi, fun, maxContentLength));
-			manager.listen();
+			serveScgi!(fun, CustomCgi, maxContentLength)();
 		} else
 		version(fastcgi) {
 			serveFastCgi!(fun, CustomCgi, maxContentLength)(this);
@@ -3402,9 +3514,40 @@ struct RequestServer {
 		}
 	}
 
+	void serveEmbeddedHttp(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)() {
+		auto manager = new ListeningConnectionManager(listeningHost, listeningPort, &doThreadHttpConnection!(CustomCgi, fun));
+		manager.listen();
+	}
+	void serveScgi(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)() {
+		auto manager = new ListeningConnectionManager(listeningHost, listeningPort, &doThreadScgiConnection!(CustomCgi, fun, maxContentLength));
+		manager.listen();
+	}
+
 	void stop() {
+		// FIXME
+	}
+}
+
+private int privDropUserId;
+private int privDropGroupId;
+
+// Added Jan 11, 2021
+private void dropPrivs() {
+	version(Posix) {
+		import core.sys.posix.unistd;
+
+		auto userId = privDropUserId;
+		auto groupId = privDropGroupId;
+
+		if((userId != 0 || groupId != 0) && getuid() == 0) {
+			if(groupId)
+				setgid(groupId);
+			if(userId)
+				setuid(userId);
+		}
 
 	}
+	// FIXME: Windows?
 }
 
 version(embedded_httpd_processes)
@@ -3450,6 +3593,7 @@ void serveEmbeddedHttpdProcesses(alias fun, CustomCgi = Cgi)(RequestServer param
 			close(sock);
 			throw new Exception("listen");
 		}
+		dropPrivs();
 	}
 
 	version(embedded_httpd_processes_accept_after_fork) {} else {
@@ -3581,6 +3725,8 @@ void serveEmbeddedHttpdProcesses(alias fun, CustomCgi = Cgi)(RequestServer param
 					// most likely cause is a timeout
 				}
 			}
+		} else if(newPid < 0) {
+			throw new Exception("fork failed");
 		} else {
 			processCount++;
 		}
@@ -3756,6 +3902,21 @@ ushort defaultListeningPort() {
 		return 0;
 }
 
+/// Default host for listening. 127.0.0.1 for scgi, null (aka all interfaces) for all others. If you want the server directly accessible from other computers on the network, normally use null. If not, 127.0.0.1 is a bit better. Settable with default handlers with --listening-host command line argument.
+string defaultListeningHost() {
+	version(netman_httpd)
+		return null;
+	else version(embedded_httpd_processes)
+		return null;
+	else version(embedded_httpd_threads)
+		return null;
+	else version(scgi)
+		return "127.0.0.1";
+	else
+		return null;
+
+}
+
 /++
 	This is the function [GenericMain] calls. View its source for some simple boilerplate you can copy/paste and modify, or you can call it yourself from your `main`.
 
@@ -3875,14 +4036,23 @@ void handleCgiRequest(alias fun, CustomCgi = Cgi, long maxContentLength = defaul
 
 version(cgi_use_fiber)
 class CgiFiber : Fiber {
+	private void function(Socket) f_handler;
+	private void f_handler_dg(Socket s) { // to avoid extra allocation w/ function
+		f_handler(s);
+	}
 	this(void function(Socket) handler) {
+		this.f_handler = handler;
+		this(&f_handler_dg);
+	}
+
+	this(void delegate(Socket) handler) {
 		this.handler = handler;
 		// FIXME: stack size
 		super(&run);
 	}
 
 	Socket connection;
-	void function(Socket) handler;
+	void delegate(Socket) handler;
 
 	void run() {
 		handler(connection);
@@ -3956,6 +4126,9 @@ void doThreadHttpConnectionGuts(CustomCgi, alias fun, bool alwaysCloseConnection
 		try {
 			cgi = new CustomCgi(ir, &closeConnection);
 			cgi._outputFileHandle = connection.handle;
+		} catch(ConnectionClosedException ce) {
+			closeConnection = true;
+			break;
 		} catch(ConnectionException ce) {
 			// broken pipe or something, just abort the connection
 			closeConnection = true;
@@ -3985,6 +4158,9 @@ void doThreadHttpConnectionGuts(CustomCgi, alias fun, bool alwaysCloseConnection
 		} catch(ConnectionException ce) {
 			// broken pipe or something, just abort the connection
 			closeConnection = true;
+		} catch(ConnectionClosedException ce) {
+			// broken pipe or something, just abort the connection
+			closeConnection = true;
 		} catch(Throwable t) {
 			// a processing error can be recovered from
 			version(CRuntime_Musl) {} else
@@ -3994,6 +4170,7 @@ void doThreadHttpConnectionGuts(CustomCgi, alias fun, bool alwaysCloseConnection
 		}
 
 		if(closeConnection || alwaysCloseConnection) {
+			connection.shutdown(SocketShutdown.BOTH);
 			connection.close();
 			ir.dispose();
 			closeConnection = false; // don't reclose after loop
@@ -4002,6 +4179,7 @@ void doThreadHttpConnectionGuts(CustomCgi, alias fun, bool alwaysCloseConnection
 			if(ir.front.length) {
 				ir.popFront(); // we can't just discard the buffer, so get the next bit and keep chugging along
 			} else if(ir.sourceClosed) {
+				ir.source.shutdown(SocketShutdown.BOTH);
 				ir.source.close();
 				ir.dispose();
 				closeConnection = false;
@@ -4013,6 +4191,7 @@ void doThreadHttpConnectionGuts(CustomCgi, alias fun, bool alwaysCloseConnection
 	}
 
 	if(closeConnection) {
+		connection.shutdown(SocketShutdown.BOTH);
 		connection.close();
 		ir.dispose();
 	}
@@ -4020,7 +4199,6 @@ void doThreadHttpConnectionGuts(CustomCgi, alias fun, bool alwaysCloseConnection
 	// I am otherwise NOT closing it here because the parent thread might still be able to make use of the keep-alive connection!
 }
 
-version(scgi)
 void doThreadScgiConnection(CustomCgi, alias fun, long maxContentLength)(Socket connection) {
 	// and now we can buffer
 	scope(failure)
@@ -4115,6 +4293,9 @@ void doThreadScgiConnection(CustomCgi, alias fun, long maxContentLength)(Socket
 		if(!handleException(cgi, t)) {
 			connection.close();
 			return;
+		} else {
+			connection.close();
+			return;
 		}
 	}
 }
@@ -4634,7 +4815,16 @@ class ListeningConnectionManager {
 	bool tcp;
 	void delegate() cleanup;
 
+	private void function(Socket) fhandler;
+	private void dg_handler(Socket s) {
+		fhandler(s);
+	}
 	this(string host, ushort port, void function(Socket) handler) {
+		fhandler = handler;
+		this(host, port, &dg_handler);
+	}
+
+	this(string host, ushort port, void delegate(Socket) handler) {
 		this.handler = handler;
 
 		listener = startListening(host, port, tcp, cleanup, 128);
@@ -4647,7 +4837,7 @@ class ListeningConnectionManager {
 	}
 
 	Socket listener;
-	void function(Socket) handler;
+	void delegate(Socket) handler;
 
 	bool running;
 	void quit() {
@@ -4695,6 +4885,9 @@ Socket startListening(string host, ushort port, ref bool tcp, ref void delegate(
 	}
 
 	listener.listen(backQueue);
+
+	dropPrivs();
+
 	return listener;
 }
 
@@ -4728,7 +4921,7 @@ class ConnectionException : Exception {
 	}
 }
 
-alias void function(Socket) CMT;
+alias void delegate(Socket) CMT;
 
 import core.thread;
 /+
@@ -4824,6 +5017,9 @@ class ConnectionThread : Thread {
 					}
 				}
 				+/
+			} catch(ConnectionClosedException e) {
+				// can just ignore this, it is fairly normal
+				socket.close();
 			} catch(Throwable e) {
 				import std.stdio; stderr.rawWrite(e.toString); stderr.rawWrite("\n");
 				socket.close();
@@ -5154,6 +5350,9 @@ version(cgi_with_websocket) {
 
 		private this(Cgi cgi) {
 			this.cgi = cgi;
+
+			Socket socket = cgi.idlol.source;
+			socket.setOption(SocketOptionLevel.SOCKET, SocketOption.RCVTIMEO, dur!"minutes"(5));
 		}
 
 		// returns true if data available, false if it timed out
@@ -5354,7 +5553,8 @@ version(cgi_with_websocket) {
 					return m;
 			} while(lowLevelReceive());
 
-			return WebSocketFrame.init; // FIXME? maybe.
+			throw new ConnectionClosedException("Websocket receive timed out");
+			//return WebSocketFrame.init; // FIXME? maybe.
 		}
 
 		/++
@@ -5462,6 +5662,11 @@ version(cgi_with_websocket) {
 				}
 			}
 
+			// the recv thing can be invalidated so gotta copy it over ugh
+			if(d.length) {
+				m.data = m.data.dup();
+			}
+
 			import core.stdc.string;
 			memmove(receiveBuffer.ptr, d.ptr, d.length);
 			receiveBufferUsedLength = d.length;
@@ -5775,11 +5980,13 @@ version(Posix) {
 
 // template for laziness
 void startAddonServer()(string arg) {
-	version(linux) {
+	version(OSX) {
+		assert(0, "Not implemented");
+	} else version(linux) {
 		import core.sys.posix.unistd;
 		pid_t pid;
 		const(char)*[16] args;
-		args[0] = "ARSD_CGI_WEBSOCKET_SERVER";
+		args[0] = "ARSD_CGI_ADDON_SERVER";
 		args[1] = arg.ptr;
 		posix_spawn(&pid, "/proc/self/exe",
 			null,
@@ -6227,6 +6434,7 @@ unittest {
 interface SessionObject {}
 
 private immutable void delegate(string[])[string] scheduledJobHandlers;
+private immutable void delegate(string[])[string] websocketServers;
 
 version(with_breaking_cgi_features)
 mixin(q{
@@ -6252,6 +6460,13 @@ mixin template ImplementRpcClientInterface(T, string serverPath, string cmdArg)
 
 			version(Posix) {{
 				auto ret = send(connectionHandle, sendable.ptr, sendable.length, 0);
+
+				if(ret == -1) {
+					throw new Exception("send returned -1, errno: " ~ to!string(errno));
+				} else if(ret == 0) {
+					throw new Exception("Connection to addon server lost");
+				} if(ret < sendable.length)
+					throw new Exception("Send failed to send all");
 				assert(ret == sendable.length);
 			}} // FIXME Windows impl
 
@@ -6307,6 +6522,7 @@ void dispatchRpcServer(Interface, Class)(Class this_, ubyte[] data, int fd) if(i
 
 	int dataLocation;
 	ubyte[] grab(int sz) {
+		if(sz == 0) assert(0);
 		auto d = data[dataLocation .. dataLocation + sz];
 		dataLocation += sz;
 		return d;
@@ -6340,7 +6556,13 @@ void dispatchRpcServer(Interface, Class)(Class this_, ubyte[] data, int fd) if(i
 
 					version(Posix) {
 						auto r = send(fd, sendable.ptr, sendable.length, 0);
-						assert(r == sendable.length);
+						if(r == -1) {
+							throw new Exception("send returned -1, errno: " ~ to!string(errno));
+						} else if(r == 0) {
+							throw new Exception("Connection to addon client lost");
+						} if(r < sendable.length)
+							throw new Exception("Send failed to send all");
+
 					} // FIXME Windows impl
 				}
 			break sw;
@@ -6809,7 +7031,9 @@ final class ScheduledJobServerImplementation : ScheduledJobServer, EventIoServer
 			if(fd == -1)
 				throw new Exception("fd timer create failed");
 
-			auto job = Job(executable, func, args, fd, nj);
+			foreach(ref arg; args)
+				arg = arg.idup;
+			auto job = Job(executable.idup, func.idup, .dup(args), fd, nj);
 
 			itimerspec value;
 			value.it_value.tv_sec = when;
@@ -6823,8 +7047,11 @@ final class ScheduledJobServerImplementation : ScheduledJobServer, EventIoServer
 
 			auto op = allocateIoOp(fd, IoOp.Read, 16, (IoOp* op, int fd) {
 				jobs.remove(nj);
+				epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fd, null);
+				close(fd);
+
 
-				spawnProcess([job.executable, "--timed-job", job.func] ~ args);
+				spawnProcess([job.executable, "--timed-job", job.func] ~ job.args);
 
 				return true;
 			});
@@ -6848,6 +7075,8 @@ final class ScheduledJobServerImplementation : ScheduledJobServer, EventIoServer
 			if(job is null)
 				return;
 
+			jobs.remove(jobId);
+
 			version(linux) {
 				import core.sys.linux.timerfd;
 				import core.sys.linux.epoll;
@@ -7246,6 +7475,13 @@ void runAddonServer(EIS)(string localListenerName, EIS eis) if(is(EIS : EventIoS
 		import core.sys.posix.signal;
 		signal(SIGPIPE, SIG_IGN);
 
+		static extern(C) void sigchldhandler(int) {
+			int status;
+			import w = core.sys.posix.sys.wait;
+			w.wait(&status);
+		}
+		signal(SIGCHLD, &sigchldhandler);
+
 		int sock = socket(AF_UNIX, SOCK_STREAM, 0);
 		if(sock == -1)
 			throw new Exception("socket " ~ to!string(errno));
@@ -7961,6 +8197,16 @@ private bool hasIfCalledFromWeb(attrs...)() {
 	return false;
 }
 
+/++
+	Implies POST path for the thing itself, then GET will get the automatic form.
+
+	The given customizer, if present, will be called as a filter on the Form object.
+
+	History:
+		Added December 27, 2020
++/
+template AutomaticForm(alias customizer) { }
+
 /+
 	Argument conversions: for the most part, it is to!Thing(string).
 
@@ -8094,6 +8340,10 @@ q"css
 				padding: 0px;
 			}
 
+			dl.automatic-data-display {
+
+			}
+
 			.automatic-form {
 				max-width: 600px;
 			}
@@ -8109,6 +8359,10 @@ q"css
 				margin-left: -0.5em;
 			}
 
+			.submit-button-holder {
+				padding-left: 2em;
+			}
+
 			.add-array-button {
 
 			}
@@ -8194,12 +8448,12 @@ html", true, true);
 	/// typeof(null) (which is also used to represent functions returning `void`) do nothing
 	/// in the default presenter - allowing the function to have full low-level control over the
 	/// response.
-	void presentSuccessfulReturn(T : typeof(null))(Cgi cgi, T ret, typeof(null) meta, string format) {
+	void presentSuccessfulReturn(T : typeof(null), Meta)(Cgi cgi, T ret, Meta meta, string format) {
 		// nothing intentionally!
 	}
 
 	/// Redirections are forwarded to [Cgi.setResponseLocation]
-	void presentSuccessfulReturn(T : Redirection)(Cgi cgi, T ret, typeof(null) meta, string format) {
+	void presentSuccessfulReturn(T : Redirection, Meta)(Cgi cgi, T ret, Meta meta, string format) {
 		cgi.setResponseLocation(ret.to, true, getHttpCodeText(ret.code));
 	}
 
@@ -8218,7 +8472,7 @@ html", true, true);
 	}
 
 	/// An instance of the [arsd.dom.FileResource] interface has its own content type; assume it is a download of some sort.
-	void presentSuccessfulReturn(T : FileResource)(Cgi cgi, T ret, typeof(null) meta, string format) {
+	void presentSuccessfulReturn(T : FileResource, Meta)(Cgi cgi, T ret, Meta meta, string format) {
 		cgi.setCache(true); // not necessarily true but meh
 		cgi.setResponseContentType(ret.contentType);
 		cgi.write(ret.getData(), true);
@@ -8241,10 +8495,15 @@ html", true, true);
 		useful forms or richer error messages for the user.
 	+/
 	void presentExceptionAsHtml(alias func, T)(Cgi cgi, Throwable t, T dg) {
+		presentExceptionAsHtmlImpl(cgi, t, createAutomaticFormForFunction!(func)(dg));
+	}
+
+	void presentExceptionAsHtmlImpl(Cgi cgi, Throwable t, Form automaticForm) {
 		if(auto mae = cast(MissingArgumentException) t) {
 			auto container = this.htmlContainer();
-			container.appendChild(Element.make("p", "Argument `" ~ mae.argumentName ~ "` of type `" ~ mae.argumentType ~ "` is missing"));
-			container.appendChild(createAutomaticFormForFunction!(func)(dg));
+			if(cgi.requestMethod == Cgi.RequestMethod.POST)
+				container.appendChild(Element.make("p", "Argument `" ~ mae.argumentName ~ "` of type `" ~ mae.argumentType ~ "` is missing"));
+			container.appendChild(automaticForm);
 
 			cgi.write(container.parentDocument.toString(), true);
 		} else {
@@ -8394,6 +8653,8 @@ html", true, true);
 
 		auto form = cast(Form) Element.make("form");
 
+		form.method = "POST"; // FIXME
+
 		form.addClass("automatic-form");
 
 		string formDisplayName = beautify(__traits(identifier, method));
@@ -8483,7 +8744,7 @@ html", true, true);
 			return Element.make("span", to!string(t), "automatic-data-display");
 		} else static if(is(T == V[K], K, V)) {
 			auto dl = Element.make("dl");
-			dl.addClass("automatic-data-display");
+			dl.addClass("automatic-data-display associative-array");
 			foreach(k, v; t) {
 				dl.addChild("dt", to!string(k));
 				dl.addChild("dd", formatReturnValueAsHtml(v));
@@ -8491,12 +8752,12 @@ html", true, true);
 			return dl;
 		} else static if(is(T == struct)) {
 			auto dl = Element.make("dl");
-			dl.addClass("automatic-data-display");
+			dl.addClass("automatic-data-display struct");
 
 			foreach(idx, memberName; __traits(allMembers, T))
 			static if(__traits(compiles, __traits(getMember, T, memberName).offsetof)) {
-				dl.addChild("dt", memberName);
-				dl.addChild("dt", formatReturnValueAsHtml(__traits(getMember, t, memberName)));
+				dl.addChild("dt", beautify(memberName));
+				dl.addChild("dd", formatReturnValueAsHtml(__traits(getMember, t, memberName)));
 			}
 
 			return dl;
@@ -8836,7 +9097,7 @@ private auto serveApiInternal(T)(string urlPrefix) {
 			static if(is(typeof(overload) R == return))
 			static if(__traits(getProtection, overload) == "public" || __traits(getProtection, overload) == "export")
 			{
-			static foreach(urlNameForMethod; urlNamesForMethod!(overload)(urlify(methodName)))
+			static foreach(urlNameForMethod; urlNamesForMethod!(overload, urlify(methodName)))
 			case urlNameForMethod:
 
 				static if(is(R : WebObject)) {
@@ -8855,6 +9116,8 @@ private auto serveApiInternal(T)(string urlPrefix) {
 						static if(is(param : Cgi)) {
 							static assert(!is(param == immutable));
 							cast() params[pidx] = cgi;
+						} else static if(is(param == typeof(presenter))) {
+							cast() param[pidx] = presenter;
 						} else static if(is(param == Session!D, D)) {
 							static assert(!is(param == immutable));
 							cast() params[pidx] = cgi.getSessionObject!D();
@@ -8893,12 +9156,16 @@ private auto serveApiInternal(T)(string urlPrefix) {
 					if(remainingUrl.length)
 						return false;
 
+					bool automaticForm;
+
 					foreach(attr; __traits(getAttributes, overload))
 						static if(is(attr == AddTrailingSlash)) {
 							if(remainingUrl is null) {
 								cgi.setResponseLocation(cgi.pathInfo ~ "/");
 								return true;
 							}
+						} else static if(__traits(isSame, AutomaticForm, attr)) {
+							automaticForm = true;
 						}
 				
 				/+
@@ -8974,10 +9241,18 @@ private auto serveApiInternal(T)(string urlPrefix) {
 					+/
 					if(callFunction)
 				+/
+
+					if(automaticForm && cgi.requestMethod == Cgi.RequestMethod.GET) {
+						// Should I still show the form on a json thing? idk...
+						auto ret = presenter.createAutomaticFormForFunction!((__traits(getOverloads, obj, methodName)[idx]))(&(__traits(getOverloads, obj, methodName)[idx]));
+						presenter.presentSuccessfulReturn(cgi, ret, presenter.methodMeta!(__traits(getOverloads, obj, methodName)[idx]), "html");
+						return true;
+					}
 					switch(cgi.request("format", defaultFormat!overload())) {
 						case "html":
 							// a void return (or typeof(null) lol) means you, the user, is doing it yourself. Gives full control.
 							try {
+
 								auto ret = callFromCgi!(__traits(getOverloads, obj, methodName)[idx])(&(__traits(getOverloads, obj, methodName)[idx]), cgi);
 								presenter.presentSuccessfulReturn(cgi, ret, presenter.methodMeta!(__traits(getOverloads, obj, methodName)[idx]), "html");
 							} catch(Throwable t) {
@@ -9016,12 +9291,12 @@ private auto serveApiInternal(T)(string urlPrefix) {
 				}
 			}
 			}
-			case "script.js":
+			case "GET script.js":
 				cgi.setResponseContentType("text/javascript");
 				cgi.gzipResponse = true;
 				cgi.write(presenter.script(), true);
 				return true;
-			case "style.css":
+			case "GET style.css":
 				cgi.setResponseContentType("text/css");
 				cgi.gzipResponse = true;
 				cgi.write(presenter.style(), true);
@@ -9051,41 +9326,57 @@ struct Paginated(T) {
 	string nextPageUrl;
 }
 
-string[] urlNamesForMethod(alias method)(string def) {
-	auto verb = Cgi.RequestMethod.GET;
-	bool foundVerb = false;
-	bool foundNoun = false;
-	foreach(attr; __traits(getAttributes, method)) {
-		static if(is(typeof(attr) == Cgi.RequestMethod)) {
-			verb = attr;
-			if(foundVerb)
-				assert(0, "Multiple http verbs on one function is not currently supported");
-			foundVerb = true;
-		}
-		static if(is(typeof(attr) == UrlName)) {
-			if(foundNoun)
-				assert(0, "Multiple url names on one function is not currently supported");
-			foundNoun = true;
-			def = attr.name;
-		}
-	}
+template urlNamesForMethod(alias method, string default_) {
+	string[] helper() {
+		auto verb = Cgi.RequestMethod.GET;
+		bool foundVerb = false;
+		bool foundNoun = false;
 
-	if(def is null)
-		def = "__null";
+		string def = default_;
 
-	string[] ret;
+		bool hasAutomaticForm = false;
 
-	static if(is(typeof(method) R == return)) {
-		static if(is(R : WebObject)) {
-			def ~= "/";
-			foreach(v; __traits(allMembers, Cgi.RequestMethod))
-				ret ~= v ~ " " ~ def;
-		} else {
-			ret ~= to!string(verb) ~ " " ~ def;
+		foreach(attr; __traits(getAttributes, method)) {
+			static if(is(typeof(attr) == Cgi.RequestMethod)) {
+				verb = attr;
+				if(foundVerb)
+					assert(0, "Multiple http verbs on one function is not currently supported");
+				foundVerb = true;
+			}
+			static if(is(typeof(attr) == UrlName)) {
+				if(foundNoun)
+					assert(0, "Multiple url names on one function is not currently supported");
+				foundNoun = true;
+				def = attr.name;
+			}
+			static if(__traits(isSame, attr, AutomaticForm)) {
+				hasAutomaticForm = true;
+			}
 		}
-	} else static assert(0);
 
-	return ret;
+		if(def is null)
+			def = "__null";
+
+		string[] ret;
+
+		static if(is(typeof(method) R == return)) {
+			static if(is(R : WebObject)) {
+				def ~= "/";
+				foreach(v; __traits(allMembers, Cgi.RequestMethod))
+					ret ~= v ~ " " ~ def;
+			} else {
+				if(hasAutomaticForm) {
+					ret ~= "GET " ~ def;
+					ret ~= "POST " ~ def;
+				} else {
+					ret ~= to!string(verb) ~ " " ~ def;
+				}
+			}
+		} else static assert(0);
+
+		return ret;
+	}
+	enum urlNamesForMethod = helper();
 }
 
 
@@ -9760,40 +10051,6 @@ auto serveStaticFileDirectory(string urlPrefix, string directory = null) {
 	return DispatcherDefinition!(internalHandler, DispatcherDetails)(urlPrefix, false, DispatcherDetails(directory));
 }
 
-private static string getHttpCodeText(int code) pure nothrow @nogc {
-	switch(code) {
-		case 200: return "200 OK";
-		case 201: return "201 Created";
-		case 202: return "202 Accepted";
-		case 203: return "203 Non-Authoritative Information";
-		case 204: return "204 No Content";
-		case 205: return "205 Reset Content";
-		//
-		case 300: return "300 300 Multiple Choices";
-		case 301: return "301 Moved Permanently";
-		case 302: return "302 Found";
-		case 303: return "303 See Other";
-		case 307: return "307 Temporary Redirect";
-		case 308: return "308 Permanent Redirect";
-		//
-		// FIXME: add more common 400 ones cgi.d might return too
-		case 400: return "400 Bad Request";
-		case 403: return "403 Forbidden";
-		case 404: return "404 Not Found";
-		case 405: return "405 Method Not Allowed";
-		case 406: return "406 Not Acceptable";
-		case 409: return "409 Conflict";
-		case 410: return "410 Gone";
-		//
-		case 500: return "500 Internal Server Error";
-		case 501: return "501 Not Implemented";
-		case 502: return "502 Bad Gateway";
-		case 503: return "503 Service Unavailable";
-		//
-		default: assert(0, "Unsupported http code");
-	}
-}
-
 /++
 	Redirects one url to another
 
@@ -9989,6 +10246,42 @@ private struct StackBuffer {
 	}
 }
 
+// duplicated in http2.d
+private static string getHttpCodeText(int code) pure nothrow @nogc {
+	switch(code) {
+		case 200: return "200 OK";
+		case 201: return "201 Created";
+		case 202: return "202 Accepted";
+		case 203: return "203 Non-Authoritative Information";
+		case 204: return "204 No Content";
+		case 205: return "205 Reset Content";
+		//
+		case 300: return "300 Multiple Choices";
+		case 301: return "301 Moved Permanently";
+		case 302: return "302 Found";
+		case 303: return "303 See Other";
+		case 307: return "307 Temporary Redirect";
+		case 308: return "308 Permanent Redirect";
+		//
+		// FIXME: add more common 400 ones cgi.d might return too
+		case 400: return "400 Bad Request";
+		case 403: return "403 Forbidden";
+		case 404: return "404 Not Found";
+		case 405: return "405 Method Not Allowed";
+		case 406: return "406 Not Acceptable";
+		case 409: return "409 Conflict";
+		case 410: return "410 Gone";
+		//
+		case 500: return "500 Internal Server Error";
+		case 501: return "501 Not Implemented";
+		case 502: return "502 Bad Gateway";
+		case 503: return "503 Service Unavailable";
+		//
+		default: assert(0, "Unsupported http code");
+	}
+}
+
+
 /+
 /++
 	This is the beginnings of my web.d 2.0 - it dispatches web requests to a class object.
@@ -10011,11 +10304,11 @@ bool apiDispatcher()(Cgi cgi) {
 }
 +/
 /*
-Copyright: Adam D. Ruppe, 2008 - 2020
+Copyright: Adam D. Ruppe, 2008 - 2021
 License:   [http://www.boost.org/LICENSE_1_0.txt|Boost License 1.0].
 Authors: Adam D. Ruppe
 
-	Copyright Adam D. Ruppe 2008 - 2020.
+	Copyright Adam D. Ruppe 2008 - 2021.
 Distributed under the Boost Software License, Version 1.0.
    (See accompanying file LICENSE_1_0.txt or copy at
 	http://www.boost.org/LICENSE_1_0.txt)