From 843bccdd421d199b0859a8e1e753bc56ed727ce8 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Tue, 14 Jul 2020 23:35:09 +0200
Subject: [PATCH 01/70] start implementing the messages for connecting to pvpgn
 servers

---
 CMakeLists.txt                 |   1 +
 src/network/online_service.cpp | 232 +++++++++++++++++++++++++++++++++
 2 files changed, 233 insertions(+)
 create mode 100644 src/network/online_service.cpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index aafba74d8..a9ebc8a64 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -236,6 +236,7 @@ set(network_SRCS
 	src/network/netconnect.cpp
 	src/network/network.cpp
 	src/network/netsockets.cpp
+        src/network/online_service.cpp
 )
 source_group(network FILES ${network_SRCS})
 
diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
new file mode 100644
index 000000000..b7f17325c
--- /dev/null
+++ b/src/network/online_service.cpp
@@ -0,0 +1,232 @@
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <ostream>
+#include <unistd.h>
+
+#ifdef USE_WIN32
+#include <winsock2.h>
+#else
+#include <netinet/in.h>
+#endif
+
+#include "assert.h"
+
+#include "network/netsockets.h"
+
+class BNCSBuf {
+public:
+    BNCSBuf(uint8_t id) {
+        // Every BNCS message has the same header:
+        //  (UINT8) Always 0xFF
+        //  (UINT8) Message ID
+        // (UINT16) Message length, including this header
+        //   (VOID) Message data
+        this->sz = 16;
+        this->buf = (uint8_t*) calloc(sizeof(uint8_t), this->sz);
+        this->pos = 0;
+        serialize8(0xff);
+        serialize8(id);
+        this->length_pos = pos;
+        serialize16((uint16_t)0);
+    };
+    ~BNCSBuf() {
+        free(buf);
+    };
+
+    void serialize32(uint32_t data) {
+        ensureSpace(sizeof(data));
+        uint32_t *view = reinterpret_cast<uint32_t *>(buf + pos);
+        *view = htonl(data);
+        pos += sizeof(data);
+    };
+    void serialize32(int32_t data) {
+        ensureSpace(sizeof(data));
+        int32_t *view = reinterpret_cast<int32_t *>(buf + pos);
+        *view = htonl(data);
+        pos += sizeof(data);
+    };
+    void serialize16(uint16_t data) {
+        ensureSpace(sizeof(data));
+        uint16_t *view = reinterpret_cast<uint16_t *>(buf + pos);
+        *view = htons(data);
+        pos += sizeof(data);
+    };
+    void serialize16(int16_t data) {
+        ensureSpace(sizeof(data));
+        uint16_t *view = reinterpret_cast<uint16_t *>(buf + pos);
+        *view = htons(data);
+        pos += sizeof(data);
+    };
+    void serialize8(uint8_t data) {
+        ensureSpace(sizeof(data));
+        *(buf + pos) = data;
+        pos++;
+    };
+    void serialize(const char* str, int len) {
+        ensureSpace(len);
+        memcpy(buf + pos, str, len);
+        pos += len;
+    };
+    void serialize(const char* str) {
+        int len = strlen(str);
+        ensureSpace(len);
+        memcpy(buf + pos, str, len);
+        pos += len;
+    };
+    uint8_t *getBuffer() {
+        // insert length to make it a valid buffer
+        uint16_t *view = reinterpret_cast<uint16_t *>(buf + length_pos);
+        *view = htons(pos);
+        return buf;
+    };
+    int getLength() {
+        return pos;
+    };
+private:
+    void ensureSpace(size_t required) {
+        if (pos + required < sz) {
+            sz = sz * 2;
+            buf = (uint8_t*) realloc(buf, sz);
+            assert(buf != NULL);
+        }
+    }
+
+    uint8_t *buf;
+    int sz;
+    int pos;
+    int length_pos;
+};
+
+class Context;
+class NetworkState {
+public:
+    virtual ~NetworkState() {};
+    virtual void doOneStep(Context *ctx) = 0;
+
+protected:
+    int send(Context *ctx, BNCSBuf *buf);
+};
+
+class Context {
+public:
+    Context() {
+        this->udpSocket = new CUDPSocket();
+        this->tcpSocket = new CTCPSocket();
+        this->state = NULL;
+    }
+
+    ~Context() {
+        if (state != NULL) {
+            delete state;
+        }
+        delete udpSocket;
+        delete tcpSocket;
+    }
+
+    CUDPSocket *getUDPSocket() { return udpSocket; }
+
+    CTCPSocket *getTCPSocket() { return tcpSocket; }
+
+    void doOneStep() { this->state->doOneStep(this); }
+
+    void setState(NetworkState* newState) {
+        assert (newState != this->state);
+        if (this->state != NULL) {
+            delete this->state;
+        }
+        this->state = newState;
+    }
+
+private:
+    NetworkState *state;
+    CUDPSocket *udpSocket;
+    CTCPSocket *tcpSocket;
+};
+
+int NetworkState::send(Context *ctx, BNCSBuf *buf) {
+    return ctx->getTCPSocket()->Send(buf->getBuffer(), buf->getLength());
+}
+
+class DisconnectedState : public NetworkState {
+public:
+    DisconnectedState(std::string message) {
+        this->message = message;
+        this->hasPrinted = false;
+    };
+
+    virtual void doOneStep(Context *ctx) {
+        if (!hasPrinted) {
+            std::cout << message << std::endl;
+            hasPrinted = true;
+        }
+        // the end
+    }
+
+private:
+    bool hasPrinted;
+    std::string message;
+};
+
+class C2S_SID_AUTH_INFO : public NetworkState {
+    virtual void doOneStep(Context *ctx) {
+        BNCSBuf buffer(0x50);
+        // (UINT32) Protocol ID
+        buffer.serialize32(0x00);
+        // (UINT32) Platform code
+        buffer.serialize("IX86", 4);
+        // (UINT32) Product code
+        buffer.serialize("W2BN", 4);
+        // (UINT32) Version byte
+        buffer.serialize32(0x4f);
+        // (UINT32) Language code
+        buffer.serialize32(0x00);
+        // (UINT32) Local IP
+        buffer.serialize32(0x00);
+        // (UINT32) Time zone bias
+        buffer.serialize32(0x00);
+        // (UINT32) MPQ locale ID
+        buffer.serialize32(0x00);
+        // (UINT32) User language ID
+        buffer.serialize32(0x00);
+        // (STRING) Country abbreviation
+        buffer.serialize("USA");
+        // (STRING) Country
+        buffer.serialize("United States");
+
+        send(ctx, &buffer);
+        // ctx->setState(new S2C_PING());
+    }
+};
+
+class ProtoByteState : public NetworkState {
+    virtual void doOneStep(Context *ctx) {
+        ctx->getTCPSocket()->Send("\x1", 1); // use game protocol
+        ctx->setState(new C2S_SID_AUTH_INFO());
+    }
+};
+
+class ConnectState : public NetworkState {
+    virtual void doOneStep(Context *ctx) {
+        CHost host("127.0.0.1", 6112); // TODO: parameterize
+        ctx->getTCPSocket()->Open(host);
+        if (ctx->getTCPSocket()->IsValid() == false) {
+            ctx->setState(new DisconnectedState("TCP open failed"));
+            return;
+        }
+        if (ctx->getTCPSocket()->Connect(host) == false) {
+            ctx->setState(new DisconnectedState("TCP connect failed"));
+            return;
+        }
+        ctx->setState(new ProtoByteState());
+    }
+};
+
+static void goOnline() {
+    Context *ctx = new Context();
+    ctx->setState(new ConnectState());
+    while (true) {
+        ctx->doOneStep();
+        sleep(1);
+    }
+}

From cb0e517ff56a6b5cd8eb0ba18a84d42b2de489f7 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Wed, 22 Jul 2020 15:23:51 +0200
Subject: [PATCH 02/70] a few more messages, up to entering chat now

---
 src/network/online_service.cpp | 556 +++++++++++++++++++++++++++++++--
 1 file changed, 538 insertions(+), 18 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index b7f17325c..a726f770e 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1,8 +1,13 @@
+#include <clocale>
+#include <cstdio>
 #include <cstdlib>
 #include <cstring>
 #include <iostream>
 #include <ostream>
+#include <sstream>
+#include <string>
 #include <unistd.h>
+#include <vector>
 
 #ifdef USE_WIN32
 #include <winsock2.h>
@@ -11,12 +16,139 @@
 #endif
 
 #include "assert.h"
-
+#include "network.h"
 #include "network/netsockets.h"
+#include "util.h"
+#include "version.h"
 
-class BNCSBuf {
+#include "./xsha1.h"
+
+class BNCSInputStream {
 public:
-    BNCSBuf(uint8_t id) {
+    BNCSInputStream(CTCPSocket *socket) {
+        this->sock = socket;
+        this->bufsize = 1024;
+        this->buffer = (char*)calloc(sizeof(char), bufsize);
+        this->avail = 0;
+        this->pos = 0;
+    };
+    ~BNCSInputStream() {};
+
+    std::string readString() {
+        if (avail == 0) {
+            return NULL;
+        }
+        std::stringstream strstr;
+        int i = pos;
+        char c;
+        while ((c = buffer[i]) != '\0' && i < avail) {
+            strstr.put(c);
+            i += 1;
+        }
+        consumeData(i);
+        return strstr.str();
+    };
+
+    std::vector<std::string> readStringlist() {
+        std::vector<std::string> stringlist;
+        while (true) {
+            std::string nxt = readString();
+            if (nxt.empty()) {
+                break;
+            } else {
+                stringlist.push_back(nxt);
+            }
+        }
+        return stringlist;
+    };
+
+    uint8_t read8() {
+        uint8_t byte = buffer[pos];
+        consumeData(1);
+        return byte;
+    }
+
+    uint16_t read16() {
+        uint16_t byte = ntohs(reinterpret_cast<uint16_t *>(buffer + pos)[0]);
+        consumeData(2);
+        return byte;
+    }
+
+    uint32_t read32() {
+        uint32_t byte = ntohs(reinterpret_cast<uint32_t *>(buffer + pos)[0]);
+        consumeData(4);
+        return byte;
+    }
+
+    uint64_t read64() {
+        uint64_t byte = ntohs(reinterpret_cast<uint64_t *>(buffer + pos)[0]);
+        consumeData(8);
+        return byte;
+    }
+
+    bool readBool8() {
+        return read8() != 0;
+    }
+
+    bool readBool32() {
+        return read32() != 0;
+    }
+
+    uint64_t readFiletime() {
+        return read64();
+    }
+
+    std::string string32() {
+        // uint32 encoded (4-byte) string
+        uint32_t data = read32();
+        char dt[5];
+        strncpy(dt, (const char*)&data, 4);
+        dt[4] = '\0';
+        return std::string(dt);
+    };
+
+    /**
+     * To be called at the start of a message, gets the entire data into memory or returns -1.
+     */
+    uint8_t readMessageId() {
+        // Every BNCS message has the same header:
+        //  (UINT8) Always 0xFF
+        //  (UINT8) Message ID
+        // (UINT16) Message length, including this header
+        //   (VOID) Message data
+        avail += this->sock->Recv(buffer + avail, 4 - avail);
+        if (avail < 4) {
+            return -1;
+        }
+        assert(read8() == 0xff);
+        uint8_t msgId = read8();
+        uint16_t len = read16();
+        avail += this->sock->Recv(buffer + avail, len - 4);
+        if (avail < len) {
+            // Didn't receive full message on the socket, yet. Reset position so
+            // this method can be used to try again
+            pos = 0;
+            return -1;
+        } else {
+            return 0;
+        }
+    };
+
+private:
+    void consumeData(int bytes) {
+        pos += bytes;
+    }
+
+    CTCPSocket *sock;
+    char *buffer;
+    int avail;
+    int pos;
+    int bufsize;
+};
+
+class BNCSOutputStream {
+public:
+    BNCSOutputStream(uint8_t id) {
         // Every BNCS message has the same header:
         //  (UINT8) Always 0xFF
         //  (UINT8) Message ID
@@ -30,7 +162,7 @@ public:
         this->length_pos = pos;
         serialize16((uint16_t)0);
     };
-    ~BNCSBuf() {
+    ~BNCSOutputStream() {
         free(buf);
     };
 
@@ -74,16 +206,23 @@ public:
         memcpy(buf + pos, str, len);
         pos += len;
     };
+
+    int flush(CTCPSocket *sock) {
+        return sock->Send(getBuffer(), pos);
+    };
+
+    void flush(CUDPSocket *sock, CHost *host) {
+        sock->Send(*host, getBuffer(), pos);
+    };
+
+private:
     uint8_t *getBuffer() {
         // insert length to make it a valid buffer
         uint16_t *view = reinterpret_cast<uint16_t *>(buf + length_pos);
         *view = htons(pos);
         return buf;
     };
-    int getLength() {
-        return pos;
-    };
-private:
+
     void ensureSpace(size_t required) {
         if (pos + required < sz) {
             sz = sz * 2;
@@ -105,7 +244,7 @@ public:
     virtual void doOneStep(Context *ctx) = 0;
 
 protected:
-    int send(Context *ctx, BNCSBuf *buf);
+    int send(Context *ctx, BNCSOutputStream *buf);
 };
 
 class Context {
@@ -113,7 +252,13 @@ public:
     Context() {
         this->udpSocket = new CUDPSocket();
         this->tcpSocket = new CTCPSocket();
+        this->istream = new BNCSInputStream(tcpSocket);
         this->state = NULL;
+        this->host = new CHost("127.0.0.1", 6112); // TODO: parameterize
+        this->clientToken = MyRand();
+        this->username = "";
+        this->password = "";
+        this->info = "";
     }
 
     ~Context() {
@@ -122,12 +267,36 @@ public:
         }
         delete udpSocket;
         delete tcpSocket;
+        delete host;
+    }
+
+    std::string getInfo() { return info; }
+
+    void setInfo(std::string arg) { info = arg; }
+
+    std::string getUsername() { return username; }
+
+    void setUsername(std::string arg) { username = arg; }
+
+    std::string getPassword() { return password; }
+
+    void setPassword(std::string arg) { password = arg; }
+
+    CHost *getHost() { return host; }
+
+    void setHost(CHost *arg) {
+        if (host != NULL) {
+            delete host;
+        }
+        host = arg;
     }
 
     CUDPSocket *getUDPSocket() { return udpSocket; }
 
     CTCPSocket *getTCPSocket() { return tcpSocket; }
 
+    BNCSInputStream *getMsgIStream() { return istream; }
+
     void doOneStep() { this->state->doOneStep(this); }
 
     void setState(NetworkState* newState) {
@@ -138,14 +307,23 @@ public:
         this->state = newState;
     }
 
+    uint32_t clientToken;
+    uint32_t serverToken;
+
 private:
     NetworkState *state;
+    CHost *host;
     CUDPSocket *udpSocket;
     CTCPSocket *tcpSocket;
+    BNCSInputStream *istream;
+
+    std::string info;
+    std::string username;
+    std::string password;
 };
 
-int NetworkState::send(Context *ctx, BNCSBuf *buf) {
-    return ctx->getTCPSocket()->Send(buf->getBuffer(), buf->getLength());
+int NetworkState::send(Context *ctx, BNCSOutputStream *buf) {
+    return buf->flush(ctx->getTCPSocket());
 }
 
 class DisconnectedState : public NetworkState {
@@ -168,21 +346,364 @@ private:
     std::string message;
 };
 
+class IN_CHAT : public NetworkState {
+    virtual void doOneStep(Context *ctx);
+};
+
+class S2C_GETCHANNELLIST : public NetworkState {
+    virtual void doOneStep(Context *ctx) {
+        if (ctx->getTCPSocket()->HasDataToRead(0)) {
+            uint8_t msg = ctx->getMsgIStream()->readMessageId();
+            if (msg == 0xff) {
+                // try again next time
+                return;
+            }
+            if (msg != 0x0b) {
+                std::string error = std::string("Expected SID_GETCHANNELLIST, got msg id ");
+                error += std::to_string(msg);
+                ctx->setState(new DisconnectedState(error));
+            }
+
+            std::vector<std::string> channels = ctx->getMsgIStream()->readStringlist();
+
+            ctx->setState(new IN_CHAT());
+        }
+    }
+};
+
+class S2C_ENTERCHAT : public NetworkState {
+    virtual void doOneStep(Context *ctx) {
+        if (ctx->getTCPSocket()->HasDataToRead(0)) {
+            uint8_t msg = ctx->getMsgIStream()->readMessageId();
+            if (msg == 0xff) {
+                // try again next time
+                return;
+            }
+            if (msg != 0x0a) {
+                std::string error = std::string("Expected SID_ENTERCHAT, got msg id ");
+                error += std::to_string(msg);
+                ctx->setState(new DisconnectedState(error));
+            }
+
+            std::string uniqueName = ctx->getMsgIStream()->readString();
+            std::string statString = ctx->getMsgIStream()->readString();
+            std::string accountName = ctx->getMsgIStream()->readString();
+
+            ctx->setUsername(uniqueName);
+            if (!statString.empty()) {
+                ctx->setInfo("Statstring after logon: " + statString);
+            }
+
+            ctx->setState(new S2C_GETCHANNELLIST());
+        }
+    }
+};
+
+class C2S_ENTERCHAT : public NetworkState {
+    virtual void doOneStep(Context *ctx) {
+        // does all of enterchar, getchannellist, and first-join joinchannel
+        BNCSOutputStream enterchat(0x0a);
+        enterchat.serialize(ctx->getUsername().c_str());
+        enterchat.serialize("");
+        enterchat.flush(ctx->getTCPSocket());
+
+        BNCSOutputStream getlist(0x0b);
+        getlist.serialize32(0x00);
+        getlist.flush(ctx->getTCPSocket());
+
+        BNCSOutputStream join(0x0c);
+        join.serialize32(0x01); // first-join
+        join.serialize("ignored");
+        join.flush(ctx->getTCPSocket());
+
+        ctx->setState(new S2C_ENTERCHAT());
+    }
+};
+
+class S2C_PKT_SERVERPING : public NetworkState {
+public:
+    S2C_PKT_SERVERPING(std::string message) {
+        this->retries = 0;
+    };
+
+    virtual void doOneStep(Context *ctx) {
+        if (ctx->getUDPSocket()->HasDataToRead(1)) {
+            // PKT_SERVERPING
+            //  (UINT8) 0xFF
+            //  (UINT8) 0x05
+            // (UINT16) 8
+            // (UINT32) UDP Code
+            char buf[8];
+            int received = ctx->getUDPSocket()->Recv(buf, 8, ctx->getHost());
+            if (received == 8) {
+                uint32_t udpCode = reinterpret_cast<uint32_t*>(buf)[1];
+                BNCSOutputStream udppingresponse(0x14);
+                udppingresponse.serialize32(udpCode);
+                udppingresponse.flush(ctx->getTCPSocket());
+            }
+        } else {
+            retries++;
+            if (retries < 5000) {
+                return;
+            }
+            // we're using a timeout of 1ms, so now we've been waiting at
+            // the very least for 5 seconds... let's skip UDP then
+        }
+        ctx->setState(new C2S_ENTERCHAT());
+    }
+
+private:
+    int retries;
+};
+
+class C2S_LOGONRESPONSE2 : public NetworkState {
+    virtual void doOneStep(Context *ctx);
+};
+
+class S2C_LOGONRESPONSE2 : public NetworkState {
+    virtual void doOneStep(Context *ctx) {
+        if (ctx->getTCPSocket()->HasDataToRead(0)) {
+            uint8_t msg = ctx->getMsgIStream()->readMessageId();
+            if (msg == 0xff) {
+                // try again next time
+                return;
+            }
+            if (msg != 0x3a) {
+                std::string error = std::string("Expected SID_LOGONRESPONSE2, got msg id ");
+                error += std::to_string(msg);
+                ctx->setState(new DisconnectedState(error));
+            }
+
+            uint32_t status = ctx->getMsgIStream()->read32();
+
+            switch (status) {
+            case 0x00:
+                // success - TODO - channel list, join chat
+                return;
+            case 0x01:
+            case 0x02:
+                ctx->setInfo("Account does not exist or incorrect password");
+                ctx->setPassword("");
+                ctx->setState(new C2S_LOGONRESPONSE2());
+                return;
+            case 0x06:
+                ctx->setInfo("Account closed: " + ctx->getMsgIStream()->readString());
+                ctx->setPassword("");
+                ctx->setState(new C2S_LOGONRESPONSE2());
+                return;
+            default:
+                ctx->setState(new DisconnectedState("unknown logon response"));
+            }
+        }
+    }
+};
+
+void C2S_LOGONRESPONSE2::doOneStep(Context *ctx) {
+    std::string user = ctx->getUsername();
+    std::string pw = ctx->getPassword();
+    if (!user.empty() && !pw.empty()) {
+        // C2S SID_LOGONRESPONSE2
+        BNCSOutputStream logon(0x3a);
+        logon.serialize32(ctx->clientToken);
+        logon.serialize32(ctx->serverToken);
+
+        uint32_t pwhash[5];
+        xsha1_calcHashBuf(pw.c_str(), pw.length(), pwhash);
+        xsha1_calcHashDat(pwhash, pwhash);
+        // Uint8[20] password hash
+        for (int i = 0; i < 20; i++) {
+            logon.serialize8(reinterpret_cast<uint8_t*>(pwhash)[i]);
+        }
+        logon.serialize(user.c_str());
+        logon.flush(ctx->getTCPSocket());
+
+        ctx->setState(new S2C_LOGONRESPONSE2());
+    }
+};
+
+class S2C_SID_AUTH_CHECK : public NetworkState {
+    virtual void doOneStep(Context *ctx) {
+        if (ctx->getTCPSocket()->HasDataToRead(0)) {
+            uint8_t msg = ctx->getMsgIStream()->readMessageId();
+            if (msg == 0xff) {
+                // try again next time
+                return;
+            }
+            if (msg != 0x51) {
+                std::string error = std::string("Expected SID_AUTH_CHECK, got msg id ");
+                error += std::to_string(msg);
+                ctx->setState(new DisconnectedState(error));
+            }
+
+            uint32_t result = ctx->getMsgIStream()->read32();
+            std::string info = ctx->getMsgIStream()->readString();
+
+            switch (result) {
+            case 0x000:
+                // Passed challenge
+                ctx->setState(new C2S_LOGONRESPONSE2());
+                return;
+            case 0x100:
+                // Old game version
+                info = "Old game version: " + info;
+                break;
+            case 0x101:
+                // Invalid version
+                info = "Invalid version: " + info;
+                break;
+            case 0x102:
+                // Game version must be downgraded
+                info = "Game version must be downgraded: " + info;
+                break;
+            case 0x200:
+                // Invalid CD key
+                info = "Invalid CD Key: " + info;
+                break;
+            case 0x201:
+                // CD Key in use
+                info = "CD Key in use: " + info;
+                break;
+            case 0x202:
+                // Banned key
+                info = "Banned key: " + info;
+                break;
+            case 0x203:
+                // Wrong product
+                info = "Wrong product: " + info;
+                break;
+            default:
+                // Invalid version code
+                info = "Invalid version code: " + info;
+            }
+            ctx->setState(new DisconnectedState(info));
+        }
+    }
+};
+
+class S2C_SID_AUTH_INFO : public NetworkState {
+    virtual void doOneStep(Context *ctx) {
+        if (ctx->getTCPSocket()->HasDataToRead(0)) {
+            uint8_t msg = ctx->getMsgIStream()->readMessageId();
+            if (msg == 0xff) {
+                // try again next time
+                return;
+            }
+            if (msg != 0x50) {
+                std::string error = std::string("Expected SID_AUTH_INFO, got msg id ");
+                error += std::to_string(msg);
+                ctx->setState(new DisconnectedState(error));
+            }
+
+            uint32_t logonType = ctx->getMsgIStream()->read32();
+            assert(logonType == 0x00); // only support Broken SHA-1 logon for now
+            uint32_t serverToken = ctx->getMsgIStream()->read32();
+            ctx->serverToken = serverToken;
+            uint32_t udpValue = ctx->getMsgIStream()->read32();
+            uint64_t mpqFiletime = ctx->getMsgIStream()->readFiletime();
+            std::string mpqFilename = ctx->getMsgIStream()->readString();
+            std::string formula = ctx->getMsgIStream()->readString();
+
+            // immediately respond with pkt_conntest2 udp msg
+            BNCSOutputStream conntest(0x09);
+            conntest.serialize32(serverToken);
+            conntest.serialize32(udpValue);
+            conntest.flush(ctx->getUDPSocket(), ctx->getHost());
+
+            // immediately respond with SID_AUTH_CHECK
+            BNCSOutputStream check(0x51);
+            check.serialize32(ctx->clientToken);
+            // EXE version (one UINT32 value, serialized in network byte order here)
+            check.serialize8(StratagusPatchLevel2);
+            check.serialize8(StratagusPatchLevel);
+            check.serialize8(StratagusMinorVersion);
+            check.serialize8(StratagusMajorVersion);
+            // EXE hash - we use lua file checksums
+            check.serialize32(FileChecksums);
+            // no CD key, not a Spawn key
+            check.serialize32(0);
+            check.serialize32(0);
+            // EXE information
+            std::string exeInfo("stratagus.exe ");
+            exeInfo += StratagusLastModifiedDate;
+            exeInfo += " ";
+            exeInfo += StratagusLastModifiedTime;
+            exeInfo += " ";
+            exeInfo += std::to_string(StratagusVersion);
+            check.serialize(exeInfo.c_str());
+            // Key owner name
+            check.serialize(DESCRIPTION);
+            check.flush(ctx->getTCPSocket());
+
+            ctx->setState(new S2C_SID_AUTH_CHECK());
+        }
+    }
+};
+
+class S2C_SID_PING : public NetworkState {
+    virtual void doOneStep(Context *ctx) {
+        if (ctx->getTCPSocket()->HasDataToRead(0)) {
+            uint8_t msg = ctx->getMsgIStream()->readMessageId();
+            if (msg == 0xff) {
+                // try again next time
+                return;
+            }
+            if (msg != 0x25) {
+                // not a ping
+                std::string error = std::string("Expected SID_PING, got msg id ");
+                error += std::to_string(msg);
+                ctx->setState(new DisconnectedState(error));
+            }
+            uint32_t pingValue = ctx->getMsgIStream()->read32();
+
+            // immediately respond
+            BNCSOutputStream buffer(0x25);
+            buffer.serialize32(pingValue);
+            send(ctx, &buffer);
+
+            ctx->setState(new S2C_SID_AUTH_INFO());
+        }
+    }
+};
+
 class C2S_SID_AUTH_INFO : public NetworkState {
     virtual void doOneStep(Context *ctx) {
-        BNCSBuf buffer(0x50);
+        BNCSOutputStream buffer(0x50);
         // (UINT32) Protocol ID
         buffer.serialize32(0x00);
         // (UINT32) Platform code
         buffer.serialize("IX86", 4);
         // (UINT32) Product code
-        buffer.serialize("W2BN", 4);
+        buffer.serialize("WAGS", 4);
         // (UINT32) Version byte
         buffer.serialize32(0x4f);
         // (UINT32) Language code
         buffer.serialize32(0x00);
         // (UINT32) Local IP
-        buffer.serialize32(0x00);
+        if (CNetworkParameter::Instance.localHost.compare("127.0.0.1")) {
+            // user set a custom local ip, use that
+            struct in_addr addr;
+            inet_aton(CNetworkParameter::Instance.localHost.c_str(), &addr);
+            buffer.serialize32(addr.s_addr);
+        } else {
+            unsigned long ips[20];
+            int networkNumInterfaces = NetworkFildes.GetSocketAddresses(ips, 20);
+            bool found_one = false;
+            if (networkNumInterfaces) {
+                // we can only advertise one, so take the first that fits in 32-bit and is thus presumably ipv4
+                for (int i = 0; i < networkNumInterfaces; i++) {
+                    uint32_t ip = ips[i];
+                    if (ip == ips[i]) {
+                        found_one = true;
+                        buffer.serialize32(ip);
+                        break;
+                    }
+                }
+            }
+            if (!found_one) {
+                // use default
+                buffer.serialize32(0x00);
+            }
+        }
         // (UINT32) Time zone bias
         buffer.serialize32(0x00);
         // (UINT32) MPQ locale ID
@@ -195,7 +716,7 @@ class C2S_SID_AUTH_INFO : public NetworkState {
         buffer.serialize("United States");
 
         send(ctx, &buffer);
-        // ctx->setState(new S2C_PING());
+        ctx->setState(new S2C_SID_PING());
     }
 };
 
@@ -208,13 +729,12 @@ class ProtoByteState : public NetworkState {
 
 class ConnectState : public NetworkState {
     virtual void doOneStep(Context *ctx) {
-        CHost host("127.0.0.1", 6112); // TODO: parameterize
-        ctx->getTCPSocket()->Open(host);
+        ctx->getTCPSocket()->Open(*ctx->getHost());
         if (ctx->getTCPSocket()->IsValid() == false) {
             ctx->setState(new DisconnectedState("TCP open failed"));
             return;
         }
-        if (ctx->getTCPSocket()->Connect(host) == false) {
+        if (ctx->getTCPSocket()->Connect(*ctx->getHost()) == false) {
             ctx->setState(new DisconnectedState("TCP connect failed"));
             return;
         }

From 4c1227ee402dd5a332a904c3f008e5fb577305d2 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Sun, 26 Jul 2020 14:20:46 +0200
Subject: [PATCH 03/70] get into chat

---
 src/network/online_service.cpp | 443 +++++++++++++++++++++++++++++----
 1 file changed, 395 insertions(+), 48 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index a726f770e..dea395c0e 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -2,10 +2,12 @@
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <ctime>
 #include <iostream>
 #include <ostream>
 #include <sstream>
 #include <string>
+#include <tuple>
 #include <unistd.h>
 #include <vector>
 
@@ -15,6 +17,8 @@
 #include <netinet/in.h>
 #endif
 
+#include "game.h"
+#include "parameters.h"
 #include "assert.h"
 #include "network.h"
 #include "network/netsockets.h"
@@ -23,6 +27,7 @@
 
 #include "./xsha1.h"
 
+
 class BNCSInputStream {
 public:
     BNCSInputStream(CTCPSocket *socket) {
@@ -257,7 +262,6 @@ public:
         this->host = new CHost("127.0.0.1", 6112); // TODO: parameterize
         this->clientToken = MyRand();
         this->username = "";
-        this->password = "";
         this->info = "";
     }
 
@@ -278,9 +282,30 @@ public:
 
     void setUsername(std::string arg) { username = arg; }
 
-    std::string getPassword() { return password; }
+    uint32_t* getPassword2() {
+        // we assume that any valid password has at least 1 non-null word hash
+        for (int i = 0; i < 5; i++) {
+            if (password[i] != 0) {
+                return password2;
+            }
+        }
+        return NULL;
+    }
 
-    void setPassword(std::string arg) { password = arg; }
+    uint32_t* getPassword1() {
+        // we assume that any valid password has at least 1 non-null word hash
+        for (int i = 0; i < 5; i++) {
+            if (password[i] != 0) {
+                return password;
+            }
+        }
+        return NULL;
+    }
+
+    void setPassword(std::string pw) {
+        xsha1_calcHashBuf(pw.c_str(), pw.length(), password);
+        xsha1_calcHashDat(password, password2);
+    }
 
     CHost *getHost() { return host; }
 
@@ -319,7 +344,8 @@ private:
 
     std::string info;
     std::string username;
-    std::string password;
+    uint32_t password[5]; // xsha1 hash of password
+    uint32_t password2[5]; // xsha1 hash of password hash
 };
 
 int NetworkState::send(Context *ctx, BNCSOutputStream *buf) {
@@ -346,8 +372,289 @@ private:
     std::string message;
 };
 
-class IN_CHAT : public NetworkState {
-    virtual void doOneStep(Context *ctx);
+/* needed
+C>S 0x02 SID_STOPADV
+
+C>S 0x09 SID_GETADVLISTEX
+S>C 0x09 SID_GETADVLISTEX
+
+C>S 0x0E SID_CHATCOMMAND
+
+S>C 0x0F SID_CHATEVENT
+
+C>S 0x1C SID_STARTADVEX3
+S>C 0x1C SID_STARTADVEX3
+
+C>S 0x22 SID_NOTIFYJOIN
+
+C>S 0x3D SID_CREATEACCOUNT2
+S>C 0x3D SID_CREATEACCOUNT2
+
+C>S 0x2C SID_GAMERESULT
+
+C>S 0x2F SID_FINDLADDERUSER
+S>C 0x2F SID_FINDLADDERUSER
+
+C>S 0x26 SID_READUSERDATA
+S>C 0x26 SID_READUSERDATA
+
+C>S 0x31 SID_CHANGEPASSWORD
+S>C 0x31 SID_CHANGEPASSWORD
+
+C>S 0x5A SID_RESETPASSWORD
+
+C>S 0x65 SID_FRIENDSLIST
+S>C 0x65 SID_FRIENDSLIST
+
+*/
+
+class Game {
+public:
+    Game(uint32_t settings, uint16_t port, uint32_t host, uint32_t status, uint32_t time, std::string name, std::string pw, std::string stats) {
+        this->gameSettings = settings;
+        this->host = CHost(host, port);
+        this->gameStatus = status;
+        this->elapsedTime = time;
+        this->gameName = name;
+        this->gamePassword = pw;
+        this->gameStatstring = stats;
+    }
+
+    CHost getHost() { return host; }
+
+    
+
+    std::string getGameStatus() {
+        switch (gameStatus) {
+        case 0x0:
+            return "OK";
+        case 0x1:
+            return "No such game";
+        case 0x2:
+            return "Wrong password";
+        case 0x3:
+            return "Game full";
+        case 0x4:
+            return "Game already started";
+        case 0x5:
+            return "Spawned CD-Key not allowed";
+        case 0x6:
+            return "Too many server requests";
+        }
+        return "Unknown status";
+    }
+
+    std::string getGameType() {
+        switch (gameSettings & 0xff) {
+        case 0x02:
+            return "Melee";
+        case 0x03:
+            return "Free 4 All";
+        case 0x04:
+            return "One vs One";
+        case 0x05:
+            return "CTF";
+        case 0x06:
+            switch (gameSettings & 0xffff0000) {
+            case 0x00010000:
+                return "Greed 2500 resources";
+            case 0x00020000:
+                return "Greed 5000 resources";
+            case 0x00030000:
+                return "Greed 7500 resources";
+            case 0x00040000:
+                return "Greed 10000 resources";
+            }
+            return "Greed";
+        case 0x07:
+            switch (gameSettings & 0xffff0000) {
+            case 0x00010000:
+                return "Slaughter 15 minutes";
+            case 0x00020000:
+                return "Slaughter 30 minutes";
+            case 0x00030000:
+                return "Slaughter 45 minutes";
+            case 0x00040000:
+                return "Slaughter 60 minutes";
+            }
+            return "Slaughter";
+        case 0x08:
+            return "Sudden Death";
+        case 0x09:
+            switch (gameSettings & 0xffff0000) {
+            case 0x00000000:
+                return "Ladder (Disconnect is not loss)";
+            case 0x00010000:
+                return "Ladder (Loss on Disconnect)";
+            }
+            return "Ladder";
+        case 0x0A:
+            return "Use Map Settings";
+        case 0x0B:
+        case 0x0C:
+        case 0x0D:
+            std::string sub("");
+            switch (gameSettings & 0xffff0000) {
+            case 0x00010000:
+                sub += " (2 teams)";
+                break;
+            case 0x00020000:
+                sub += " (3 teams)";
+                break;
+            case 0x00030000:
+                sub += " (4 teams)";
+                break;                
+            }
+            switch (gameSettings & 0xff) {
+            case 0x0B:
+                return "Team Melee" + sub;
+            case 0x0C:
+                return "Team Free 4 All" + sub;
+            case 0x0D:
+                return "Team CTF" + sub;
+            }
+        case 0x0F:
+            switch (gameSettings & 0xffff0000) {
+            case 0x00010000:
+                return "Top vs Bottom (1v7)";
+            case 0x00020000:
+                return "Top vs Bottom (2v6)";
+            case 0x00030000:
+                return "Top vs Bottom (3v5)";
+            case 0x00040000:
+                return "Top vs Bottom (4v4)";
+            case 0x00050000:
+                return "Top vs Bottom (5v3)";
+            case 0x00060000:
+                return "Top vs Bottom (6v2)";
+            case 0x00070000:
+                return "Top vs Bottom (7v1)";
+            }
+            return "Top vs Bottom";
+        case 0x10:
+            return "Iron Man Ladder";
+        default:
+            return "Unknown";
+        }
+    }
+
+private:
+    uint32_t gameSettings;
+    uint32_t languageId;
+    CHost host;
+    uint32_t gameStatus;
+    uint32_t elapsedTime;
+    std::string gameName;
+    std::string gamePassword;
+    std::string gameStatstring;
+}
+
+class S2C_CHATEVENT : public NetworkState {
+public:
+    S2C_CHATEVENT() {
+        this->ticks = 0;
+    }
+
+    virtual void doOneStep(Context *ctx) {
+        ticks++;
+        if ((ticks % 5000) == 0) {
+            // C>S 0x07 PKT_KEEPALIVE
+            // ~5000 frames @ ~50fps == 100 seconds
+            BNCSOutputStream keepalive(0x07);
+            keepalive.serialize32(ticks);
+            keepalive.flush(ctx->getUDPSocket(), ctx->getHost());
+        }
+
+        if (ctx->getTCPSocket()->HasDataToRead(0)) {
+            uint8_t msg = ctx->getMsgIStream()->readMessageId();
+            if (msg == 0xff) {
+                // try again next time
+                return;
+            }
+
+            switch (msg) {
+            case 0x00: // SID_NULL
+                break;
+            case 0x0f: // CHATEVENT
+                handleChatevent(ctx);
+                break;
+            case 0x09:
+                // S>C 0x09 SID_GETADVLISTEX
+                handleGamelist(ctx);
+            default:
+                // S>C 0x68 SID_FRIENDSREMOVE
+                // S>C 0x67 SID_FRIENDSADD
+                std::cout << "Unhandled message ID: " << std::hex << msg << std::endl;
+            }
+        }
+    }
+
+private:
+    void handleGamelist(Context *ctx) {
+        uint32_t cnt = ctx->getMsgIStream()->read32();
+        std::vector<std::tuple<CHost, std::string, uint32_t, std::string, std::string, std::string>> games();
+        if (cnt == 0) {
+            ctx->setGamelist(games);
+            return;
+        }
+    }
+
+    void handleChatevent(Context *ctx) {
+        uint32_t eventId = ctx->getMsgIStream()->read32();
+        uint32_t userFlags = ctx->getMsgIStream()->read32();
+        uint32_t ping = ctx->getMsgIStream()->read32();
+        uint32_t ip = ctx->getMsgIStream()->read32();
+        uint32_t acn = ctx->getMsgIStream()->read32();
+        uint32_t reg = ctx->getMsgIStream()->read32();
+        std::string username = ctx->getMsgIStream()->readString();
+        std::string text = ctx->getMsgIStream()->readString();
+        switch (eventId) {
+        case 0x01: // sent for user that is already in channel
+            ctx->addUser(username);
+            break;
+        case 0x02: // user joined channel
+            ctx->addUser(username);
+            ctx->showInfo(username " joined");
+            break;
+        case 0x03: // user left channel
+            ctx->rmUser(username);
+            ctx->showInfo(username " left");
+        case 0x04: // recv whisper
+            ctx->showChat(username + " whispers: " + text);
+            break;
+        case 0x05: // recv chat
+            ctx->showChat(username + ": " + text);
+            break;
+        case 0x06: // recv broadcast
+            ctx->showChat("[BROADCAST]: " + text);
+            break;
+        case 0x07: // channel info
+            break;
+        case 0x09: // user flags update
+            break;
+        case 0x0a: // sent whisper
+            break;
+        case 0x0d: // channel full
+            ctx->showInfo("Channel full");
+            break;
+        case 0x0e: // channel does not exist
+            ctx->showInfo("Channel does not exist");
+            break;
+        case 0x0f: // channel is restricted
+            ctx->showInfo("Channel restricted");
+            break;
+        case 0x12: // general info text
+            ctx->showInfo("[INFO]: " + text);
+            break;
+        case 0x13: // error message
+            ctx->showError("[ERROR]: " + text);
+            break;
+        case 0x17: // emote
+            break;
+        }
+    }
+
+    uint32_t ticks;
 };
 
 class S2C_GETCHANNELLIST : public NetworkState {
@@ -363,10 +670,22 @@ class S2C_GETCHANNELLIST : public NetworkState {
                 error += std::to_string(msg);
                 ctx->setState(new DisconnectedState(error));
             }
-
+            
             std::vector<std::string> channels = ctx->getMsgIStream()->readStringlist();
+            ctx->setChannels(channels);
 
-            ctx->setState(new IN_CHAT());
+            BNCSOutputStream getadvlistex(0x09);
+            getadvlistex.serialize16(0x00); // all games
+            getadvlistex.serialize16(0x01); // no sub game type
+            getadvlistex.serialize32(0xff80); // show all games
+            getadvlistex.serialize32(0x00); // reserved field
+            getadvlistex.serialize32(0xff); // return all games
+            getadvlistex.serialize(""); // no game name
+            getadvlistex.serialize(""); // no game pw
+            getadvlistex.serialize(""); // no game statstring
+            getadvlistex.flush(ctx->getTCPSocket());
+
+            ctx->setState(new S2C_CHATEVENT());
         }
     }
 };
@@ -422,7 +741,7 @@ class C2S_ENTERCHAT : public NetworkState {
 
 class S2C_PKT_SERVERPING : public NetworkState {
 public:
-    S2C_PKT_SERVERPING(std::string message) {
+    S2C_PKT_SERVERPING() {
         this->retries = 0;
     };
 
@@ -478,7 +797,8 @@ class S2C_LOGONRESPONSE2 : public NetworkState {
 
             switch (status) {
             case 0x00:
-                // success - TODO - channel list, join chat
+                // success. we need to send SID_UDPPINGRESPONSE before entering chat
+                ctx->setState(new S2C_PKT_SERVERPING());
                 return;
             case 0x01:
             case 0x02:
@@ -488,7 +808,6 @@ class S2C_LOGONRESPONSE2 : public NetworkState {
                 return;
             case 0x06:
                 ctx->setInfo("Account closed: " + ctx->getMsgIStream()->readString());
-                ctx->setPassword("");
                 ctx->setState(new C2S_LOGONRESPONSE2());
                 return;
             default:
@@ -500,19 +819,30 @@ class S2C_LOGONRESPONSE2 : public NetworkState {
 
 void C2S_LOGONRESPONSE2::doOneStep(Context *ctx) {
     std::string user = ctx->getUsername();
-    std::string pw = ctx->getPassword();
-    if (!user.empty() && !pw.empty()) {
-        // C2S SID_LOGONRESPONSE2
+    uint32_t *pw = ctx->getPassword1(); // single-hashed for SID_LOGONRESPONSE2
+    if (!user.empty() && pw) {
         BNCSOutputStream logon(0x3a);
         logon.serialize32(ctx->clientToken);
         logon.serialize32(ctx->serverToken);
 
-        uint32_t pwhash[5];
-        xsha1_calcHashBuf(pw.c_str(), pw.length(), pwhash);
-        xsha1_calcHashDat(pwhash, pwhash);
-        // Uint8[20] password hash
+        // Battle.net password hashes are hashed twice using XSHA-1. First, the
+        // password is hashed by itself, then the following data is hashed again
+        // and sent to Battle.net:
+        // (UINT32) Client Token
+        // (UINT32) Server Token
+        // (UINT32)[5] First password hash
+        uint32_t data[7];
+        data[0] = ctx->clientToken;
+        data[1] = ctx->serverToken;
+        data[2] = pw[0];
+        data[3] = pw[1];
+        data[4] = pw[2];
+        data[5] = pw[3];
+        data[6] = pw[4];
+        uint32_t sendHash[5];
+        xsha1_calcHashDat(data, sendHash);
         for (int i = 0; i < 20; i++) {
-            logon.serialize8(reinterpret_cast<uint8_t*>(pwhash)[i]);
+            logon.serialize8(reinterpret_cast<uint8_t*>(sendHash)[i]);
         }
         logon.serialize(user.c_str());
         logon.flush(ctx->getTCPSocket());
@@ -623,7 +953,16 @@ class S2C_SID_AUTH_INFO : public NetworkState {
             check.serialize32(0);
             check.serialize32(0);
             // EXE information
-            std::string exeInfo("stratagus.exe ");
+            std::string exeInfo("");
+            if (!FullGameName.empty()) {
+                exeInfo += FullGameName;
+            } else {
+                if (!GameName.empty()) {
+                    exeInfo += GameName;
+                } else {
+                    exeInfo += Parameters::Instance.applicationName;
+                }
+            }
             exeInfo += StratagusLastModifiedDate;
             exeInfo += " ";
             exeInfo += StratagusLastModifiedTime;
@@ -655,7 +994,7 @@ class S2C_SID_PING : public NetworkState {
             }
             uint32_t pingValue = ctx->getMsgIStream()->read32();
 
-            // immediately respond
+            // immediately respond with C2S_SID_PING
             BNCSOutputStream buffer(0x25);
             buffer.serialize32(pingValue);
             send(ctx, &buffer);
@@ -665,16 +1004,31 @@ class S2C_SID_PING : public NetworkState {
     }
 };
 
-class C2S_SID_AUTH_INFO : public NetworkState {
+class ConnectState : public NetworkState {
     virtual void doOneStep(Context *ctx) {
+        // Connect
+        ctx->getTCPSocket()->Open(*ctx->getHost());
+        if (ctx->getTCPSocket()->IsValid() == false) {
+            ctx->setState(new DisconnectedState("TCP open failed"));
+            return;
+        }
+        if (ctx->getTCPSocket()->Connect(*ctx->getHost()) == false) {
+            ctx->setState(new DisconnectedState("TCP connect failed"));
+            return;
+        }
+        // Send proto byte
+        ctx->getTCPSocket()->Send("\x1", 1);
+
+        // C>S SID_AUTH_INFO
         BNCSOutputStream buffer(0x50);
         // (UINT32) Protocol ID
         buffer.serialize32(0x00);
         // (UINT32) Platform code
         buffer.serialize("IX86", 4);
-        // (UINT32) Product code
-        buffer.serialize("WAGS", 4);
-        // (UINT32) Version byte
+        // (UINT32) Product code - we'll just use W2BN and encode what we are in
+        // the version
+        buffer.serialize("W2BN", 4);
+        // (UINT32) Version byte - just use the last from W2BN
         buffer.serialize32(0x4f);
         // (UINT32) Language code
         buffer.serialize32(0x00);
@@ -705,7 +1059,22 @@ class C2S_SID_AUTH_INFO : public NetworkState {
             }
         }
         // (UINT32) Time zone bias
-        buffer.serialize32(0x00);
+        uint32_t bias = 0;
+        std::time_t systemtime = std::time(NULL);
+        struct std::tm *utc = std::gmtime(&systemtime);
+        if (utc != NULL) {
+            std::time_t utctime_since_epoch = std::mktime(utc);
+            if (utctime_since_epoch != -1) {
+                struct std::tm *localtime = std::localtime(&systemtime);
+                if (localtime != 0) {
+                    std::time_t localtime_since_epoch = std::mktime(localtime);
+                    if (localtime_since_epoch != -1) {
+                        bias = std::difftime(utctime_since_epoch, localtime_since_epoch) / 60;
+                    }
+                }
+            }
+        }
+        buffer.serialize32(bias);
         // (UINT32) MPQ locale ID
         buffer.serialize32(0x00);
         // (UINT32) User language ID
@@ -720,28 +1089,6 @@ class C2S_SID_AUTH_INFO : public NetworkState {
     }
 };
 
-class ProtoByteState : public NetworkState {
-    virtual void doOneStep(Context *ctx) {
-        ctx->getTCPSocket()->Send("\x1", 1); // use game protocol
-        ctx->setState(new C2S_SID_AUTH_INFO());
-    }
-};
-
-class ConnectState : public NetworkState {
-    virtual void doOneStep(Context *ctx) {
-        ctx->getTCPSocket()->Open(*ctx->getHost());
-        if (ctx->getTCPSocket()->IsValid() == false) {
-            ctx->setState(new DisconnectedState("TCP open failed"));
-            return;
-        }
-        if (ctx->getTCPSocket()->Connect(*ctx->getHost()) == false) {
-            ctx->setState(new DisconnectedState("TCP connect failed"));
-            return;
-        }
-        ctx->setState(new ProtoByteState());
-    }
-};
-
 static void goOnline() {
     Context *ctx = new Context();
     ctx->setState(new ConnectState());

From d4bfbf1b2ec2efdb6c187cdf8b80acaa7861e925 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Mon, 27 Jul 2020 09:18:52 +0200
Subject: [PATCH 04/70] base logon and chat interaction messages

---
 src/network/online_service.cpp | 610 +++++++++++++++++++++++----------
 1 file changed, 427 insertions(+), 183 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index dea395c0e..85de113ae 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1,11 +1,16 @@
+#include <arpa/inet.h>
 #include <clocale>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
 #include <ctime>
 #include <iostream>
+#include <map>
 #include <ostream>
+#include <queue>
+#include <set>
 #include <sstream>
+#include <stdexcept>
 #include <string>
 #include <tuple>
 #include <unistd.h>
@@ -242,6 +247,307 @@ private:
     int length_pos;
 };
 
+class Friend {
+public:
+    Friend(std::string name, uint8_t status, uint8_t location, uint32_t product, std::string locationName) {
+        this->name = name;
+        this->status = status;
+        this->location = location;
+        this->product = product;
+        this->locationName = locationName;
+    }
+
+    std::string getStatus() {
+        switch (location) {
+        case 0:
+            return "offline";
+        case 1:
+            return "not in chat";
+        case 2:
+            return "in chat";
+        case 3:
+            return "in public game " + locationName;
+        case 4:
+            return "in a private game";
+        case 5:
+            return "in private game " + locationName;
+        default:
+            return "unknown";
+        }
+    }
+
+    std::string getName() { return name; }
+
+    std::string getProduct() {
+        switch (product) {
+        case 0x1:
+        case 0x2:
+        case 0x6:
+        case 0xb:
+            return "Starcraft";
+        case 0x3:
+            return "Warcraft II";
+        case 0x4:
+        case 0x5:
+            return "Diablo II";
+        case 0x7:
+        case 0x8:
+        case 0xc:
+            return "Warcraft III";
+        case 0x9:
+        case 0xa:
+            return "Diablo";
+        default:
+            return "Unknown Game";
+        }
+    }
+
+private:
+    std::string name;
+    uint8_t status;
+    uint8_t location;
+    uint32_t product;
+    std::string locationName;
+};
+
+class Game {
+public:
+    Game(uint32_t settings, uint16_t port, uint32_t host, uint32_t status, uint32_t time, std::string name, std::string pw, std::string stats) {
+        this->gameSettings = settings;
+        this->host = CHost(host, port);
+        this->gameStatus = status;
+        this->elapsedTime = time;
+        this->gameName = name;
+        this->gamePassword = pw;
+        this->gameStatstring = stats;
+        splitStatstring();
+    }
+
+    CHost getHost() { return host; }
+
+    bool isSavedGame() {
+        return !gameStats[0].empty();
+    };
+
+    std::tuple<int, int> mapSize() {
+        if (gameStats[1].empty()) {
+            return {128, 128};
+        }
+        char w = gameStats[1].at(0);
+        char h = gameStats[1].at(1);
+        return {w * 32, h * 32};
+    };
+
+    int maxPlayers() {
+        if (gameStats[2].empty()) {
+            return 8;
+        } else {
+            return std::stoi(gameStats[2]) - 10;
+        }
+    };
+
+    std::string getApproval() {
+        return "Not approved";
+    }
+
+    std::string getGameSettings() {
+        if (gameStats[9].empty()) {
+            return "Map default";
+        }
+        long settings = std::stol(gameStats[9]);
+        std::string result;
+        if (settings & 0x200) {
+            result += " 1 worker";
+        }
+        if (settings & 0x400) {
+            result += " fixed placement";
+        }
+        switch (settings & 0x23000) {
+        case 0x01000:
+            result += " low resources";
+            break;
+        case 0x02000:
+            result += " medium resources";
+            break;
+        case 0x03000:
+            result += " high resources";
+            break;
+        case 0x20000:
+            result += " random resources";
+            break;
+        }
+        switch (settings & 0x1C000) {
+        case 0x04000:
+            result += " forest";
+            break;
+        case 0x08000:
+            result += " winter";
+            break;
+        case 0x0c000:
+            result += " wasteland";
+            break;
+        case 0x14000:
+            result += " random";
+            break;
+        case 0x1c000:
+            result += " orc swamp";
+            break;
+        }
+        return result;
+    };
+
+    std::string getCreator() {
+        int end = gameStats[10].find("\r");
+        return gameStats[10].substr(0, end);
+    };
+
+    std::string getMap() {
+        int begin = gameStats[10].find("\r") + 1;
+        return gameStats[10].substr(begin);
+    };
+
+    std::string getGameStatus() {
+        switch (gameStatus) {
+        case 0x0:
+            return "OK";
+        case 0x1:
+            return "No such game";
+        case 0x2:
+            return "Wrong password";
+        case 0x3:
+            return "Game full";
+        case 0x4:
+            return "Game already started";
+        case 0x5:
+            return "Spawned CD-Key not allowed";
+        case 0x6:
+            return "Too many server requests";
+        }
+        return "Unknown status";
+    }
+
+    std::string getGameType() {
+        std::string sub("");
+        switch (gameSettings & 0xff) {
+        case 0x02:
+            return "Melee";
+        case 0x03:
+            return "Free 4 All";
+        case 0x04:
+            return "One vs One";
+        case 0x05:
+            return "CTF";
+        case 0x06:
+            switch (gameSettings & 0xffff0000) {
+            case 0x00010000:
+                return "Greed 2500 resources";
+            case 0x00020000:
+                return "Greed 5000 resources";
+            case 0x00030000:
+                return "Greed 7500 resources";
+            case 0x00040000:
+                return "Greed 10000 resources";
+            }
+            return "Greed";
+        case 0x07:
+            switch (gameSettings & 0xffff0000) {
+            case 0x00010000:
+                return "Slaughter 15 minutes";
+            case 0x00020000:
+                return "Slaughter 30 minutes";
+            case 0x00030000:
+                return "Slaughter 45 minutes";
+            case 0x00040000:
+                return "Slaughter 60 minutes";
+            }
+            return "Slaughter";
+        case 0x08:
+            return "Sudden Death";
+        case 0x09:
+            switch (gameSettings & 0xffff0000) {
+            case 0x00000000:
+                return "Ladder (Disconnect is not loss)";
+            case 0x00010000:
+                return "Ladder (Loss on Disconnect)";
+            }
+            return "Ladder";
+        case 0x0A:
+            return "Use Map Settings";
+        case 0x0B:
+        case 0x0C:
+        case 0x0D:
+            switch (gameSettings & 0xffff0000) {
+            case 0x00010000:
+                sub += " (2 teams)";
+                break;
+            case 0x00020000:
+                sub += " (3 teams)";
+                break;
+            case 0x00030000:
+                sub += " (4 teams)";
+                break;
+            }
+            switch (gameSettings & 0xff) {
+            case 0x0B:
+                return "Team Melee" + sub;
+            case 0x0C:
+                return "Team Free 4 All" + sub;
+            case 0x0D:
+                return "Team CTF" + sub;
+            }
+        case 0x0F:
+            switch (gameSettings & 0xffff0000) {
+            case 0x00010000:
+                return "Top vs Bottom (1v7)";
+            case 0x00020000:
+                return "Top vs Bottom (2v6)";
+            case 0x00030000:
+                return "Top vs Bottom (3v5)";
+            case 0x00040000:
+                return "Top vs Bottom (4v4)";
+            case 0x00050000:
+                return "Top vs Bottom (5v3)";
+            case 0x00060000:
+                return "Top vs Bottom (6v2)";
+            case 0x00070000:
+                return "Top vs Bottom (7v1)";
+            }
+            return "Top vs Bottom";
+        case 0x10:
+            return "Iron Man Ladder";
+        default:
+            return "Unknown";
+        }
+    }
+
+private:
+    void splitStatstring() {
+        // statstring is a comma-delimited list of values
+        int pos = 0;
+        while (true) {
+            int newpos = gameStatstring.find(",", pos);
+            gameStats.push_back(gameStatstring.substr(pos, newpos));
+            pos = newpos + 1;
+            if (pos == 0) {
+                break;
+            }
+        }
+        while (gameStats.size() < 10) {
+            gameStats.push_back("");
+        }
+    }
+
+    uint32_t gameSettings;
+    uint32_t languageId;
+    CHost host;
+    uint32_t gameStatus;
+    uint32_t elapsedTime;
+    std::string gameName;
+    std::string gamePassword;
+    std::string gameStatstring;
+    std::vector<std::string> gameStats;
+};
+
 class Context;
 class NetworkState {
 public:
@@ -262,7 +568,6 @@ public:
         this->host = new CHost("127.0.0.1", 6112); // TODO: parameterize
         this->clientToken = MyRand();
         this->username = "";
-        this->info = "";
     }
 
     ~Context() {
@@ -274,9 +579,43 @@ public:
         delete host;
     }
 
-    std::string getInfo() { return info; }
+    void setGamelist(std::vector<Game*> games) {
+        for (const auto value : this->games) {
+            delete value;
+        }
+        this->games = games;
+    }
 
-    void setInfo(std::string arg) { info = arg; }
+    void setFriendslist(std::vector<Friend*> friends) {
+        for (const auto value : this->friends) {
+            delete value;
+        }
+        this->friends = friends;
+    }
+
+    void reportUserdata(uint32_t id, std::vector<std::string> values) {
+        this->extendedInfoValues[id] = values;
+    }
+
+    std::queue<std::string> getInfo() { return info; }
+
+    void showInfo(std::string arg) { info.push("*** " + arg + " ***"); }
+
+    void showError(std::string arg) { info.push("!!! " + arg + " !!!"); }
+
+    void showChat(std::string arg) { info.push(arg); }
+
+    void addUser(std::string name) {
+        userList.insert(name);
+    }
+
+    void removeUser(std::string name) {
+        userList.erase(name);
+    }
+
+    void setChannels(std::vector<std::string> channels) {
+        this->channelList = channels;
+    }
 
     std::string getUsername() { return username; }
 
@@ -342,10 +681,18 @@ private:
     CTCPSocket *tcpSocket;
     BNCSInputStream *istream;
 
-    std::string info;
     std::string username;
     uint32_t password[5]; // xsha1 hash of password
     uint32_t password2[5]; // xsha1 hash of password hash
+
+    std::string currentChannel;
+    std::set<std::string> userList;
+    std::vector<std::string> channelList;
+    std::queue<std::string> info;
+    std::vector<Game*> games;
+    std::vector<Friend*> friends;
+    std::map<uint32_t, std::vector<std::string>> extendedInfoKeys;
+    std::map<uint32_t, std::vector<std::string>> extendedInfoValues;
 };
 
 int NetworkState::send(Context *ctx, BNCSOutputStream *buf) {
@@ -373,181 +720,22 @@ private:
 };
 
 /* needed
-C>S 0x02 SID_STOPADV
 
 C>S 0x09 SID_GETADVLISTEX
-S>C 0x09 SID_GETADVLISTEX
-
 C>S 0x0E SID_CHATCOMMAND
-
-S>C 0x0F SID_CHATEVENT
-
-C>S 0x1C SID_STARTADVEX3
-S>C 0x1C SID_STARTADVEX3
-
 C>S 0x22 SID_NOTIFYJOIN
-
-C>S 0x3D SID_CREATEACCOUNT2
-S>C 0x3D SID_CREATEACCOUNT2
-
-C>S 0x2C SID_GAMERESULT
-
-C>S 0x2F SID_FINDLADDERUSER
-S>C 0x2F SID_FINDLADDERUSER
-
-C>S 0x26 SID_READUSERDATA
-S>C 0x26 SID_READUSERDATA
-
-C>S 0x31 SID_CHANGEPASSWORD
-S>C 0x31 SID_CHANGEPASSWORD
-
-C>S 0x5A SID_RESETPASSWORD
-
 C>S 0x65 SID_FRIENDSLIST
-S>C 0x65 SID_FRIENDSLIST
+C>S 0x26 SID_READUSERDATA
 
 */
 
-class Game {
-public:
-    Game(uint32_t settings, uint16_t port, uint32_t host, uint32_t status, uint32_t time, std::string name, std::string pw, std::string stats) {
-        this->gameSettings = settings;
-        this->host = CHost(host, port);
-        this->gameStatus = status;
-        this->elapsedTime = time;
-        this->gameName = name;
-        this->gamePassword = pw;
-        this->gameStatstring = stats;
+class C2S_GAMERESULT_OR_STOPADV : public NetworkState {
+    virtual void doOneStep(Context *ctx) {
+        // TODO - wait until the game lobby is left or the game is over and then send the result
+        // C>S 0x02 SID_STOPADV
+        // C>S 0x2C SID_GAMERESULT
     }
-
-    CHost getHost() { return host; }
-
-    
-
-    std::string getGameStatus() {
-        switch (gameStatus) {
-        case 0x0:
-            return "OK";
-        case 0x1:
-            return "No such game";
-        case 0x2:
-            return "Wrong password";
-        case 0x3:
-            return "Game full";
-        case 0x4:
-            return "Game already started";
-        case 0x5:
-            return "Spawned CD-Key not allowed";
-        case 0x6:
-            return "Too many server requests";
-        }
-        return "Unknown status";
-    }
-
-    std::string getGameType() {
-        switch (gameSettings & 0xff) {
-        case 0x02:
-            return "Melee";
-        case 0x03:
-            return "Free 4 All";
-        case 0x04:
-            return "One vs One";
-        case 0x05:
-            return "CTF";
-        case 0x06:
-            switch (gameSettings & 0xffff0000) {
-            case 0x00010000:
-                return "Greed 2500 resources";
-            case 0x00020000:
-                return "Greed 5000 resources";
-            case 0x00030000:
-                return "Greed 7500 resources";
-            case 0x00040000:
-                return "Greed 10000 resources";
-            }
-            return "Greed";
-        case 0x07:
-            switch (gameSettings & 0xffff0000) {
-            case 0x00010000:
-                return "Slaughter 15 minutes";
-            case 0x00020000:
-                return "Slaughter 30 minutes";
-            case 0x00030000:
-                return "Slaughter 45 minutes";
-            case 0x00040000:
-                return "Slaughter 60 minutes";
-            }
-            return "Slaughter";
-        case 0x08:
-            return "Sudden Death";
-        case 0x09:
-            switch (gameSettings & 0xffff0000) {
-            case 0x00000000:
-                return "Ladder (Disconnect is not loss)";
-            case 0x00010000:
-                return "Ladder (Loss on Disconnect)";
-            }
-            return "Ladder";
-        case 0x0A:
-            return "Use Map Settings";
-        case 0x0B:
-        case 0x0C:
-        case 0x0D:
-            std::string sub("");
-            switch (gameSettings & 0xffff0000) {
-            case 0x00010000:
-                sub += " (2 teams)";
-                break;
-            case 0x00020000:
-                sub += " (3 teams)";
-                break;
-            case 0x00030000:
-                sub += " (4 teams)";
-                break;                
-            }
-            switch (gameSettings & 0xff) {
-            case 0x0B:
-                return "Team Melee" + sub;
-            case 0x0C:
-                return "Team Free 4 All" + sub;
-            case 0x0D:
-                return "Team CTF" + sub;
-            }
-        case 0x0F:
-            switch (gameSettings & 0xffff0000) {
-            case 0x00010000:
-                return "Top vs Bottom (1v7)";
-            case 0x00020000:
-                return "Top vs Bottom (2v6)";
-            case 0x00030000:
-                return "Top vs Bottom (3v5)";
-            case 0x00040000:
-                return "Top vs Bottom (4v4)";
-            case 0x00050000:
-                return "Top vs Bottom (5v3)";
-            case 0x00060000:
-                return "Top vs Bottom (6v2)";
-            case 0x00070000:
-                return "Top vs Bottom (7v1)";
-            }
-            return "Top vs Bottom";
-        case 0x10:
-            return "Iron Man Ladder";
-        default:
-            return "Unknown";
-        }
-    }
-
-private:
-    uint32_t gameSettings;
-    uint32_t languageId;
-    CHost host;
-    uint32_t gameStatus;
-    uint32_t elapsedTime;
-    std::string gameName;
-    std::string gamePassword;
-    std::string gameStatstring;
-}
+};
 
 class S2C_CHATEVENT : public NetworkState {
 public:
@@ -581,7 +769,25 @@ public:
             case 0x09:
                 // S>C 0x09 SID_GETADVLISTEX
                 handleGamelist(ctx);
+                break;
+            case 0x1c:
+                // S>C 0x1C SID_STARTADVEX3
+                if (ctx->getMsgIStream()->read32()) {
+                    ctx->showError("Game creation failed");
+                } else {
+                    ctx->setState(new C2S_GAMERESULT_OR_STOPADV());
+                }
+                break;
+            case 0x65:
+                // S>C 0x65 SID_FRIENDSLIST
+                handleFriendlist(ctx);
+                break;
+            case 0x26:
+                // S>C 0x26 SID_READUSERDATA
+                handleUserdata(ctx);
+                break;
             default:
+                // TODO:
                 // S>C 0x68 SID_FRIENDSREMOVE
                 // S>C 0x67 SID_FRIENDSADD
                 std::cout << "Unhandled message ID: " << std::hex << msg << std::endl;
@@ -592,11 +798,47 @@ public:
 private:
     void handleGamelist(Context *ctx) {
         uint32_t cnt = ctx->getMsgIStream()->read32();
-        std::vector<std::tuple<CHost, std::string, uint32_t, std::string, std::string, std::string>> games();
-        if (cnt == 0) {
-            ctx->setGamelist(games);
-            return;
+        std::vector<Game*> games;
+        while (cnt--) {
+            uint32_t settings = ctx->getMsgIStream()->read32();
+            uint32_t lang = ctx->getMsgIStream()->read32();
+            uint16_t addr_fam = ctx->getMsgIStream()->read16();
+            uint16_t port = ctx->getMsgIStream()->read16();
+            uint32_t ip = ctx->getMsgIStream()->read32();
+            uint32_t sinzero1 = ctx->getMsgIStream()->read32();
+            uint32_t sinzero2 = ctx->getMsgIStream()->read32();
+            uint32_t status = ctx->getMsgIStream()->read32();
+            uint32_t time = ctx->getMsgIStream()->read32();
+            std::string name = ctx->getMsgIStream()->readString();
+            std::string pw = ctx->getMsgIStream()->readString();
+            std::string stat = ctx->getMsgIStream()->readString();
+            games.push_back(new Game(settings, port, ip, status, time, name, pw, stat));
         }
+        ctx->setGamelist(games);
+    }
+
+
+    void handleFriendlist(Context *ctx) {
+        uint32_t cnt = ctx->getMsgIStream()->read32();
+        std::vector<Friend*> friends;
+        while (cnt--) {
+            std::string user = ctx->getMsgIStream()->readString();
+            uint8_t status = ctx->getMsgIStream()->read8();
+            uint8_t location = ctx->getMsgIStream()->read8();
+            uint32_t product = ctx->getMsgIStream()->read32();
+            std::string locname = ctx->getMsgIStream()->readString();
+            friends.push_back(new Friend(user, status, location, product, locname));
+        }
+        ctx->setFriendslist(friends);
+    }
+
+    void handleUserdata(Context *ctx) {
+        uint32_t cnt = ctx->getMsgIStream()->read32();
+        assert(cnt == 1);
+        uint32_t keys = ctx->getMsgIStream()->read32();
+        uint32_t reqId = ctx->getMsgIStream()->read32();
+        std::vector<std::string> values = ctx->getMsgIStream()->readStringlist();
+        ctx->reportUserdata(reqId, values);
     }
 
     void handleChatevent(Context *ctx) {
@@ -614,11 +856,11 @@ private:
             break;
         case 0x02: // user joined channel
             ctx->addUser(username);
-            ctx->showInfo(username " joined");
+            ctx->showInfo(username + " joined");
             break;
         case 0x03: // user left channel
-            ctx->rmUser(username);
-            ctx->showInfo(username " left");
+            ctx->removeUser(username);
+            ctx->showInfo(username + " left");
         case 0x04: // recv whisper
             ctx->showChat(username + " whispers: " + text);
             break;
@@ -670,13 +912,13 @@ class S2C_GETCHANNELLIST : public NetworkState {
                 error += std::to_string(msg);
                 ctx->setState(new DisconnectedState(error));
             }
-            
+
             std::vector<std::string> channels = ctx->getMsgIStream()->readStringlist();
             ctx->setChannels(channels);
 
             BNCSOutputStream getadvlistex(0x09);
-            getadvlistex.serialize16(0x00); // all games
-            getadvlistex.serialize16(0x01); // no sub game type
+            getadvlistex.serialize16((uint16_t) 0x00); // all games
+            getadvlistex.serialize16((uint16_t) 0x01); // no sub game type
             getadvlistex.serialize32(0xff80); // show all games
             getadvlistex.serialize32(0x00); // reserved field
             getadvlistex.serialize32(0xff); // return all games
@@ -710,7 +952,7 @@ class S2C_ENTERCHAT : public NetworkState {
 
             ctx->setUsername(uniqueName);
             if (!statString.empty()) {
-                ctx->setInfo("Statstring after logon: " + statString);
+                ctx->showInfo("Statstring after logon: " + statString);
             }
 
             ctx->setState(new S2C_GETCHANNELLIST());
@@ -802,12 +1044,14 @@ class S2C_LOGONRESPONSE2 : public NetworkState {
                 return;
             case 0x01:
             case 0x02:
-                ctx->setInfo("Account does not exist or incorrect password");
+                ctx->showInfo("Account does not exist or incorrect password");
                 ctx->setPassword("");
+                // C>S 0x3D SID_CREATEACCOUNT2
+                // S>C 0x3D SID_CREATEACCOUNT2
                 ctx->setState(new C2S_LOGONRESPONSE2());
                 return;
             case 0x06:
-                ctx->setInfo("Account closed: " + ctx->getMsgIStream()->readString());
+                ctx->showInfo("Account closed: " + ctx->getMsgIStream()->readString());
                 ctx->setState(new C2S_LOGONRESPONSE2());
                 return;
             default:

From a2c133a6dbc445c49dbeb562db4031104cf7b68f Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Mon, 27 Jul 2020 10:03:31 +0200
Subject: [PATCH 05/70] finish account creation and provide api for user/UI
 actions that need server interaction

---
 src/network/online_service.cpp | 184 ++++++++++++++++++++++++++++-----
 1 file changed, 160 insertions(+), 24 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 85de113ae..f238d0f22 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -579,6 +579,66 @@ public:
         delete host;
     }
 
+    // User and UI actions
+    void sendText(std::string txt) {
+        // C>S 0x0E SID_CHATCOMMAND
+        int pos = 0;
+        for (int pos = 0; pos < txt.size(); pos += 220) {
+            std::string text = txt.substr(pos, pos + 220);
+            if (pos + 220 < txt.size()) {
+                text += "...";
+            }
+            BNCSOutputStream msg(0x0e);
+            msg.serialize(text.c_str());
+            msg.flush(getTCPSocket());
+        }
+    }
+
+    void requestExtraUserInfo(std::string username) {
+        BNCSOutputStream msg(0x26);
+        msg.serialize32(1); // num accounts
+        msg.serialize32(5); // num keys
+        msg.serialize32((uint32_t) extendedInfoIdx.size());
+        msg.serialize(username.c_str());
+        msg.serialize("record\\GAME\\0\\wins");
+        msg.serialize("record\\GAME\\0\\losses");
+        msg.serialize("record\\GAME\\0\\disconnects");
+        msg.serialize("record\\GAME\\0\\last game");
+        msg.serialize("record\\GAME\\0\\last game result");
+        msg.flush(getTCPSocket());
+    }
+
+    void refreshGames() {
+        // C>S 0x09 SID_GETADVLISTEX
+        BNCSOutputStream getadvlistex(0x09);
+        getadvlistex.serialize16((uint16_t) 0x00); // all games
+        getadvlistex.serialize16((uint16_t) 0x01); // no sub game type
+        getadvlistex.serialize32(0xff80); // show all games
+        getadvlistex.serialize32(0x00); // reserved field
+        getadvlistex.serialize32(0xff); // return all games
+        getadvlistex.serialize(""); // no game name
+        getadvlistex.serialize(""); // no game pw
+        getadvlistex.serialize(""); // no game statstring
+        getadvlistex.flush(getTCPSocket());
+    }
+
+    void refreshFriends() {
+        // C>S 0x65 SID_FRIENDSLIST
+        BNCSOutputStream msg(0x65);
+        msg.flush(getTCPSocket());
+    }
+
+    void joinGame(std::string name, std::string pw) {
+        // C>S 0x22 SID_NOTIFYJOIN
+        BNCSOutputStream msg(0x09);
+        msg.serialize("W2BN", 4);
+        msg.serialize32(0x4f);
+        msg.serialize(name.c_str());
+        msg.serialize(pw.c_str());
+        msg.flush(getTCPSocket());
+    }
+
+    // UI information
     void setGamelist(std::vector<Game*> games) {
         for (const auto value : this->games) {
             delete value;
@@ -617,6 +677,7 @@ public:
         this->channelList = channels;
     }
 
+    // State
     std::string getUsername() { return username; }
 
     void setUsername(std::string arg) { username = arg; }
@@ -646,6 +707,7 @@ public:
         xsha1_calcHashDat(password, password2);
     }
 
+    // Protocol
     CHost *getHost() { return host; }
 
     void setHost(CHost *arg) {
@@ -691,7 +753,7 @@ private:
     std::queue<std::string> info;
     std::vector<Game*> games;
     std::vector<Friend*> friends;
-    std::map<uint32_t, std::vector<std::string>> extendedInfoKeys;
+    std::map<std::string, uint32_t> extendedInfoIdx;
     std::map<uint32_t, std::vector<std::string>> extendedInfoValues;
 };
 
@@ -719,16 +781,6 @@ private:
     std::string message;
 };
 
-/* needed
-
-C>S 0x09 SID_GETADVLISTEX
-C>S 0x0E SID_CHATCOMMAND
-C>S 0x22 SID_NOTIFYJOIN
-C>S 0x65 SID_FRIENDSLIST
-C>S 0x26 SID_READUSERDATA
-
-*/
-
 class C2S_GAMERESULT_OR_STOPADV : public NetworkState {
     virtual void doOneStep(Context *ctx) {
         // TODO - wait until the game lobby is left or the game is over and then send the result
@@ -916,16 +968,9 @@ class S2C_GETCHANNELLIST : public NetworkState {
             std::vector<std::string> channels = ctx->getMsgIStream()->readStringlist();
             ctx->setChannels(channels);
 
-            BNCSOutputStream getadvlistex(0x09);
-            getadvlistex.serialize16((uint16_t) 0x00); // all games
-            getadvlistex.serialize16((uint16_t) 0x01); // no sub game type
-            getadvlistex.serialize32(0xff80); // show all games
-            getadvlistex.serialize32(0x00); // reserved field
-            getadvlistex.serialize32(0xff); // return all games
-            getadvlistex.serialize(""); // no game name
-            getadvlistex.serialize(""); // no game pw
-            getadvlistex.serialize(""); // no game statstring
-            getadvlistex.flush(ctx->getTCPSocket());
+            // request our user info and refresh the active games list
+            ctx->requestExtraUserInfo(ctx->getUsername());
+            ctx->refreshGames();
 
             ctx->setState(new S2C_CHATEVENT());
         }
@@ -1021,6 +1066,83 @@ class C2S_LOGONRESPONSE2 : public NetworkState {
     virtual void doOneStep(Context *ctx);
 };
 
+class S2C_CREATEACCOUNT2 : public NetworkState {
+    virtual void doOneStep(Context *ctx) {
+        if (ctx->getTCPSocket()->HasDataToRead(0)) {
+            uint8_t msg = ctx->getMsgIStream()->readMessageId();
+            if (msg == 0xff) {
+                // try again next time
+                return;
+            }
+            if (msg != 0x3d) {
+                std::string error = std::string("Expected SID_CREATEACCOUNT2, got msg id ");
+                error += std::to_string(msg);
+                ctx->setState(new DisconnectedState(error));
+            }
+
+            uint32_t status = ctx->getMsgIStream()->read32();
+            std::string nameSugg = ctx->getMsgIStream()->readString();
+
+            if (!nameSugg.empty()) {
+                nameSugg = " (try username: " + nameSugg + ")";
+            }
+
+            switch (status) {
+            case 0x00:
+                // login into created account
+                ctx->setState(new C2S_LOGONRESPONSE2());
+                return;
+            case 0x01:
+                ctx->showError("Name too short" + nameSugg);
+                ctx->setUsername("");
+                ctx->setState(new C2S_LOGONRESPONSE2());
+                return;
+            case 0x02:
+                ctx->showError("Name contains invalid character(s)" + nameSugg);
+                ctx->setUsername("");
+                ctx->setState(new C2S_LOGONRESPONSE2());
+                return;
+            case 0x03:
+                ctx->showError("Name contains banned word(s)" + nameSugg);
+                ctx->setUsername("");
+                ctx->setState(new C2S_LOGONRESPONSE2());
+                return;
+            case 0x04:
+                ctx->showError("Account already exists" + nameSugg);
+                ctx->setUsername("");
+                ctx->setState(new C2S_LOGONRESPONSE2());
+                return;
+            case 0x05:
+                ctx->showError("Account is still being created" + nameSugg);
+                ctx->setUsername("");
+                ctx->setState(new C2S_LOGONRESPONSE2());
+                return;
+            case 0x06:
+                ctx->showError("Name does not contain enough alphanumeric characters" + nameSugg);
+                ctx->setUsername("");
+                ctx->setState(new C2S_LOGONRESPONSE2());
+                return;
+            case 0x07:
+                ctx->showError("Name contained adjacent punctuation characters" + nameSugg);
+                ctx->setUsername("");
+                ctx->setState(new C2S_LOGONRESPONSE2());
+                return;
+            case 0x08:
+                ctx->showError("Name contained too many punctuation characters" + nameSugg);
+                ctx->setUsername("");
+                ctx->setState(new C2S_LOGONRESPONSE2());
+                return;
+            default:
+                ctx->showError("Unknown error creating account" + nameSugg);
+                ctx->setUsername("");
+                ctx->setState(new C2S_LOGONRESPONSE2());
+                return;
+            }
+
+        }
+    }
+};
+
 class S2C_LOGONRESPONSE2 : public NetworkState {
     virtual void doOneStep(Context *ctx) {
         if (ctx->getTCPSocket()->HasDataToRead(0)) {
@@ -1043,15 +1165,17 @@ class S2C_LOGONRESPONSE2 : public NetworkState {
                 ctx->setState(new S2C_PKT_SERVERPING());
                 return;
             case 0x01:
+                ctx->showInfo("Account does not exist, creating it...");
+                createAccount(ctx);
+                return;
             case 0x02:
-                ctx->showInfo("Account does not exist or incorrect password");
+                ctx->showInfo("Incorrect password");
                 ctx->setPassword("");
-                // C>S 0x3D SID_CREATEACCOUNT2
-                // S>C 0x3D SID_CREATEACCOUNT2
                 ctx->setState(new C2S_LOGONRESPONSE2());
                 return;
             case 0x06:
                 ctx->showInfo("Account closed: " + ctx->getMsgIStream()->readString());
+                ctx->setPassword("");
                 ctx->setState(new C2S_LOGONRESPONSE2());
                 return;
             default:
@@ -1059,6 +1183,18 @@ class S2C_LOGONRESPONSE2 : public NetworkState {
             }
         }
     }
+
+private:
+    void createAccount(Context *ctx) {
+        BNCSOutputStream msg(0x3d);
+        uint32_t *pw = ctx->getPassword1();
+        for (int i = 0; i < 20; i++) {
+            msg.serialize8(reinterpret_cast<uint8_t*>(pw)[i]);
+        }
+        msg.serialize(ctx->getUsername().c_str());
+        msg.flush(ctx->getTCPSocket());
+        ctx->setState(new S2C_CREATEACCOUNT2());
+    }
 };
 
 void C2S_LOGONRESPONSE2::doOneStep(Context *ctx) {

From d1dee86433f649144793dd68e149a25de3a3de10 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Tue, 28 Jul 2020 23:18:52 +0200
Subject: [PATCH 06/70] very crude hardcoded ui

---
 src/network/online_service.cpp | 224 ++++++++++++++++++++++++++++++++-
 1 file changed, 218 insertions(+), 6 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index f238d0f22..c383079df 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -22,6 +22,13 @@
 #include <netinet/in.h>
 #endif
 
+#include "cursor.h"
+#include "font.h"
+#include "input.h"
+#include "stratagus.h"
+#include "ui.h"
+#include "video.h"
+#include "widgets.h"
 #include "game.h"
 #include "parameters.h"
 #include "assert.h"
@@ -639,6 +646,12 @@ public:
     }
 
     // UI information
+    void setCurrentChannel(std::string name) {
+        this->currentChannel = name;
+    }
+
+    std::string getCurrentChannel() { return currentChannel; }
+
     void setGamelist(std::vector<Game*> games) {
         for (const auto value : this->games) {
             delete value;
@@ -646,6 +659,8 @@ public:
         this->games = games;
     }
 
+    std::vector<Game*> getGames() { return games; }
+
     void setFriendslist(std::vector<Friend*> friends) {
         for (const auto value : this->friends) {
             delete value;
@@ -653,6 +668,8 @@ public:
         this->friends = friends;
     }
 
+    std::vector<Friend*> getFriends() { return friends; }
+
     void reportUserdata(uint32_t id, std::vector<std::string> values) {
         this->extendedInfoValues[id] = values;
     }
@@ -673,10 +690,14 @@ public:
         userList.erase(name);
     }
 
+    std::set<std::string> getUsers() { return userList; }
+
     void setChannels(std::vector<std::string> channels) {
         this->channelList = channels;
     }
 
+    std::vector<std::string> getChannels() { return channelList; }
+
     // State
     std::string getUsername() { return username; }
 
@@ -923,6 +944,8 @@ private:
             ctx->showChat("[BROADCAST]: " + text);
             break;
         case 0x07: // channel info
+            ctx->setCurrentChannel(username);
+            ctx->showInfo("Joined channel " + username);
             break;
         case 0x09: // user flags update
             break;
@@ -1469,11 +1492,200 @@ class ConnectState : public NetworkState {
     }
 };
 
-static void goOnline() {
-    Context *ctx = new Context();
-    ctx->setState(new ConnectState());
-    while (true) {
-        ctx->doOneStep();
-        sleep(1);
+extern gcn::Gui *Gui;
+static gcn::Container *onlineServiceContainer;
+static gcn::ScrollArea *messageArea;
+static gcn::ScrollArea *usersArea;
+static gcn::ScrollArea *friendsArea;
+static gcn::ScrollArea *channelsArea;
+static gcn::ScrollArea *gamelistArea;
+static gcn::TextField *chatInput;
+static gcn::Window *loginWindow;
+static gcn::Container *loginWindowContainer;
+static gcn::TextField *username;
+static gcn::TextField *password;
+
+static Context *ctx;
+
+class ChatInputListener : public gcn::ActionListener {
+    virtual void action(const std::string &) {
+        ctx->sendText(chatInput->getText());
+        chatInput->setText("");
     }
+};
+
+class UsernameInputListener : public gcn::ActionListener {
+    virtual void action(const std::string &) {
+        ctx->setUsername(username->getText());
+    }
+};
+
+class PasswordInputListener : public gcn::ActionListener {
+    virtual void action(const std::string &) {
+        ctx->setPassword(password->getText());
+    }
+};
+
+static void GoOnline() {
+    std::string nc, rc;
+    GetDefaultTextColors(nc, rc);
+
+    gcn::Widget *oldTop = Gui->getTop();
+    Gui->setUseDirtyDrawing(false);
+
+    ctx = new Context();
+    ctx->setState(new ConnectState());
+
+    onlineServiceContainer = new gcn::Container();
+    onlineServiceContainer->setDimension(gcn::Rectangle(0, 0, Video.Width, Video.Height));
+    onlineServiceContainer->setOpaque(false);
+    Gui->setTop(onlineServiceContainer);
+
+    static int chatInputHeight = 24;
+    messageArea = new gcn::ScrollArea(new gcn::TextBox());
+    messageArea->setBackgroundColor(gcn::Color(200, 200, 120));
+    messageArea->setSize(Video.Width * 0.7, Video.Height * 0.7 - chatInputHeight);
+    static_cast<gcn::TextBox*>(messageArea->getContent())->setEditable(false);
+    onlineServiceContainer->add(messageArea, Video.Width * 0.3, Video.Height * 0.3);
+
+    chatInput = new gcn::TextField();
+    chatInput->setSize(Video.Width * 0.7, chatInputHeight);
+    onlineServiceContainer->add(chatInput, Video.Width * 0.3, Video.Height * 0.7 - chatInputHeight);
+    chatInput->addActionListener(new ChatInputListener());
+
+    usersArea = new gcn::ScrollArea(new gcn::TextBox());
+    usersArea->setBackgroundColor(gcn::Color(120, 200, 200));
+    usersArea->setSize(Video.Width * 0.1, Video.Height * 0.7);
+    static_cast<gcn::TextBox*>(usersArea->getContent())->setEditable(false);
+    onlineServiceContainer->add(usersArea, Video.Width * 0.2, Video.Height * 0.3);
+
+    friendsArea = new gcn::ScrollArea(new gcn::TextBox());
+    friendsArea->setBackgroundColor(gcn::Color(200, 120, 200));
+    friendsArea->setSize(Video.Width * 0.1, Video.Height * 0.5);
+    static_cast<gcn::TextBox*>(friendsArea->getContent())->setEditable(false);
+    onlineServiceContainer->add(friendsArea, 0, 0);
+
+    channelsArea = new gcn::ScrollArea(new gcn::TextBox());
+    channelsArea->setBackgroundColor(gcn::Color(255, 255, 255));
+    channelsArea->setSize(Video.Width * 0.1, Video.Height * 0.5);
+    static_cast<gcn::TextBox*>(channelsArea->getContent())->setEditable(false);
+    onlineServiceContainer->add(channelsArea, 0, Video.Height * 0.5);
+
+    gamelistArea = new gcn::ScrollArea(new gcn::TextBox());
+    gamelistArea->setBackgroundColor(gcn::Color(120, 120, 120));
+    gamelistArea->setSize(Video.Width * 0.8, Video.Height * 0.3);
+    static_cast<gcn::TextBox*>(gamelistArea->getContent())->setEditable(false);
+    onlineServiceContainer->add(gamelistArea, Video.Width * 0.2, 0);
+
+    loginWindow = new gcn::Window();
+    loginWindow->setBaseColor(gcn::Color(120, 120, 120, 120));
+    loginWindow->setCaption("Username / Password");
+    loginWindow->setWidth(Video.Width / 2);
+    loginWindow->setHeight(chatInputHeight * 8);
+
+    loginWindowContainer = new gcn::Container();
+    loginWindowContainer->setWidth(loginWindow->getWidth());
+    loginWindowContainer->setHeight(loginWindow->getHeight());
+    loginWindowContainer->setOpaque(false);
+    loginWindow->setContent(loginWindowContainer);
+
+    username = new gcn::TextField();
+    username->setSize(Video.Width * 0.3, chatInputHeight);
+    username->addActionListener(new UsernameInputListener());
+    loginWindowContainer->add(username, Video.Width * 0.1, chatInputHeight);
+
+    password = new gcn::TextField();
+    password->setSize(Video.Width * 0.3, chatInputHeight);
+    password->addActionListener(new PasswordInputListener());
+    loginWindowContainer->add(password, Video.Width * 0.1, chatInputHeight * 4);
+
+    onlineServiceContainer->add(loginWindow, Video.Width / 4, Video.Height / 2 - chatInputHeight * 4);
+
+    SetVideoSync();
+    GameCursor = UI.Point.Cursor;
+    InterfaceState = IfaceStateNormal;
+    UI.SelectedViewport = UI.Viewports;
+    while (1) {
+        ctx->doOneStep();
+
+        if (!ctx->getCurrentChannel().empty()) {
+            loginWindow->setVisible(false);
+
+            if ((FrameCounter % (FRAMES_PER_SECOND * 5)) == 0) {
+                ctx->refreshGames();
+                ctx->refreshFriends();
+            }
+
+            if ((FrameCounter % (FRAMES_PER_SECOND * 1)) == 0) {
+                static_cast<gcn::TextBox*>(gamelistArea->getContent())->setText("");
+                for (auto g : ctx->getGames()) {
+                    static_cast<gcn::TextBox*>(gamelistArea->getContent())->addRow(g->getMap() + " " +
+                                                                                   g->getCreator() + " " +
+                                                                                   g->getGameType() + " " +
+                                                                                   g->getGameSettings() + " " +
+                                                                                   std::to_string(g->maxPlayers()));
+                }
+
+                static_cast<gcn::TextBox*>(usersArea->getContent())->setText("");
+                for (auto u : ctx->getUsers()) {
+                    static_cast<gcn::TextBox*>(usersArea->getContent())->addRow(u);
+                }
+
+                static_cast<gcn::TextBox*>(channelsArea->getContent())->setText("");
+                for (auto u : ctx->getChannels()) {
+                    static_cast<gcn::TextBox*>(channelsArea->getContent())->addRow(u);
+                }
+
+                static_cast<gcn::TextBox*>(friendsArea->getContent())->setText("");
+                for (auto u : ctx->getFriends()) {
+                    static_cast<gcn::TextBox*>(friendsArea->getContent())->addRow(u->getName() + ", " + u->getStatus() + ", " + u->getProduct());
+                }
+            }
+        }
+
+        if (!ctx->getInfo().empty()) {
+            std::string info = ctx->getInfo().front();
+            ctx->getInfo().pop();
+            while (!info.empty()) {
+                static_cast<gcn::TextBox*>(messageArea->getContent())->addRow(info);
+            }
+        }
+
+        Gui->draw();
+        DrawCursor();
+        Invalidate();
+        RealizeVideoMemory();
+
+        WaitEventsOneFrame();
+    }
+
+    CleanModules();
+    LoadCcl(Parameters::Instance.luaStartFilename); // Reload the main config file
+    // PreMenuSetup();
+
+    InterfaceState = IfaceStateMenu;
+    GameCursor = UI.Point.Cursor;
+
+    Gui->setTop(oldTop);
+    SetDefaultTextColors(nc, rc);
+
+    Video.ClearScreen();
+    Invalidate();
+
+    delete onlineServiceContainer;
+    delete messageArea->getContent();
+    delete messageArea;
+    delete usersArea->getContent();
+    delete usersArea;
+    delete friendsArea->getContent();
+    delete friendsArea;
+    delete channelsArea->getContent();
+    delete channelsArea;
+    delete gamelistArea->getContent();
+    delete gamelistArea;
+    delete chatInput;
+    delete loginWindow;
+    delete loginWindowContainer;
+    delete username;
+    delete password;
 }

From f4091c5c6b53514583bcdedb3e02836a5bde82a2 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Tue, 28 Jul 2020 23:24:17 +0200
Subject: [PATCH 07/70] add missing header, expose GoOnline to lua

---
 CMakeLists.txt                 |  2 +
 src/include/online_service.h   |  6 +++
 src/network/online_service.cpp |  4 +-
 src/network/xsha1.h            | 77 ++++++++++++++++++++++++++++++++++
 src/tolua/online_service.pkg   |  3 ++
 5 files changed, 91 insertions(+), 1 deletion(-)
 create mode 100644 src/include/online_service.h
 create mode 100644 src/network/xsha1.h
 create mode 100644 src/tolua/online_service.pkg

diff --git a/CMakeLists.txt b/CMakeLists.txt
index a9ebc8a64..d60679d1e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -363,6 +363,7 @@ source_group(win32 FILES ${win32_SRCS})
 set(tolua_FILES
 	src/tolua/ai.pkg
 	src/tolua/editor.pkg
+	src/tolua/online_service.pkg
 	src/tolua/font.pkg
 	src/tolua/game.pkg
 	src/tolua/map.pkg
@@ -527,6 +528,7 @@ set(stratagus_generic_HDRS
 	src/include/cursor.h
 	src/include/depend.h
 	src/include/editor.h
+	src/include/online_service.h
 	src/include/font.h
 	src/include/game.h
 	src/include/icons.h
diff --git a/src/include/online_service.h b/src/include/online_service.h
new file mode 100644
index 000000000..12d14f22e
--- /dev/null
+++ b/src/include/online_service.h
@@ -0,0 +1,6 @@
+#ifndef __ONLINE_SERVICE_H__
+#define __ONLINE_SERVICE_H__
+
+void GoOnline();
+
+#endif // !__EDITOR_H__
diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index c383079df..ced60ff42 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1,3 +1,5 @@
+#include "online_service.h"
+
 #include <arpa/inet.h>
 #include <clocale>
 #include <cstdio>
@@ -1526,7 +1528,7 @@ class PasswordInputListener : public gcn::ActionListener {
     }
 };
 
-static void GoOnline() {
+void GoOnline() {
     std::string nc, rc;
     GetDefaultTextColors(nc, rc);
 
diff --git a/src/network/xsha1.h b/src/network/xsha1.h
new file mode 100644
index 000000000..75bfb3e4c
--- /dev/null
+++ b/src/network/xsha1.h
@@ -0,0 +1,77 @@
+/*
+MBNCSUtil -- Managed Battle.net Authentication Library
+Copyright (C) 2005-2008 by Robert Paveza
+X-SHA-1 ported to C by wjlafrance, January 3rd 2013.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1.) Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+2.) Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+3.) The name of the author may not be used to endorse or promote products derived
+from this software without specific prior written permission.
+
+See LICENSE.TXT that should have accompanied this software for full terms and
+conditions.
+*/
+
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+
+uint32_t ROL(uint32_t val, uint32_t shift) {
+    shift &= 0x1f;
+    val = (val >> (0x20 - shift)) | (val << shift);
+    return val;
+}
+
+void xsha1_calcHashDat(uint32_t* data, uint32_t* result) {
+    for (int i = 16; i < 80; i++) {
+        data[i] = ROL(1, (int) (data[i-16] ^ data[i-8] ^ data[i-14] ^ data[i-3]) % 32);
+    }
+
+    uint32_t A = 0x67452301;
+    uint32_t B = 0xefcdab89;
+    uint32_t C = 0x98badcfe;
+    uint32_t D = 0x10325476;
+    uint32_t E = 0xc3d2e1f0;
+
+    uint32_t temp = 0;
+    for (int i = 0; i < 20; i++) {
+        temp = *data++ + ROL(A, 5) + E + ((B & C) | (~B & D)) + 0x5A827999;
+        E = D; D = C; C = ROL(B, 30); B = A; A = temp;
+    }
+
+    for (int i = 0; i < 20; i++) {
+        temp = (D ^ C ^ B) + E + ROL(temp, 5) + *data++ + 0x6ed9eba1;
+        E = D; D = C; C = ROL(B, 30); B = A; A = temp;
+    }
+
+    for (int i = 0; i < 20; i++) {
+        temp = *data++ + ROL(temp, 5) + E + ((C & B) | (D & C) | (D & B)) - 0x70E44324;
+        E = D; D = C; C = ROL(B, 30); B = A; A = temp;
+    }
+
+    for (int i = 0; i < 20; i++) {
+        temp = (D ^ C ^ B) + E + ROL(temp, 5) + *data++ - 0x359d3e2a;
+        E = D; D = C; C = ROL(B, 30); B = A; A = temp;
+    }
+
+    result[0] = A + 0x67452301;
+    result[1] = B + 0xefcdab89;
+    result[2] = C + 0x98badcfe;
+    result[3] = D + 0x10325476;
+    result[4] = E + 0xc3d2e1f0;
+}
+
+void xsha1_calcHashBuf(const char* input, size_t length, uint32_t* result) {
+    void *dataptr = malloc(1024);
+    memset(dataptr, 0, 1024);
+    uint32_t *data = (uint32_t *) dataptr;
+    memcpy(data, input, length);
+    xsha1_calcHashDat(data, result);
+    free(dataptr);
+}
diff --git a/src/tolua/online_service.pkg b/src/tolua/online_service.pkg
new file mode 100644
index 000000000..831419847
--- /dev/null
+++ b/src/tolua/online_service.pkg
@@ -0,0 +1,3 @@
+$#include "online_service.h"
+
+void GoOnline();

From 3f61f6102ad75ddeddabf6170b0ad40d1b7a9e89 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Tue, 28 Jul 2020 23:26:24 +0200
Subject: [PATCH 08/70] open the udp port

---
 src/network/online_service.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index ced60ff42..33a8f29f2 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1421,6 +1421,12 @@ class ConnectState : public NetworkState {
             ctx->setState(new DisconnectedState("TCP connect failed"));
             return;
         }
+
+        if (!ctx->getUDPSocket()->Open(*ctx->getHost())) {
+            ctx->setState(new DisconnectedState("UDP open failed"));
+            return;
+        }
+
         // Send proto byte
         ctx->getTCPSocket()->Send("\x1", 1);
 

From 5d3eadd8edbe7dfaa554364539505716035c47f2 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Wed, 29 Jul 2020 23:40:56 +0200
Subject: [PATCH 09/70] begin connection

---
 src/network/online_service.cpp | 42 ++++++++++++++++++++--------------
 1 file changed, 25 insertions(+), 17 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 33a8f29f2..f55b95497 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -214,13 +214,14 @@ public:
         *(buf + pos) = data;
         pos++;
     };
-    void serialize(const char* str, int len) {
-        ensureSpace(len);
-        memcpy(buf + pos, str, len);
-        pos += len;
+    void serialize32(const char* str) {
+        assert(strlen(str) == 4);
+        uint32_t value;
+        memcpy(&value, str, 4);
+        serialize32(value);
     };
     void serialize(const char* str) {
-        int len = strlen(str);
+        int len = strlen(str) + 1; // include NULL byte
         ensureSpace(len);
         memcpy(buf + pos, str, len);
         pos += len;
@@ -238,12 +239,12 @@ private:
     uint8_t *getBuffer() {
         // insert length to make it a valid buffer
         uint16_t *view = reinterpret_cast<uint16_t *>(buf + length_pos);
-        *view = htons(pos);
+        *view = pos;
         return buf;
     };
 
     void ensureSpace(size_t required) {
-        if (pos + required < sz) {
+        if (pos + required >= sz) {
             sz = sz * 2;
             buf = (uint8_t*) realloc(buf, sz);
             assert(buf != NULL);
@@ -640,7 +641,7 @@ public:
     void joinGame(std::string name, std::string pw) {
         // C>S 0x22 SID_NOTIFYJOIN
         BNCSOutputStream msg(0x09);
-        msg.serialize("W2BN", 4);
+        msg.serialize32("W2BN");
         msg.serialize32(0x4f);
         msg.serialize(name.c_str());
         msg.serialize(pw.c_str());
@@ -1412,19 +1413,23 @@ class S2C_SID_PING : public NetworkState {
 class ConnectState : public NetworkState {
     virtual void doOneStep(Context *ctx) {
         // Connect
-        ctx->getTCPSocket()->Open(*ctx->getHost());
-        if (ctx->getTCPSocket()->IsValid() == false) {
+        
+        if (!ctx->getTCPSocket()->Open(CHost("0.0.0.0", 6113))) { // TODO...
             ctx->setState(new DisconnectedState("TCP open failed"));
             return;
         }
-        if (ctx->getTCPSocket()->Connect(*ctx->getHost()) == false) {
+        if (!ctx->getTCPSocket()->IsValid()) {
+            ctx->setState(new DisconnectedState("TCP not valid"));
+            return;
+        }
+        if (!ctx->getTCPSocket()->Connect(*ctx->getHost())) {
             ctx->setState(new DisconnectedState("TCP connect failed"));
             return;
         }
-
         if (!ctx->getUDPSocket()->Open(*ctx->getHost())) {
-            ctx->setState(new DisconnectedState("UDP open failed"));
-            return;
+            std::cerr << "UDP open failed" << std::endl;
+            // ctx->setState(new DisconnectedState("UDP open failed"));
+            // return;
         }
 
         // Send proto byte
@@ -1435,10 +1440,10 @@ class ConnectState : public NetworkState {
         // (UINT32) Protocol ID
         buffer.serialize32(0x00);
         // (UINT32) Platform code
-        buffer.serialize("IX86", 4);
+        buffer.serialize32("IX86");
         // (UINT32) Product code - we'll just use W2BN and encode what we are in
         // the version
-        buffer.serialize("W2BN", 4);
+        buffer.serialize32("W2BN");
         // (UINT32) Version byte - just use the last from W2BN
         buffer.serialize32(0x4f);
         // (UINT32) Language code
@@ -1480,7 +1485,7 @@ class ConnectState : public NetworkState {
                 if (localtime != 0) {
                     std::time_t localtime_since_epoch = std::mktime(localtime);
                     if (localtime_since_epoch != -1) {
-                        bias = std::difftime(utctime_since_epoch, localtime_since_epoch) / 60;
+                        bias = (localtime_since_epoch - utctime_since_epoch) / 60;
                     }
                 }
             }
@@ -1538,6 +1543,8 @@ void GoOnline() {
     std::string nc, rc;
     GetDefaultTextColors(nc, rc);
 
+    const EventCallback *old_callbacks = GetCallbacks();
+
     gcn::Widget *oldTop = Gui->getTop();
     Gui->setUseDirtyDrawing(false);
 
@@ -1674,6 +1681,7 @@ void GoOnline() {
     InterfaceState = IfaceStateMenu;
     GameCursor = UI.Point.Cursor;
 
+    SetCallbacks(old_callbacks);
     Gui->setTop(oldTop);
     SetDefaultTextColors(nc, rc);
 

From 70e44191b3e6f00663b0b19cddc1b55284df89d6 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 30 Jul 2020 00:07:35 +0200
Subject: [PATCH 10/70] AUTHREQ

---
 src/network/online_service.cpp | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index f55b95497..9a696b299 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -90,19 +90,19 @@ public:
     uint16_t read16() {
         uint16_t byte = ntohs(reinterpret_cast<uint16_t *>(buffer + pos)[0]);
         consumeData(2);
-        return byte;
+        return ntohs(byte);
     }
 
     uint32_t read32() {
         uint32_t byte = ntohs(reinterpret_cast<uint32_t *>(buffer + pos)[0]);
         consumeData(4);
-        return byte;
+        return ntohl(byte);
     }
 
     uint64_t read64() {
         uint64_t byte = ntohs(reinterpret_cast<uint64_t *>(buffer + pos)[0]);
         consumeData(8);
-        return byte;
+        return ntohl(byte & (uint32_t)-1) | ntohl(byte >> 32);
     }
 
     bool readBool8() {
@@ -142,15 +142,21 @@ public:
         assert(read8() == 0xff);
         uint8_t msgId = read8();
         uint16_t len = read16();
-        avail += this->sock->Recv(buffer + avail, len - 4);
-        if (avail < len) {
-            // Didn't receive full message on the socket, yet. Reset position so
-            // this method can be used to try again
-            pos = 0;
-            return -1;
-        } else {
-            return 0;
+        // we still need to have len in total for this message, so if we have
+        // more available than len minus the current position and minus the
+        // first 4 bytes that we already consumed, we'll have enough
+        long needed = len - avail + pos - 4;
+        if (needed > 0) {
+            long got = this->sock->Recv(buffer + avail, needed);
+            avail += got;
+            if (got < needed) {
+                // Didn't receive full message on the socket, yet. Reset position so
+                // this method can be used to try again
+                pos = 0;
+                return -1;
+            }
         }
+        return msgId;
     };
 
 private:
@@ -1369,6 +1375,7 @@ class S2C_SID_AUTH_INFO : public NetworkState {
                     exeInfo += Parameters::Instance.applicationName;
                 }
             }
+            exeInfo += " ";
             exeInfo += StratagusLastModifiedDate;
             exeInfo += " ";
             exeInfo += StratagusLastModifiedTime;

From 82dbd03b028ac3b3a1d56b9ffd2d47da412b2f4d Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Sun, 2 Aug 2020 10:20:53 +0200
Subject: [PATCH 11/70] fix creating account and get until logging in

---
 src/network/online_service.cpp | 76 +++++++++++++++++++++++-----------
 src/network/xsha1.h            | 14 +++++--
 2 files changed, 62 insertions(+), 28 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 9a696b299..bf2cb8101 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -48,23 +48,23 @@ public:
         this->sock = socket;
         this->bufsize = 1024;
         this->buffer = (char*)calloc(sizeof(char), bufsize);
-        this->avail = 0;
+        this->received_bytes = 0;
         this->pos = 0;
     };
     ~BNCSInputStream() {};
 
     std::string readString() {
-        if (avail == 0) {
+        if (received_bytes == 0) {
             return NULL;
         }
         std::stringstream strstr;
         int i = pos;
         char c;
-        while ((c = buffer[i]) != '\0' && i < avail) {
+        while ((c = buffer[i]) != '\0' && i < received_bytes) {
             strstr.put(c);
             i += 1;
         }
-        consumeData(i);
+        consumeData(i + 1 - pos);
         return strstr.str();
     };
 
@@ -135,21 +135,27 @@ public:
         //  (UINT8) Message ID
         // (UINT16) Message length, including this header
         //   (VOID) Message data
-        avail += this->sock->Recv(buffer + avail, 4 - avail);
-        if (avail < 4) {
+        received_bytes += this->sock->Recv(buffer + received_bytes, 4 - received_bytes);
+        if (received_bytes != 4) {
             return -1;
         }
+        assert(received_bytes == 4);
         assert(read8() == 0xff);
         uint8_t msgId = read8();
         uint16_t len = read16();
         // we still need to have len in total for this message, so if we have
         // more available than len minus the current position and minus the
         // first 4 bytes that we already consumed, we'll have enough
-        long needed = len - avail + pos - 4;
-        if (needed > 0) {
-            long got = this->sock->Recv(buffer + avail, needed);
-            avail += got;
-            if (got < needed) {
+        assert(pos == 4);
+        long needed = len - received_bytes;
+        if (needed != 0) {
+            assert(needed > 0);
+            if (len >= bufsize) {
+                buffer = (char*)realloc(buffer, sizeof(char) * len + 1);
+                bufsize = len + 1;
+            }
+            received_bytes += this->sock->Recv(buffer + received_bytes, needed);
+            if (received_bytes < len) {
                 // Didn't receive full message on the socket, yet. Reset position so
                 // this method can be used to try again
                 pos = 0;
@@ -159,6 +165,11 @@ public:
         return msgId;
     };
 
+    void finishMessage() {
+        received_bytes = 0;
+        pos = 0;
+    }
+
 private:
     void consumeData(int bytes) {
         pos += bytes;
@@ -166,7 +177,7 @@ private:
 
     CTCPSocket *sock;
     char *buffer;
-    int avail;
+    int received_bytes;
     int pos;
     int bufsize;
 };
@@ -584,6 +595,7 @@ public:
         this->host = new CHost("127.0.0.1", 6112); // TODO: parameterize
         this->clientToken = MyRand();
         this->username = "";
+        setPassword("");
     }
 
     ~Context() {
@@ -683,7 +695,7 @@ public:
         this->extendedInfoValues[id] = values;
     }
 
-    std::queue<std::string> getInfo() { return info; }
+    std::queue<std::string> *getInfo() { return &info; }
 
     void showInfo(std::string arg) { info.push("*** " + arg + " ***"); }
 
@@ -733,8 +745,14 @@ public:
     }
 
     void setPassword(std::string pw) {
-        xsha1_calcHashBuf(pw.c_str(), pw.length(), password);
-        xsha1_calcHashDat(password, password2);
+        if (pw.empty()) {
+            for (int i = 0; i < sizeof(password); i++) {
+                this->password[i] = 0;
+            }
+        } else {
+            xsha1_calcHashBuf(pw.c_str(), pw.length(), password);
+            xsha1_calcHashDat(password, 5, password2);
+        }
     }
 
     // Protocol
@@ -756,6 +774,7 @@ public:
     void doOneStep() { this->state->doOneStep(this); }
 
     void setState(NetworkState* newState) {
+        std::cout << "new state" << std::endl;
         assert (newState != this->state);
         if (this->state != NULL) {
             delete this->state;
@@ -874,6 +893,8 @@ public:
                 // S>C 0x67 SID_FRIENDSADD
                 std::cout << "Unhandled message ID: " << std::hex << msg << std::endl;
             }
+
+            ctx->getMsgIStream()->finishMessage();
         }
     }
 
@@ -998,6 +1019,7 @@ class S2C_GETCHANNELLIST : public NetworkState {
             }
 
             std::vector<std::string> channels = ctx->getMsgIStream()->readStringlist();
+            ctx->getMsgIStream()->finishMessage();
             ctx->setChannels(channels);
 
             // request our user info and refresh the active games list
@@ -1026,6 +1048,7 @@ class S2C_ENTERCHAT : public NetworkState {
             std::string uniqueName = ctx->getMsgIStream()->readString();
             std::string statString = ctx->getMsgIStream()->readString();
             std::string accountName = ctx->getMsgIStream()->readString();
+            ctx->getMsgIStream()->finishMessage();
 
             ctx->setUsername(uniqueName);
             if (!statString.empty()) {
@@ -1114,6 +1137,7 @@ class S2C_CREATEACCOUNT2 : public NetworkState {
 
             uint32_t status = ctx->getMsgIStream()->read32();
             std::string nameSugg = ctx->getMsgIStream()->readString();
+            ctx->getMsgIStream()->finishMessage();
 
             if (!nameSugg.empty()) {
                 nameSugg = " (try username: " + nameSugg + ")";
@@ -1190,6 +1214,7 @@ class S2C_LOGONRESPONSE2 : public NetworkState {
             }
 
             uint32_t status = ctx->getMsgIStream()->read32();
+            ctx->getMsgIStream()->finishMessage();
 
             switch (status) {
             case 0x00:
@@ -1197,15 +1222,18 @@ class S2C_LOGONRESPONSE2 : public NetworkState {
                 ctx->setState(new S2C_PKT_SERVERPING());
                 return;
             case 0x01:
+            case 0x010000:
                 ctx->showInfo("Account does not exist, creating it...");
                 createAccount(ctx);
                 return;
             case 0x02:
+            case 0x020000:
                 ctx->showInfo("Incorrect password");
                 ctx->setPassword("");
                 ctx->setState(new C2S_LOGONRESPONSE2());
                 return;
             case 0x06:
+            case 0x060000:
                 ctx->showInfo("Account closed: " + ctx->getMsgIStream()->readString());
                 ctx->setPassword("");
                 ctx->setState(new C2S_LOGONRESPONSE2());
@@ -1252,7 +1280,7 @@ void C2S_LOGONRESPONSE2::doOneStep(Context *ctx) {
         data[5] = pw[3];
         data[6] = pw[4];
         uint32_t sendHash[5];
-        xsha1_calcHashDat(data, sendHash);
+        xsha1_calcHashDat(data, 7, sendHash);
         for (int i = 0; i < 20; i++) {
             logon.serialize8(reinterpret_cast<uint8_t*>(sendHash)[i]);
         }
@@ -1279,6 +1307,7 @@ class S2C_SID_AUTH_CHECK : public NetworkState {
 
             uint32_t result = ctx->getMsgIStream()->read32();
             std::string info = ctx->getMsgIStream()->readString();
+            ctx->getMsgIStream()->finishMessage();
 
             switch (result) {
             case 0x000:
@@ -1344,6 +1373,7 @@ class S2C_SID_AUTH_INFO : public NetworkState {
             uint64_t mpqFiletime = ctx->getMsgIStream()->readFiletime();
             std::string mpqFilename = ctx->getMsgIStream()->readString();
             std::string formula = ctx->getMsgIStream()->readString();
+            ctx->getMsgIStream()->finishMessage();
 
             // immediately respond with pkt_conntest2 udp msg
             BNCSOutputStream conntest(0x09);
@@ -1406,6 +1436,7 @@ class S2C_SID_PING : public NetworkState {
                 ctx->setState(new DisconnectedState(error));
             }
             uint32_t pingValue = ctx->getMsgIStream()->read32();
+            ctx->getMsgIStream()->finishMessage();
 
             // immediately respond with C2S_SID_PING
             BNCSOutputStream buffer(0x25);
@@ -1611,12 +1642,12 @@ void GoOnline() {
     loginWindowContainer->setOpaque(false);
     loginWindow->setContent(loginWindowContainer);
 
-    username = new gcn::TextField();
+    username = new gcn::TextField("");
     username->setSize(Video.Width * 0.3, chatInputHeight);
     username->addActionListener(new UsernameInputListener());
     loginWindowContainer->add(username, Video.Width * 0.1, chatInputHeight);
 
-    password = new gcn::TextField();
+    password = new gcn::TextField("");
     password->setSize(Video.Width * 0.3, chatInputHeight);
     password->addActionListener(new PasswordInputListener());
     loginWindowContainer->add(password, Video.Width * 0.1, chatInputHeight * 4);
@@ -1665,12 +1696,9 @@ void GoOnline() {
             }
         }
 
-        if (!ctx->getInfo().empty()) {
-            std::string info = ctx->getInfo().front();
-            ctx->getInfo().pop();
-            while (!info.empty()) {
-                static_cast<gcn::TextBox*>(messageArea->getContent())->addRow(info);
-            }
+        while (!ctx->getInfo()->empty()) {
+            static_cast<gcn::TextBox*>(messageArea->getContent())->addRow(ctx->getInfo()->front());
+            ctx->getInfo()->pop();
         }
 
         Gui->draw();
diff --git a/src/network/xsha1.h b/src/network/xsha1.h
index 75bfb3e4c..2479c0801 100644
--- a/src/network/xsha1.h
+++ b/src/network/xsha1.h
@@ -18,17 +18,18 @@ See LICENSE.TXT that should have accompanied this software for full terms and
 conditions.
 */
 
+#include <cassert>
 #include <stdint.h>
 #include <string.h>
 #include <stdlib.h>
 
-uint32_t ROL(uint32_t val, uint32_t shift) {
+static uint32_t ROL(uint32_t val, uint32_t shift) {
     shift &= 0x1f;
     val = (val >> (0x20 - shift)) | (val << shift);
     return val;
 }
 
-void xsha1_calcHashDat(uint32_t* data, uint32_t* result) {
+static void calcHash(uint32_t data[1024], uint32_t result[5]) {
     for (int i = 16; i < 80; i++) {
         data[i] = ROL(1, (int) (data[i-16] ^ data[i-8] ^ data[i-14] ^ data[i-3]) % 32);
     }
@@ -67,11 +68,16 @@ void xsha1_calcHashDat(uint32_t* data, uint32_t* result) {
     result[4] = E + 0xc3d2e1f0;
 }
 
-void xsha1_calcHashBuf(const char* input, size_t length, uint32_t* result) {
+void xsha1_calcHashBuf(const char* input, size_t length, uint32_t *result) {
+    assert(length < 1024);
     void *dataptr = malloc(1024);
     memset(dataptr, 0, 1024);
     uint32_t *data = (uint32_t *) dataptr;
     memcpy(data, input, length);
-    xsha1_calcHashDat(data, result);
+    calcHash(data, result);
     free(dataptr);
 }
+
+void xsha1_calcHashDat(uint32_t* data, size_t length, uint32_t *result) {
+    xsha1_calcHashBuf((const char*)data, length * 4, result);
+}

From b9395ed81b2fe38ecb7ba2b17bffff7c8e95cf3e Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Sun, 2 Aug 2020 10:22:11 +0200
Subject: [PATCH 12/70] only send chat when logged in, accept both user/pw on
 enter

---
 src/network/online_service.cpp | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index bf2cb8101..d25d5a259 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -774,7 +774,6 @@ public:
     void doOneStep() { this->state->doOneStep(this); }
 
     void setState(NetworkState* newState) {
-        std::cout << "new state" << std::endl;
         assert (newState != this->state);
         if (this->state != NULL) {
             delete this->state;
@@ -1560,19 +1559,23 @@ static Context *ctx;
 
 class ChatInputListener : public gcn::ActionListener {
     virtual void action(const std::string &) {
-        ctx->sendText(chatInput->getText());
-        chatInput->setText("");
+        if (!ctx->getCurrentChannel().empty()) {
+            ctx->sendText(chatInput->getText());
+            chatInput->setText("");
+        }
     }
 };
 
 class UsernameInputListener : public gcn::ActionListener {
     virtual void action(const std::string &) {
         ctx->setUsername(username->getText());
+        ctx->setPassword(password->getText());
     }
 };
 
 class PasswordInputListener : public gcn::ActionListener {
     virtual void action(const std::string &) {
+        ctx->setUsername(username->getText());
         ctx->setPassword(password->getText());
     }
 };

From 2540d6f256396e79ac1c6deaea3f242681ae1504 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Sun, 2 Aug 2020 10:53:10 +0200
Subject: [PATCH 13/70] simplify serialization code

---
 src/network/online_service.cpp | 28 ++++++++--------------------
 1 file changed, 8 insertions(+), 20 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index d25d5a259..502f53134 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -94,13 +94,13 @@ public:
     }
 
     uint32_t read32() {
-        uint32_t byte = ntohs(reinterpret_cast<uint32_t *>(buffer + pos)[0]);
+        uint32_t byte = ntohl(reinterpret_cast<uint32_t *>(buffer + pos)[0]);
         consumeData(4);
         return ntohl(byte);
     }
 
     uint64_t read64() {
-        uint64_t byte = ntohs(reinterpret_cast<uint64_t *>(buffer + pos)[0]);
+        uint64_t byte = reinterpret_cast<uint64_t *>(buffer + pos)[0];
         consumeData(8);
         return ntohl(byte & (uint32_t)-1) | ntohl(byte >> 32);
     }
@@ -208,30 +208,18 @@ public:
         *view = htonl(data);
         pos += sizeof(data);
     };
-    void serialize32(int32_t data) {
-        ensureSpace(sizeof(data));
-        int32_t *view = reinterpret_cast<int32_t *>(buf + pos);
-        *view = htonl(data);
-        pos += sizeof(data);
-    };
     void serialize16(uint16_t data) {
         ensureSpace(sizeof(data));
         uint16_t *view = reinterpret_cast<uint16_t *>(buf + pos);
         *view = htons(data);
         pos += sizeof(data);
     };
-    void serialize16(int16_t data) {
-        ensureSpace(sizeof(data));
-        uint16_t *view = reinterpret_cast<uint16_t *>(buf + pos);
-        *view = htons(data);
-        pos += sizeof(data);
-    };
     void serialize8(uint8_t data) {
         ensureSpace(sizeof(data));
         *(buf + pos) = data;
         pos++;
     };
-    void serialize32(const char* str) {
+    void serializeC32(const char* str) {
         assert(strlen(str) == 4);
         uint32_t value;
         memcpy(&value, str, 4);
@@ -639,8 +627,8 @@ public:
     void refreshGames() {
         // C>S 0x09 SID_GETADVLISTEX
         BNCSOutputStream getadvlistex(0x09);
-        getadvlistex.serialize16((uint16_t) 0x00); // all games
-        getadvlistex.serialize16((uint16_t) 0x01); // no sub game type
+        getadvlistex.serialize16(0x00); // all games
+        getadvlistex.serialize16(0x01); // no sub game type
         getadvlistex.serialize32(0xff80); // show all games
         getadvlistex.serialize32(0x00); // reserved field
         getadvlistex.serialize32(0xff); // return all games
@@ -659,7 +647,7 @@ public:
     void joinGame(std::string name, std::string pw) {
         // C>S 0x22 SID_NOTIFYJOIN
         BNCSOutputStream msg(0x09);
-        msg.serialize32("W2BN");
+        msg.serializeC32("W2BN");
         msg.serialize32(0x4f);
         msg.serialize(name.c_str());
         msg.serialize(pw.c_str());
@@ -1477,10 +1465,10 @@ class ConnectState : public NetworkState {
         // (UINT32) Protocol ID
         buffer.serialize32(0x00);
         // (UINT32) Platform code
-        buffer.serialize32("IX86");
+        buffer.serializeC32("IX86");
         // (UINT32) Product code - we'll just use W2BN and encode what we are in
         // the version
-        buffer.serialize32("W2BN");
+        buffer.serializeC32("W2BN");
         // (UINT32) Version byte - just use the last from W2BN
         buffer.serialize32(0x4f);
         // (UINT32) Language code

From ed0b5ed563c87c95ef1706b10d8ca02d46cc8397 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Sun, 2 Aug 2020 10:53:41 +0200
Subject: [PATCH 14/70] assume serverToken in network byte order

---
 src/network/online_service.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 502f53134..564688526 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1355,7 +1355,7 @@ class S2C_SID_AUTH_INFO : public NetworkState {
             uint32_t logonType = ctx->getMsgIStream()->read32();
             assert(logonType == 0x00); // only support Broken SHA-1 logon for now
             uint32_t serverToken = ctx->getMsgIStream()->read32();
-            ctx->serverToken = serverToken;
+            ctx->serverToken = htonl(serverToken); // keep in network order
             uint32_t udpValue = ctx->getMsgIStream()->read32();
             uint64_t mpqFiletime = ctx->getMsgIStream()->readFiletime();
             std::string mpqFilename = ctx->getMsgIStream()->readString();

From e3bec2ed49238b0f4e0e5bd725b364379fb1884c Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Sun, 2 Aug 2020 12:04:37 +0200
Subject: [PATCH 15/70] use pvpgn code for password hashing

---
 src/network/online_service.cpp |  64 ++--
 src/network/xsha1.h            | 547 ++++++++++++++++++++++++++++-----
 2 files changed, 509 insertions(+), 102 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 564688526..100f9ae6c 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -712,16 +712,6 @@ public:
 
     void setUsername(std::string arg) { username = arg; }
 
-    uint32_t* getPassword2() {
-        // we assume that any valid password has at least 1 non-null word hash
-        for (int i = 0; i < 5; i++) {
-            if (password[i] != 0) {
-                return password2;
-            }
-        }
-        return NULL;
-    }
-
     uint32_t* getPassword1() {
         // we assume that any valid password has at least 1 non-null word hash
         for (int i = 0; i < 5; i++) {
@@ -738,8 +728,7 @@ public:
                 this->password[i] = 0;
             }
         } else {
-            xsha1_calcHashBuf(pw.c_str(), pw.length(), password);
-            xsha1_calcHashDat(password, 5, password2);
+            pvpgn::sha1_hash(&password, pw.length(), pw.c_str());
         }
     }
 
@@ -781,7 +770,6 @@ private:
 
     std::string username;
     uint32_t password[5]; // xsha1 hash of password
-    uint32_t password2[5]; // xsha1 hash of password hash
 
     std::string currentChannel;
     std::set<std::string> userList;
@@ -878,7 +866,7 @@ public:
                 // TODO:
                 // S>C 0x68 SID_FRIENDSREMOVE
                 // S>C 0x67 SID_FRIENDSADD
-                std::cout << "Unhandled message ID: " << std::hex << msg << std::endl;
+                std::cout << "Unhandled message ID: 0x" << std::hex << msg << std::endl;
             }
 
             ctx->getMsgIStream()->finishMessage();
@@ -1026,6 +1014,13 @@ class S2C_ENTERCHAT : public NetworkState {
                 // try again next time
                 return;
             }
+            if (msg == 0x3a) {
+                // pvpgn seems to send a successful logonresponse again
+                uint32_t status = ctx->getMsgIStream()->read32();
+                assert(status == 0);
+                ctx->getMsgIStream()->finishMessage();
+                return;
+            }
             if (msg != 0x0a) {
                 std::string error = std::string("Expected SID_ENTERCHAT, got msg id ");
                 error += std::to_string(msg);
@@ -1091,7 +1086,7 @@ public:
             }
         } else {
             retries++;
-            if (retries < 5000) {
+            if (retries < 50) {
                 return;
             }
             // we're using a timeout of 1ms, so now we've been waiting at
@@ -1258,18 +1253,33 @@ void C2S_LOGONRESPONSE2::doOneStep(Context *ctx) {
         // (UINT32) Client Token
         // (UINT32) Server Token
         // (UINT32)[5] First password hash
-        uint32_t data[7];
-        data[0] = ctx->clientToken;
-        data[1] = ctx->serverToken;
-        data[2] = pw[0];
-        data[3] = pw[1];
-        data[4] = pw[2];
-        data[5] = pw[3];
-        data[6] = pw[4];
-        uint32_t sendHash[5];
-        xsha1_calcHashDat(data, 7, sendHash);
+        // The logic below is taken straight from pvpgn
+        struct {
+            pvpgn::bn_int ticks;
+            pvpgn::bn_int sessionkey;
+            pvpgn::bn_int passhash1[5];
+        } temp;
+        uint32_t passhash2[5];
+
+        pvpgn::bn_int_set(&temp.ticks, ntohl(ctx->clientToken));
+        pvpgn::bn_int_set(&temp.sessionkey, ntohl(ctx->serverToken));
+        pvpgn::hash_to_bnhash((pvpgn::t_hash const *)pw, temp.passhash1);
+        pvpgn::bnet_hash(&passhash2, sizeof(temp), &temp);	/* do the double hash */
+
+        // std::cout << std::endl << "Password 1 hash: ";
+        // for (int i = 0; i < 5; i++) {
+        //     std::cout << std::hex << pw[i] << " ";
+        // }
+        // std::cout << std::endl << "Password 2 hash: ";
+        // for (int i = 0; i < 5; i++) {
+        //     std::cout << std::hex << passhash2[i] << " ";
+        // }
+        // std::cout << std::endl;
+        // std::cout << "client token: " << *((uint32_t*)temp.ticks) << std::endl;
+        // std::cout << "server token: " << *((uint32_t*)temp.sessionkey) << std::endl;
+
         for (int i = 0; i < 20; i++) {
-            logon.serialize8(reinterpret_cast<uint8_t*>(sendHash)[i]);
+            logon.serialize8(reinterpret_cast<uint8_t*>(passhash2)[i]);
         }
         logon.serialize(user.c_str());
         logon.flush(ctx->getTCPSocket());
@@ -1645,6 +1655,8 @@ void GoOnline() {
 
     onlineServiceContainer->add(loginWindow, Video.Width / 4, Video.Height / 2 - chatInputHeight * 4);
 
+    username->requestFocus();
+
     SetVideoSync();
     GameCursor = UI.Point.Cursor;
     InterfaceState = IfaceStateNormal;
diff --git a/src/network/xsha1.h b/src/network/xsha1.h
index 2479c0801..5b4d31c0a 100644
--- a/src/network/xsha1.h
+++ b/src/network/xsha1.h
@@ -1,83 +1,478 @@
 /*
-MBNCSUtil -- Managed Battle.net Authentication Library
-Copyright (C) 2005-2008 by Robert Paveza
-X-SHA-1 ported to C by wjlafrance, January 3rd 2013.
+ * Copyright (C) 2020  The Stratagus Project
+ * Copyright (C) 1999  Descolada (dyn1-tnt9-237.chicago.il.ameritech.net)
+ * Copyright (C) 1999,2000,2001  Ross Combs (rocombs@cs.nmsu.edu)
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ */
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
 
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
+/* valid for 0<=n and w>0 , depends on 2's complement */
+#define ROTL(x,n,w) (((x)<<((n)&(w-1))) | ((x)>>(((-(n))&(w-1)))))
 
-1.) Redistributions of source code must retain the above copyright notice,
-this list of conditions and the following disclaimer.
-2.) Redistributions in binary form must reproduce the above copyright notice,
-this list of conditions and the following disclaimer in the documentation
-and/or other materials provided with the distribution.
-3.) The name of the author may not be used to endorse or promote products derived
-from this software without specific prior written permission.
+/* valid for 0<=n and w>0 , uses three mods and an ugly conditional */
+/* FIXME: and also a bug because it doesn't work on PPC */
+/*#define ROTL(x,n,w) (((n)%(w)) ? (((x)<<((n)%(w))) | ((x)>>((w)-((n)%(w))))) : (x))*/
 
-See LICENSE.TXT that should have accompanied this software for full terms and
-conditions.
-*/
+#define ROTL32(x,n) ROTL(x,n,32)
+#define ROTL16(x,n) ROTL(x,n,16)
 
-#include <cassert>
-#include <stdint.h>
-#include <string.h>
-#include <stdlib.h>
 
-static uint32_t ROL(uint32_t val, uint32_t shift) {
-    shift &= 0x1f;
-    val = (val >> (0x20 - shift)) | (val << shift);
-    return val;
-}
-
-static void calcHash(uint32_t data[1024], uint32_t result[5]) {
-    for (int i = 16; i < 80; i++) {
-        data[i] = ROL(1, (int) (data[i-16] ^ data[i-8] ^ data[i-14] ^ data[i-3]) % 32);
-    }
-
-    uint32_t A = 0x67452301;
-    uint32_t B = 0xefcdab89;
-    uint32_t C = 0x98badcfe;
-    uint32_t D = 0x10325476;
-    uint32_t E = 0xc3d2e1f0;
-
-    uint32_t temp = 0;
-    for (int i = 0; i < 20; i++) {
-        temp = *data++ + ROL(A, 5) + E + ((B & C) | (~B & D)) + 0x5A827999;
-        E = D; D = C; C = ROL(B, 30); B = A; A = temp;
-    }
-
-    for (int i = 0; i < 20; i++) {
-        temp = (D ^ C ^ B) + E + ROL(temp, 5) + *data++ + 0x6ed9eba1;
-        E = D; D = C; C = ROL(B, 30); B = A; A = temp;
-    }
-
-    for (int i = 0; i < 20; i++) {
-        temp = *data++ + ROL(temp, 5) + E + ((C & B) | (D & C) | (D & B)) - 0x70E44324;
-        E = D; D = C; C = ROL(B, 30); B = A; A = temp;
-    }
-
-    for (int i = 0; i < 20; i++) {
-        temp = (D ^ C ^ B) + E + ROL(temp, 5) + *data++ - 0x359d3e2a;
-        E = D; D = C; C = ROL(B, 30); B = A; A = temp;
-    }
-
-    result[0] = A + 0x67452301;
-    result[1] = B + 0xefcdab89;
-    result[2] = C + 0x98badcfe;
-    result[3] = D + 0x10325476;
-    result[4] = E + 0xc3d2e1f0;
-}
-
-void xsha1_calcHashBuf(const char* input, size_t length, uint32_t *result) {
-    assert(length < 1024);
-    void *dataptr = malloc(1024);
-    memset(dataptr, 0, 1024);
-    uint32_t *data = (uint32_t *) dataptr;
-    memcpy(data, input, length);
-    calcHash(data, result);
-    free(dataptr);
-}
-
-void xsha1_calcHashDat(uint32_t* data, size_t length, uint32_t *result) {
-    xsha1_calcHashBuf((const char*)data, length * 4, result);
+namespace pvpgn
+{
+	using bn_basic = std::uint8_t;
+	using bn_byte = bn_basic[1];
+	using bn_short = bn_basic[2];
+	using bn_int = bn_basic[4];
+	using bn_long = bn_basic[8];
+	using t_hash = std::uint32_t[5];
+
+	int bn_int_set(bn_int * dst, std::uint32_t src)
+	{
+		if (!dst)
+		{
+			return -1;
+		}
+
+		(*dst)[0] = (std::uint8_t)((src)& 0xff);
+		(*dst)[1] = (std::uint8_t)((src >> 8) & 0xff);
+		(*dst)[2] = (std::uint8_t)((src >> 16) & 0xff);
+		(*dst)[3] = (std::uint8_t)((src >> 24));
+		return 0;
+	}
+
+	int bn_int_nset(bn_int * dst, std::uint32_t src)
+	{
+		if (!dst)
+			{
+				return -1;
+			}
+
+		(*dst)[0] = (std::uint8_t)((src >> 24));
+		(*dst)[1] = (std::uint8_t)((src >> 16) & 0xff);
+		(*dst)[2] = (std::uint8_t)((src >> 8) & 0xff);
+		(*dst)[3] = (std::uint8_t)((src)& 0xff);
+		return 0;
+	}
+
+	std::uint32_t bn_int_get(bn_int const src)
+	{
+		std::uint32_t temp;
+
+		if (!src)
+		{
+			return 0;
+		}
+
+		temp = ((std::uint32_t)src[0]);
+		temp |= ((std::uint32_t)src[1]) << 8;
+		temp |= ((std::uint32_t)src[2]) << 16;
+		temp |= ((std::uint32_t)src[3]) << 24;
+		return temp;
+	}
+
+	typedef enum {
+		do_blizzard_hash,
+		do_sha1_hash
+	} t_hash_variant;
+
+	static void hash_init(t_hash * hash);
+	static void do_hash(t_hash * hash, std::uint32_t * tmp);
+	static void hash_set_16(std::uint32_t * dst, unsigned char const * src, unsigned int count, t_hash_variant hash_variant);
+
+
+	static void hash_init(t_hash * hash)
+	{
+		(*hash)[0] = 0x67452301;
+		(*hash)[1] = 0xefcdab89;
+		(*hash)[2] = 0x98badcfe;
+		(*hash)[3] = 0x10325476;
+		(*hash)[4] = 0xc3d2e1f0;
+	}
+
+
+	static void do_hash(t_hash * hash, std::uint32_t * tmp, t_hash_variant hash_variant)
+	{
+		unsigned int i;
+		std::uint32_t     a, b, c, d, e, g;
+
+		for (i = 0; i < 64; i++)
+		if (hash_variant == do_blizzard_hash)
+			tmp[i + 16] = ROTL32(1, tmp[i] ^ tmp[i + 8] ^ tmp[i + 2] ^ tmp[i + 13]);
+		else
+			tmp[i + 16] = ROTL32(tmp[i] ^ tmp[i + 8] ^ tmp[i + 2] ^ tmp[i + 13], 1);
+
+		a = (*hash)[0];
+		b = (*hash)[1];
+		c = (*hash)[2];
+		d = (*hash)[3];
+		e = (*hash)[4];
+
+		for (i = 0; i < 20 * 1; i++)
+		{
+			g = tmp[i] + ROTL32(a, 5) + e + ((b & c) | (~b & d)) + 0x5a827999;
+			e = d;
+			d = c;
+			c = ROTL32(b, 30);
+			b = a;
+			a = g;
+		}
+
+		for (; i < 20 * 2; i++)
+		{
+			g = (d ^ c ^ b) + e + ROTL32(g, 5) + tmp[i] + 0x6ed9eba1;
+			e = d;
+			d = c;
+			c = ROTL32(b, 30);
+			b = a;
+			a = g;
+		}
+
+		for (; i < 20 * 3; i++)
+		{
+			g = tmp[i] + ROTL32(g, 5) + e + ((c & b) | (d & c) | (d & b)) - 0x70e44324;
+			e = d;
+			d = c;
+			c = ROTL32(b, 30);
+			b = a;
+			a = g;
+		}
+
+		for (; i < 20 * 4; i++)
+		{
+			g = (d ^ c ^ b) + e + ROTL32(g, 5) + tmp[i] - 0x359d3e2a;
+			e = d;
+			d = c;
+			c = ROTL32(b, 30);
+			b = a;
+			a = g;
+		}
+
+		(*hash)[0] += g;
+		(*hash)[1] += b;
+		(*hash)[2] += c;
+		(*hash)[3] += d;
+		(*hash)[4] += e;
+	}
+
+
+	/*
+	 * Fill 16 elements of the array of 32 bit values with the bytes from
+	 * dst up to count in little endian order. Fill left over space with
+	 * zeros. In case of SHA1 hash variant a binary 1 is appended after
+	 * the actual data.
+	 */
+	static void hash_set_16(std::uint32_t * dst, unsigned char const * src, unsigned int count,
+		t_hash_variant hash_variant)
+	{
+		unsigned int i;
+		unsigned int pos;
+
+		for (pos = 0, i = 0; i < 16; i++)
+		{
+			dst[i] = 0;
+
+			if (hash_variant == do_blizzard_hash) {
+				if (pos < count)
+					dst[i] |= ((std::uint32_t)src[pos]);
+			}
+			else {
+				if (pos < count)
+					dst[i] |= ((std::uint32_t)src[pos]) << 24;
+				else if (pos == count)
+					dst[i] |= ((std::uint32_t)0x80000000);
+			}
+			pos++;
+
+			if (hash_variant == do_blizzard_hash) {
+				if (pos < count)
+					dst[i] |= ((std::uint32_t)src[pos]) << 8;
+			}
+			else {
+				if (pos < count)
+					dst[i] |= ((std::uint32_t)src[pos]) << 16;
+				else if (pos == count)
+					dst[i] |= ((std::uint32_t)0x800000);
+			}
+			pos++;
+
+			if (hash_variant == do_blizzard_hash) {
+				if (pos < count)
+					dst[i] |= ((std::uint32_t)src[pos]) << 16;
+			}
+			else {
+				if (pos < count)
+					dst[i] |= ((std::uint32_t)src[pos]) << 8;
+				else if (pos == count)
+					dst[i] |= ((std::uint32_t)0x8000);
+			}
+			pos++;
+
+			if (hash_variant == do_blizzard_hash) {
+				if (pos < count)
+					dst[i] |= ((std::uint32_t)src[pos]) << 24;
+			}
+			else {
+				if (pos < count)
+					dst[i] |= ((std::uint32_t)src[pos]);
+				else if (pos == count)
+					dst[i] |= ((std::uint32_t)0x80);
+			}
+			pos++;
+		}
+	}
+
+
+	extern int bnet_hash(t_hash * hashout, unsigned int size, void const * datain)
+	{
+		std::uint32_t tmp[64 + 16];
+		const unsigned char* data;
+		unsigned int inc;
+
+		if (!hashout)
+		{
+			return -1;
+		}
+		if (size > 0 && !datain)
+		{
+			return -1;
+		}
+
+		hash_init(hashout);
+
+		data = (const unsigned char*)datain;
+		while (size > 0)
+		{
+			if (size > 64)
+				inc = 64;
+			else
+				inc = size;
+
+			hash_set_16(tmp, data, inc, do_blizzard_hash);
+			do_hash(hashout, tmp, do_blizzard_hash);
+
+			data += inc;
+			size -= inc;
+		}
+
+		return 0;
+	}
+
+	static void hash_set_length(std::uint32_t * dst, unsigned int size){
+		std::uint32_t size_high = 0;
+		std::uint32_t size_low = 0;
+		unsigned int counter;
+		for (counter = 0; counter < size; counter++){
+			size_low += 8;
+			if (size_low == 0)
+				size_high++;
+		}
+
+		dst[14] |= ((size_high >> 24) & 0xff) << 24;
+		dst[14] |= ((size_high >> 16) & 0xff) << 16;
+		dst[14] |= ((size_high >> 8) & 0xff) << 8;
+		dst[14] |= ((size_high)& 0xff);
+
+		dst[15] |= ((size_low >> 24) & 0xff) << 24;
+		dst[15] |= ((size_low >> 16) & 0xff) << 16;
+		dst[15] |= ((size_low >> 8) & 0xff) << 8;
+		dst[15] |= ((size_low)& 0xff);
+	}
+
+	extern int sha1_hash(t_hash * hashout, unsigned int size, void const * datain)
+	{
+		std::uint32_t         tmp[64 + 16];
+		unsigned char const * data;
+		unsigned int          inc;
+		unsigned int          orgSize;
+
+		if (!hashout)
+		{
+			return -1;
+		}
+		if (size > 0 && !datain)
+		{
+			return -1;
+		}
+
+		hash_init(hashout);
+		orgSize = size;
+
+		data = (const unsigned char*)datain;
+		while (size > 0)
+		{
+			if (size >= 64)
+				inc = 64;
+			else
+				inc = size;
+
+			if (size >= 64)
+			{
+				hash_set_16(tmp, data, inc, do_sha1_hash);
+				do_hash(hashout, tmp, do_sha1_hash);
+			}
+			else if (size > 55){
+
+				hash_set_16(tmp, data, inc, do_sha1_hash);
+				do_hash(hashout, tmp, do_sha1_hash);
+
+				// now use blizz variant as we only wanna fill in zeros
+				hash_set_16(tmp, data, 0, do_blizzard_hash);
+				hash_set_length(tmp, orgSize);
+				do_hash(hashout, tmp, do_sha1_hash);
+			}
+			else{
+				hash_set_16(tmp, data, inc, do_sha1_hash);
+				hash_set_length(tmp, orgSize);
+				do_hash(hashout, tmp, do_sha1_hash);
+			}
+
+			data += inc;
+			size -= inc;
+		}
+
+		return 0;
+	}
+
+	extern int little_endian_sha1_hash(t_hash * hashout, unsigned int size, void const * datain)
+	{
+		bn_int value;
+		unsigned int i;
+		sha1_hash(hashout, size, datain);
+		for (i = 0; i < 5; i++)
+		{
+			bn_int_nset(&value, (*hashout)[i]);
+			(*hashout)[i] = bn_int_get(value);
+		}
+		return 0;
+	}
+
+	extern int hash_eq(t_hash const h1, t_hash const h2)
+	{
+		unsigned int i;
+
+		if (!h1 || !h2)
+		{
+			return -1;
+		}
+
+		for (i = 0; i < 5; i++) {
+			if (h1[i] != h2[i]) {
+				return 0;
+			}
+		}
+
+		return 1;
+	}
+
+
+	extern char const * hash_get_str(t_hash const hash)
+	{
+		static char  temp[8 * 5 + 1]; /* each of 5 ints to 8 chars + null */
+		unsigned int i;
+
+		if (!hash)
+		{
+			return NULL;
+		}
+
+		for (i = 0; i < 5; i++)
+			std::sprintf(&temp[i * 8], "%08x", hash[i]);
+
+		return temp;
+	}
+
+	extern char const * little_endian_hash_get_str(t_hash const hash)
+	{
+		bn_int value;
+		t_hash be_hash;
+		unsigned int i;
+		for (i = 0; i < 5; i++)
+		{
+			bn_int_nset(&value, hash[i]);
+			be_hash[i] = bn_int_get(value);
+		}
+		return hash_get_str(be_hash);
+	}
+
+
+	extern int hash_set_str(t_hash * hash, char const * str)
+	{
+		unsigned int i;
+
+		if (!hash)
+		{
+			return -1;
+		}
+		if (!*hash)
+		{
+			return -1;
+		}
+		if (!str)
+		{
+			return -1;
+		}
+		if (std::strlen(str) != 5 * 8)
+		{
+			return -1;
+		}
+
+		for (i = 0; i < 5; i++)
+		if (std::sscanf(&str[i * 8], "%8x", &(*hash)[i]) != 1)
+		{
+			return -1;
+		}
+
+		return 0;
+	}
+
+	void bnhash_to_hash(bn_int const * bnhash, t_hash * hash)
+	{
+		unsigned int i;
+
+		if (!bnhash)
+		{
+			return;
+		}
+		if (!hash)
+		{
+			return;
+		}
+
+		for (i = 0; i < 5; i++)
+			(*hash)[i] = bn_int_get(bnhash[i]);
+	}
+
+
+	void hash_to_bnhash(t_hash const * hash, bn_int * bnhash)
+	{
+		unsigned int i;
+
+		if (!bnhash)
+		{
+			return;
+		}
+		if (!hash)
+		{
+			return;
+		}
+
+		for (i = 0; i < 5; i++)
+			bn_int_set(&bnhash[i], (*hash)[i]);
+	}
+
 }

From 99e3e420d44abeef9ab9bdf4f106090aa6ab08f4 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Sun, 2 Aug 2020 15:26:54 +0200
Subject: [PATCH 16/70] use the metaserver host/port data for the online
 service

---
 src/include/master.h           |  2 ++
 src/network/online_service.cpp | 15 +++++++++++----
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/include/master.h b/src/include/master.h
index e0abb2d53..e15b6942f 100644
--- a/src/include/master.h
+++ b/src/include/master.h
@@ -63,6 +63,8 @@ public:
 	CClientLog *GetLastMessage() { return events.back(); }
 	int CreateGame(std::string desc, std::string map, std::string players);
 
+	CHost *GetMetaServer() { return new CHost(metaHost.c_str(), metaPort); }
+
 private:
 	CTCPSocket metaSocket;                     /// This is a TCP socket
 	std::string metaHost;                      /// Address of metaserver
diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 100f9ae6c..fa8a00a36 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1,4 +1,5 @@
 #include "online_service.h"
+#include "master.h"
 
 #include <arpa/inet.h>
 #include <clocale>
@@ -580,7 +581,7 @@ public:
         this->tcpSocket = new CTCPSocket();
         this->istream = new BNCSInputStream(tcpSocket);
         this->state = NULL;
-        this->host = new CHost("127.0.0.1", 6112); // TODO: parameterize
+        this->host = new CHost("127.0.0.1", 6112);
         this->clientToken = MyRand();
         this->username = "";
         setPassword("");
@@ -795,6 +796,7 @@ public:
     virtual void doOneStep(Context *ctx) {
         if (!hasPrinted) {
             std::cout << message << std::endl;
+            ctx->showInfo(message);
             hasPrinted = true;
         }
         // the end
@@ -1448,8 +1450,12 @@ class S2C_SID_PING : public NetworkState {
 class ConnectState : public NetworkState {
     virtual void doOneStep(Context *ctx) {
         // Connect
-        
-        if (!ctx->getTCPSocket()->Open(CHost("0.0.0.0", 6113))) { // TODO...
+
+        std::string localHost = CNetworkParameter::Instance.localHost;
+	if (!localHost.compare("127.0.0.1")) {
+            localHost = "0.0.0.0";
+	}
+        if (!ctx->getTCPSocket()->Open(CHost(localHost.c_str(), CNetworkParameter::Instance.localPort))) {
             ctx->setState(new DisconnectedState("TCP open failed"));
             return;
         }
@@ -1458,7 +1464,7 @@ class ConnectState : public NetworkState {
             return;
         }
         if (!ctx->getTCPSocket()->Connect(*ctx->getHost())) {
-            ctx->setState(new DisconnectedState("TCP connect failed"));
+            ctx->setState(new DisconnectedState("TCP connect failed for server " + ctx->getHost()->toString()));
             return;
         }
         if (!ctx->getUDPSocket()->Open(*ctx->getHost())) {
@@ -1588,6 +1594,7 @@ void GoOnline() {
     Gui->setUseDirtyDrawing(false);
 
     ctx = new Context();
+    ctx->setHost(MetaClient.GetMetaServer());
     ctx->setState(new ConnectState());
 
     onlineServiceContainer = new gcn::Container();

From a858d5f864a982f230c9359b0417b3df8ae2e299 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Sun, 9 Aug 2020 13:59:37 +0200
Subject: [PATCH 17/70] little more debugging

---
 src/network/online_service.cpp | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index fa8a00a36..4b37130f0 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -118,6 +118,16 @@ public:
         return read64();
     }
 
+    /**
+     * For debugging and development: read the entire thing as a char
+     * array. Caller must "free" the out-char
+     */
+    int readAll(char** out) {
+        *out = (char*)calloc(received_bytes, sizeof(char));
+        strncpy(*out, buffer, received_bytes);
+        return received_bytes;
+    }
+
     std::string string32() {
         // uint32 encoded (4-byte) string
         uint32_t data = read32();
@@ -869,6 +879,11 @@ public:
                 // S>C 0x68 SID_FRIENDSREMOVE
                 // S>C 0x67 SID_FRIENDSADD
                 std::cout << "Unhandled message ID: 0x" << std::hex << msg << std::endl;
+                std::cout << "Raw contents >>>" << std::endl;
+                char* out;
+                int len = ctx->getMsgIStream()->readAll(&out);
+                std::cout.write(out, len);
+                std::cout << "<<<" << std::endl;
             }
 
             ctx->getMsgIStream()->finishMessage();

From 201773b155529e0b881935fd574d8c2ac537f8ef Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Sun, 9 Aug 2020 14:01:50 +0200
Subject: [PATCH 18/70] handle ping from server during chat

---
 src/network/online_service.cpp | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 4b37130f0..4fd93a150 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -851,6 +851,9 @@ public:
             switch (msg) {
             case 0x00: // SID_NULL
                 break;
+            case 0x25: // SID_PING
+                handlePing(ctx);
+                break;
             case 0x0f: // CHATEVENT
                 handleChatevent(ctx);
                 break;
@@ -891,6 +894,14 @@ public:
     }
 
 private:
+    void handlePing(Context *ctx) {
+        uint32_t pingValue = ctx->getMsgIStream()->read32();
+        ctx->getMsgIStream()->finishMessage();
+        BNCSOutputStream buffer(0x25);
+        buffer.serialize32(pingValue);
+        send(ctx, &buffer);
+    }
+
     void handleGamelist(Context *ctx) {
         uint32_t cnt = ctx->getMsgIStream()->read32();
         std::vector<Game*> games;

From 61d4818a683a8bdacb177a5ee60434f8e2ab7c74 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 13 Aug 2020 13:43:39 +0200
Subject: [PATCH 19/70] begin removing all references to metaserver in favor of
 pvpgn

---
 CMakeLists.txt                 |  50 -------
 src/include/master.h           |  86 ------------
 src/include/online_service.h   |  21 ++-
 src/network/master.cpp         | 244 ---------------------------------
 src/network/netconnect.cpp     |   1 -
 src/network/online_service.cpp |  76 ++++++----
 src/tolua/master.pkg           |  23 ----
 src/video/sdl.cpp              |   5 +
 8 files changed, 75 insertions(+), 431 deletions(-)
 delete mode 100644 src/include/master.h
 delete mode 100644 src/network/master.cpp
 delete mode 100644 src/tolua/master.pkg

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d60679d1e..1091478b5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -232,7 +232,6 @@ set(network_SRCS
 	src/network/commands.cpp
 	src/network/net_lowlevel.cpp
 	src/network/net_message.cpp
-	src/network/master.cpp
 	src/network/netconnect.cpp
 	src/network/network.cpp
 	src/network/netsockets.cpp
@@ -367,7 +366,6 @@ set(tolua_FILES
 	src/tolua/font.pkg
 	src/tolua/game.pkg
 	src/tolua/map.pkg
-	src/tolua/master.pkg
 	src/tolua/minimap.pkg
 	src/tolua/network.pkg
 	src/tolua/particle.pkg
@@ -537,7 +535,6 @@ set(stratagus_generic_HDRS
 	src/include/iolib.h
 	src/include/luacallback.h
 	src/include/map.h
-	src/include/master.h
 	src/include/menus.h
 	src/include/minimap.h
 	src/include/missile.h
@@ -1104,43 +1101,6 @@ endif()
 
 ########### next target ###############
 
-set(metaserver_SRCS
-	metaserver/cmd.cpp
-	metaserver/db.cpp
-	metaserver/games.cpp
-	metaserver/main.cpp
-	metaserver/netdriver.cpp
-	src/network/net_lowlevel.cpp
-)
-
-set(metaserver_HDRS
-	metaserver/cmd.h
-	metaserver/db.h
-	metaserver/games.h
-	metaserver/netdriver.h
-)
-
-source_group(metaserver FILES ${metaserver_SRCS} ${metaserver_HDRS})
-
-if(SQLITE_FOUND)
-	add_executable(metaserver ${metaserver_SRCS} ${metaserver_HDRS})
-	target_link_libraries(metaserver ${SDL2_LIBRARY} ${SQLITE_LIBRARIES})
-
-	if(WIN32)
-		target_link_libraries(metaserver winmm ws2_32)
-	endif()
-
-	if(WIN32 AND MINGW)
-		target_link_libraries(metaserver dxguid)
-	endif()
-
-	if(WIN32 AND MINGW AND ENABLE_STATIC)
-		set_target_properties(metaserver PROPERTIES LINK_FLAGS "${LINK_FLAGS} -static-libgcc -static-libstdc++")
-	endif()
-endif()
-
-########### next target ###############
-
 set(png2stratagus_SRCS
 	tools/png2stratagus.cpp
 )
@@ -1175,8 +1135,6 @@ set(doxygen_FILES
 	doc/Doxyfile-header.html
 	${stratagus_SRCS}
 	${stratagus_HDRS}
-	${metaserver_SRCS}
-	${metaserver_HDRS}
 	${gameheaders_HDRS}
 	${png2stratagus_SRCS}
 )
@@ -1211,9 +1169,6 @@ endmacro()
 if(ENABLE_UPX AND SELF_PACKER_FOR_EXECUTABLE)
 	self_packer(stratagus)
 	self_packer(png2stratagus)
-	if(SQLITE_FOUND)
-		self_packer(metaserver)
-	endif()
 endif()
 
 ########### next target ###############
@@ -1238,10 +1193,6 @@ endif()
 install(TARGETS stratagus DESTINATION ${GAMEDIR})
 install(TARGETS png2stratagus DESTINATION ${BINDIR})
 
-if(SQLITE_FOUND)
-	install(TARGETS metaserver DESTINATION ${BINDIR} RENAME stratagus-metaserver)
-endif()
-
 if(ENABLE_DOC AND DOXYGEN_FOUND)
 	install(FILES doc/stratagus.6 DESTINATION ${MANDIR})
 	install(FILES
@@ -1253,7 +1204,6 @@ if(ENABLE_DOC AND DOXYGEN_FOUND)
 		doc/index.html
 		doc/install.html
 		doc/media.html
-		doc/metaserver_protocol.txt
 		doc/README-SDL.txt
 		DESTINATION share/doc/stratagus
 	)
diff --git a/src/include/master.h b/src/include/master.h
deleted file mode 100644
index e15b6942f..000000000
--- a/src/include/master.h
+++ /dev/null
@@ -1,86 +0,0 @@
-//       _________ __                 __
-//      /   _____//  |_____________ _/  |______     ____  __ __  ______
-//      \_____  \\   __\_  __ \__  \\   __\__  \   / ___\|  |  \/  ___/
-//      /        \|  |  |  | \// __ \|  |  / __ \_/ /_/  >  |  /\___ |
-//     /_______  /|__|  |__|  (____  /__| (____  /\___  /|____//____  >
-//             \/                  \/          \//_____/            \/
-//  ______________________                           ______________________
-//                        T H E   W A R   B E G I N S
-//         Stratagus - A free fantasy real time strategy game engine
-//
-/**@name master.h - The master server headerfile. */
-//
-//      (c) Copyright 2003-2007 by Tom Zickel and Jimmy Salmon
-//
-//      This program is free software; you can redistribute it and/or modify
-//      it under the terms of the GNU General Public License as published by
-//      the Free Software Foundation; only version 2 of the License.
-//
-//      This program is distributed in the hope that it will be useful,
-//      but WITHOUT ANY WARRANTY; without even the implied warranty of
-//      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-//      GNU General Public License for more details.
-//
-//      You should have received a copy of the GNU General Public License
-//      along with this program; if not, write to the Free Software
-//      Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
-//      02111-1307, USA.
-//
-
-#ifndef __MASTER_H__
-#define __MASTER_H__
-
-//@{
-
-#include <list>
-#include "network/netsockets.h"
-
-/*----------------------------------------------------------------------------
---  Declarations
-----------------------------------------------------------------------------*/
-
-struct lua_State;
-
-// Log data used in metaserver client
-struct CClientLog {
-	std::string entry;     // command itself
-};
-
-
-// Class representing meta server client structure
-class CMetaClient
-{
-public:
-	CMetaClient() : metaSocket(), metaPort(-1), lastRecvState(-1) {}
-	~CMetaClient();
-	void SetMetaServer(const std::string host, const int port);
-	int Init();
-	void Close();
-	int Send(const std::string cmd);
-	int Recv();
-	int GetLastRecvState() { return lastRecvState; }
-	int GetLogSize() { return events.size(); }
-	CClientLog *GetLastMessage() { return events.back(); }
-	int CreateGame(std::string desc, std::string map, std::string players);
-
-	CHost *GetMetaServer() { return new CHost(metaHost.c_str(), metaPort); }
-
-private:
-	CTCPSocket metaSocket;                     /// This is a TCP socket
-	std::string metaHost;                      /// Address of metaserver
-	int metaPort;                              /// Port of metaserver
-	std::list <CClientLog *> events;           /// All commands received from metaserver
-	int lastRecvState;                         /// Now many bytes have been received in last reply
-};
-
-
-
-/*----------------------------------------------------------------------------
---  Variables
-----------------------------------------------------------------------------*/
-// Metaserver itself
-extern CMetaClient MetaClient;
-
-//@}
-
-#endif // !__MASTER_H__
diff --git a/src/include/online_service.h b/src/include/online_service.h
index 12d14f22e..2146ae2c6 100644
--- a/src/include/online_service.h
+++ b/src/include/online_service.h
@@ -1,6 +1,23 @@
 #ifndef __ONLINE_SERVICE_H__
 #define __ONLINE_SERVICE_H__
 
-void GoOnline();
+class OnlineContext {
+public:
+    virtual void doOneStep();
 
-#endif // !__EDITOR_H__
+    virtual void goOnline();
+
+    virtual void joinGame(std:string name, std::string pw);
+
+    // TODO: allow passing all the other options, like 1 peon only, resource amount, game type, ...
+    virtual void advertiseGame(std::string name, std::string pw, std::string creatorName, std::string mapName,
+                               int mapX, int mapY, int maxPlayers, int playersInGame);
+
+    virtual void stopAdvertisingGame();
+
+    virtual void reportGameResult();
+};
+
+extern OnlineContext *OnlineContextHandler;
+
+#endif
diff --git a/src/network/master.cpp b/src/network/master.cpp
deleted file mode 100644
index 6c8bc4d07..000000000
--- a/src/network/master.cpp
+++ /dev/null
@@ -1,244 +0,0 @@
-//       _________ __                 __
-//      /   _____//  |_____________ _/  |______     ____  __ __  ______
-//      \_____  \\   __\_  __ \__  \\   __\__  \   / ___\|  |  \/  ___/
-//      /        \|  |  |  | \// __ \|  |  / __ \_/ /_/  >  |  /\___ |
-//     /_______  /|__|  |__|  (____  /__| (____  /\___  /|____//____  >
-//             \/                  \/          \//_____/            \/
-//  ______________________                           ______________________
-//                        T H E   W A R   B E G I N S
-//         Stratagus - A free fantasy real time strategy game engine
-//
-/**@name master.cpp - The master server. */
-//
-//      (c) Copyright 2003-2007 by Tom Zickel and Jimmy Salmon
-//
-//      This program is free software; you can redistribute it and/or modify
-//      it under the terms of the GNU General Public License as published by
-//      the Free Software Foundation; only version 2 of the License.
-//
-//      This program is distributed in the hope that it will be useful,
-//      but WITHOUT ANY WARRANTY; without even the implied warranty of
-//      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-//      GNU General Public License for more details.
-//
-//      You should have received a copy of the GNU General Public License
-//      along with this program; if not, write to the Free Software
-//      Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
-//      02111-1307, USA.
-//
-
-//@{
-
-/*----------------------------------------------------------------------------
---  Includes
-----------------------------------------------------------------------------*/
-
-#include <errno.h>
-#include <stdarg.h>
-#include <ctype.h>
-#include <limits.h>
-
-#include "stratagus.h"
-
-#include "master.h"
-
-#include "game.h"
-#include "network/netsockets.h"
-#include "network.h"
-#include "net_lowlevel.h"
-#include "parameters.h"
-#include "script.h"
-#include "version.h"
-
-
-/*----------------------------------------------------------------------------
---  Variables
-----------------------------------------------------------------------------*/
-
-CMetaClient MetaClient;
-
-/*----------------------------------------------------------------------------
---  Functions
-----------------------------------------------------------------------------*/
-
-/**
-**  Set the metaserver to use for internet play.
-**
-**  @param host  Host to connect
-**  @param port  Port to use to connect
-*/
-void CMetaClient::SetMetaServer(const std::string host, const int port)
-{
-	metaHost = host;
-	metaPort = port;
-}
-
-CMetaClient::~CMetaClient()
-{
-	for (std::list<CClientLog *>::iterator it = events.begin(); it != events.end(); ++it) {
-		CClientLog *log = *it;
-		delete log;
-	}
-	events.clear();
-	this->Close();
-}
-
-/**
-**  Initialize the TCP connection to the Meta Server and send test ping to it.
-**
-**  @return  -1 fail, 0 success.
-*/
-int CMetaClient::Init()
-{
-	if (metaPort == -1) {
-		return -1;
-	}
-
-	// Server socket
-	CHost metaServerHost(metaHost.c_str(), metaPort);
-	// Client socket
-
-	// open on all interfaces, not the loopback, unless we have an override from the commandline
-	std::string localHost = CNetworkParameter::Instance.localHost;
-	if (!localHost.compare("127.0.0.1")) {
-		localHost = "0.0.0.0";
-	}
-	CHost metaClientHost(localHost.c_str(), CNetworkParameter::Instance.localPort);
-	metaSocket.Open(metaClientHost);
-	if (metaSocket.IsValid() == false) {
-		fprintf(stderr, "METACLIENT: No free port %d available, aborting\n", metaServerHost.getPort());
-		return -1;
-	}
-	if (metaSocket.Connect(metaServerHost) == false) {
-		fprintf(stderr, "METACLIENT: Unable to connect to host %s\n", metaServerHost.toString().c_str());
-		MetaClient.Close();
-		return -1;
-	}
-
-	if (this->Send("PING") == -1) { // not sent
-		MetaClient.Close();
-		return -1;
-	}
-	if (this->Recv() == -1) { // not received
-		MetaClient.Close();
-		return -1;
-	}
-	CClientLog &log = *GetLastMessage();
-	if (log.entry.find("PING_OK") != std::string::npos) {
-		// Everything is OK
-		return 0;
-	} else {
-		fprintf(stderr, "METACLIENT: inappropriate message received from %s\n", metaServerHost.toString().c_str());
-		MetaClient.Close();
-		return -1;
-	}
-}
-
-/**
-**  Close Connection to Master Server
-**
-**  @return  nothing
-*/
-void CMetaClient::Close()
-{
-	if (metaSocket.IsValid()) {
-		metaSocket.Close();
-	}
-}
-
-
-/**
-**  Send a command to the meta server
-**
-**  @param cmd   command to send
-**
-**  @returns     -1 if failed, otherwise length of command
-*/
-int CMetaClient::Send(const std::string cmd)
-{
-	int ret = -1;
-	if (metaSocket.IsValid()) {
-		std::string mes(cmd);
-		mes.append("\n");
-		ret = metaSocket.Send(mes.c_str(), mes.size());
-	}
-	return ret;
-}
-
-/**
-**  Receive reply from Meta Server
-**
-**  @return error or number of bytes
-*/
-int CMetaClient::Recv()
-{
-	if (metaSocket.HasDataToRead(5000) == -1) {
-		return -1;
-	}
-
-	char buf[1024];
-	memset(&buf, 0, sizeof(buf));
-	int n = metaSocket.Recv(&buf, sizeof(buf));
-	if (n == -1) {
-		return n;
-	}
-	// We know we now have the whole command.
-	// Convert to standard notation
-	std::string cmd(buf, strlen(buf));
-	cmd += '\n';
-	cmd += '\0';
-	CClientLog *log = new CClientLog;
-	log->entry = cmd;
-	events.push_back(log);
-	lastRecvState = n;
-	return n;
-}
-
-//@}
-
-int CMetaClient::CreateGame(std::string desc, std::string map, std::string players) {
-	if (metaSocket.IsValid() == false) {
-		return -1;
-	}
-	if (NetworkFildes.IsValid() == false) {
-		return -1;
-	}
-	CHost metaServerHost(metaHost.c_str(), metaPort);
-
-	// Advertise an external IP address if we can
-	unsigned long ips[1];
-	int networkNumInterfaces = NetworkFildes.GetSocketAddresses(ips, 1);
-	std::string ipport = "";
-	if (!networkNumInterfaces || CNetworkParameter::Instance.localHost.compare("127.0.0.1")) {
-	    ipport += CNetworkParameter::Instance.localHost.c_str();
-	} else {
-		ipport += inet_ntoa(((struct in_addr *)ips)[0]);
-	}
-	ipport += " ";
-	ipport += std::to_string(CNetworkParameter::Instance.localPort);
-
-	std::string cmd("CREATEGAME \"");
-	cmd += desc;
-	cmd += "\" \"";
-	cmd += map;
-	cmd += "\" ";
-	cmd += players;
-	cmd += " ";
-	cmd += ipport;
-
-	if (this->Send(cmd.c_str()) == -1) { // not sent
-		return -1;
-	}
-	if (this->Recv() == -1) { // not received
-		return -1;
-	}
-	CClientLog &log = *GetLastMessage();
-	if (log.entry.find("CREATEGAME_OK") != std::string::npos) {
-		// Everything is OK, let's inform metaserver of our UDP info
-		NetworkFildes.Send(metaServerHost, ipport.c_str(), ipport.size());
-		return 0;
-	} else {
-		fprintf(stderr, "METACLIENT: failed to create game: %s\n", log.entry.c_str());
-		return -1;
-	}
-}
diff --git a/src/network/netconnect.cpp b/src/network/netconnect.cpp
index 38c64bd2c..a6350e04a 100644
--- a/src/network/netconnect.cpp
+++ b/src/network/netconnect.cpp
@@ -39,7 +39,6 @@
 
 #include "interface.h"
 #include "map.h"
-#include "master.h"
 #include "network.h"
 #include "parameters.h"
 #include "player.h"
diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 4fd93a150..07514ea5a 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -584,7 +584,7 @@ protected:
     int send(Context *ctx, BNCSOutputStream *buf);
 };
 
-class Context {
+class Context : public OnlineContext {
 public:
     Context() {
         this->udpSocket = new CUDPSocket();
@@ -606,7 +606,29 @@ public:
         delete host;
     }
 
+    boolean isConnected() {
+        return !getCurrentChannel().empty();
+    }
+
     // User and UI actions
+    void disconnect() {
+        if (isConnected()) {
+            // SID_STOPADV: according to bnetdocs.org, this is always sent when
+            // clients disconnect, regardless of state
+            BNCSOutputStream stop(0x02);
+            stop.flush(tcpSocket);
+            // SID_LEAVECHAT
+            BNCSOutputStream leave(0x10);
+            leave.flush(tcpSocket);
+        }
+        udpSocket->Close();
+        tcpSocket->Close();
+        state = NULL;
+        clientToken = MyRand();
+        username = "";
+        setPassword("");
+    }
+
     void sendText(std::string txt) {
         // C>S 0x0E SID_CHATCOMMAND
         int pos = 0;
@@ -759,7 +781,7 @@ public:
 
     BNCSInputStream *getMsgIStream() { return istream; }
 
-    void doOneStep() { this->state->doOneStep(this); }
+    virtual void doOneStep() { if (this->state != NULL) this->state->doOneStep(this); }
 
     void setState(NetworkState* newState) {
         assert (newState != this->state);
@@ -808,6 +830,7 @@ public:
             std::cout << message << std::endl;
             ctx->showInfo(message);
             hasPrinted = true;
+            ctx->disconnect();
         }
         // the end
     }
@@ -822,6 +845,7 @@ class C2S_GAMERESULT_OR_STOPADV : public NetworkState {
         // TODO - wait until the game lobby is left or the game is over and then send the result
         // C>S 0x02 SID_STOPADV
         // C>S 0x2C SID_GAMERESULT
+        // C>S 0x22 SID_NOTIFYJOIN
     }
 };
 
@@ -850,6 +874,7 @@ public:
 
             switch (msg) {
             case 0x00: // SID_NULL
+                handleNull(ctx);
                 break;
             case 0x25: // SID_PING
                 handlePing(ctx);
@@ -894,9 +919,13 @@ public:
     }
 
 private:
+    void handleNull(Context *ctx) {
+        BNCSOutputStream buffer(0x00);
+        send(ctx, &buffer);
+    }
+
     void handlePing(Context *ctx) {
         uint32_t pingValue = ctx->getMsgIStream()->read32();
-        ctx->getMsgIStream()->finishMessage();
         BNCSOutputStream buffer(0x25);
         buffer.serialize32(pingValue);
         send(ctx, &buffer);
@@ -1585,12 +1614,13 @@ static gcn::Container *loginWindowContainer;
 static gcn::TextField *username;
 static gcn::TextField *password;
 
-static Context *ctx;
+static Context _ctx;
+Context *OnlineContext = &_ctx;
 
 class ChatInputListener : public gcn::ActionListener {
     virtual void action(const std::string &) {
-        if (!ctx->getCurrentChannel().empty()) {
-            ctx->sendText(chatInput->getText());
+        if (!OnlineContext->getCurrentChannel().empty()) {
+            OnlineContext->sendText(chatInput->getText());
             chatInput->setText("");
         }
     }
@@ -1598,15 +1628,15 @@ class ChatInputListener : public gcn::ActionListener {
 
 class UsernameInputListener : public gcn::ActionListener {
     virtual void action(const std::string &) {
-        ctx->setUsername(username->getText());
-        ctx->setPassword(password->getText());
+        OnlineContext->setUsername(username->getText());
+        OnlineContext->setPassword(password->getText());
     }
 };
 
 class PasswordInputListener : public gcn::ActionListener {
     virtual void action(const std::string &) {
-        ctx->setUsername(username->getText());
-        ctx->setPassword(password->getText());
+        OnlineContext->setUsername(username->getText());
+        OnlineContext->setPassword(password->getText());
     }
 };
 
@@ -1619,9 +1649,7 @@ void GoOnline() {
     gcn::Widget *oldTop = Gui->getTop();
     Gui->setUseDirtyDrawing(false);
 
-    ctx = new Context();
-    ctx->setHost(MetaClient.GetMetaServer());
-    ctx->setState(new ConnectState());
+    OnlineContext->setState(new ConnectState());
 
     onlineServiceContainer = new gcn::Container();
     onlineServiceContainer->setDimension(gcn::Rectangle(0, 0, Video.Width, Video.Height));
@@ -1695,19 +1723,17 @@ void GoOnline() {
     InterfaceState = IfaceStateNormal;
     UI.SelectedViewport = UI.Viewports;
     while (1) {
-        ctx->doOneStep();
-
-        if (!ctx->getCurrentChannel().empty()) {
+        if (OnlineContext->isConnected()) {
             loginWindow->setVisible(false);
 
             if ((FrameCounter % (FRAMES_PER_SECOND * 5)) == 0) {
-                ctx->refreshGames();
-                ctx->refreshFriends();
+                OnlineContext->refreshGames();
+                OnlineContext->refreshFriends();
             }
 
             if ((FrameCounter % (FRAMES_PER_SECOND * 1)) == 0) {
                 static_cast<gcn::TextBox*>(gamelistArea->getContent())->setText("");
-                for (auto g : ctx->getGames()) {
+                for (auto g : OnlineContext->getGames()) {
                     static_cast<gcn::TextBox*>(gamelistArea->getContent())->addRow(g->getMap() + " " +
                                                                                    g->getCreator() + " " +
                                                                                    g->getGameType() + " " +
@@ -1716,25 +1742,25 @@ void GoOnline() {
                 }
 
                 static_cast<gcn::TextBox*>(usersArea->getContent())->setText("");
-                for (auto u : ctx->getUsers()) {
+                for (auto u : OnlineContext->getUsers()) {
                     static_cast<gcn::TextBox*>(usersArea->getContent())->addRow(u);
                 }
 
                 static_cast<gcn::TextBox*>(channelsArea->getContent())->setText("");
-                for (auto u : ctx->getChannels()) {
+                for (auto u : OnlineContext->getChannels()) {
                     static_cast<gcn::TextBox*>(channelsArea->getContent())->addRow(u);
                 }
 
                 static_cast<gcn::TextBox*>(friendsArea->getContent())->setText("");
-                for (auto u : ctx->getFriends()) {
+                for (auto u : OnlineContext->getFriends()) {
                     static_cast<gcn::TextBox*>(friendsArea->getContent())->addRow(u->getName() + ", " + u->getStatus() + ", " + u->getProduct());
                 }
             }
         }
 
-        while (!ctx->getInfo()->empty()) {
-            static_cast<gcn::TextBox*>(messageArea->getContent())->addRow(ctx->getInfo()->front());
-            ctx->getInfo()->pop();
+        while (!OnlineContext->getInfo()->empty()) {
+            static_cast<gcn::TextBox*>(messageArea->getContent())->addRow(OnlineContext->getInfo()->front());
+            OnlineContext->getInfo()->pop();
         }
 
         Gui->draw();
diff --git a/src/tolua/master.pkg b/src/tolua/master.pkg
deleted file mode 100644
index 955ff29f1..000000000
--- a/src/tolua/master.pkg
+++ /dev/null
@@ -1,23 +0,0 @@
-$#include "master.h"
-
-struct CClientLog
-{
-	std::string entry;
-};
-
-class CMetaClient
-{
-public:
-	CMetaClient();
-	void SetMetaServer(const std::string host, const int port);
-	int Init();
-	void Close();
-	int Send(const std::string cmd);
-	int Recv();
-	int GetLastRecvState() { return lastRecvState; }
-	int GetLogSize() { return events.size(); }
-	CClientLog* GetLastMessage() { return events.back(); }
-	int CreateGame(std::string desc, std::string map, std::string players);
-};
-
-CMetaClient MetaClient;
diff --git a/src/video/sdl.cpp b/src/video/sdl.cpp
index e4ae9a962..8e8c93487 100644
--- a/src/video/sdl.cpp
+++ b/src/video/sdl.cpp
@@ -35,6 +35,7 @@
 ----------------------------------------------------------------------------*/
 
 #include "stratagus.h"
+#include "online_service.h"
 
 #ifdef DEBUG
 #include <signal.h>
@@ -676,6 +677,10 @@ void WaitEventsOneFrame()
 				GetCallbacks()->NetworkEvent();
 			}
 		}
+
+		// Online session
+		OnlineContextHandler->doOneStep();
+
 		// No more input and time for frame over: return
 		if (!i && s <= 0 && interrupts) {
 			break;

From fd8c5eb329844a6c3f8304b19ae9fef375b0452d Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 13 Aug 2020 15:10:35 +0200
Subject: [PATCH 20/70] tighten down calls for game join/part and start/stop

---
 CMakeLists.txt                 |   1 -
 src/game/game.cpp              |   2 +
 src/include/netconnect.h       |   2 +
 src/include/online_service.h   |  20 +++--
 src/network/netconnect.cpp     |   9 +-
 src/network/online_service.cpp | 159 +++++++++++++++++++++++++++++++--
 src/stratagus/mainloop.cpp     |   5 ++
 src/tolua/online_service.pkg   |   3 -
 8 files changed, 183 insertions(+), 18 deletions(-)
 delete mode 100644 src/tolua/online_service.pkg

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1091478b5..b4bffc924 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -362,7 +362,6 @@ source_group(win32 FILES ${win32_SRCS})
 set(tolua_FILES
 	src/tolua/ai.pkg
 	src/tolua/editor.pkg
-	src/tolua/online_service.pkg
 	src/tolua/font.pkg
 	src/tolua/game.pkg
 	src/tolua/map.pkg
diff --git a/src/game/game.cpp b/src/game/game.cpp
index 1d86945fd..d07bba4b8 100644
--- a/src/game/game.cpp
+++ b/src/game/game.cpp
@@ -56,6 +56,7 @@
 #include "missile.h"
 #include "netconnect.h"
 #include "network.h"
+#include "online_service.h"
 #include "parameters.h"
 #include "pathfinder.h"
 #include "player.h"
@@ -1507,6 +1508,7 @@ void LuaRegisterModules()
 	UpgradesCclRegister();
 	UserInterfaceCclRegister();
 	VideoCclRegister();
+	OnlineServiceCclRegister();
 }
 
 
diff --git a/src/include/netconnect.h b/src/include/netconnect.h
index 9baf8e3ca..5902c6e8c 100644
--- a/src/include/netconnect.h
+++ b/src/include/netconnect.h
@@ -100,6 +100,8 @@ extern int NetLocalPlayerNumber;           /// Player number of local client
 extern CServerSetup ServerSetupState;      /// Network menu: Multiplayer Server Menu selections state
 extern CServerSetup LocalSetupState;       /// Network menu: Multiplayer Client Menu selections local state
 
+extern int NoRandomPlacementMultiplayer; /// Disable the random placement of players in muliplayer mode
+
 /*----------------------------------------------------------------------------
 --  Functions
 ----------------------------------------------------------------------------*/
diff --git a/src/include/online_service.h b/src/include/online_service.h
index 2146ae2c6..944cfb8e2 100644
--- a/src/include/online_service.h
+++ b/src/include/online_service.h
@@ -1,23 +1,31 @@
 #ifndef __ONLINE_SERVICE_H__
 #define __ONLINE_SERVICE_H__
 
+#include <string>
+
 class OnlineContext {
 public:
+    // called in the sdl event loop
     virtual void doOneStep();
 
-    virtual void goOnline();
+    // called when joining a network game
+    virtual void joinGame(std::string hostPlayerName, std::string pw);
 
-    virtual void joinGame(std:string name, std::string pw);
+    // called when leaving a network game
+    virtual void leaveGame();
 
-    // TODO: allow passing all the other options, like 1 peon only, resource amount, game type, ...
-    virtual void advertiseGame(std::string name, std::string pw, std::string creatorName, std::string mapName,
-                               int mapX, int mapY, int maxPlayers, int playersInGame);
+    // called when advertised game is starting (just reports the game as in-progress)
+    virtual void startAdvertising(bool isStarted = false);
 
-    virtual void stopAdvertisingGame();
+    // called when advertised game is left by the server
+    virtual void stopAdvertising();
 
+    // called when network game ends
     virtual void reportGameResult();
 };
 
 extern OnlineContext *OnlineContextHandler;
 
+extern void OnlineServiceCclRegister();
+
 #endif
diff --git a/src/network/netconnect.cpp b/src/network/netconnect.cpp
index a6350e04a..12c37b371 100644
--- a/src/network/netconnect.cpp
+++ b/src/network/netconnect.cpp
@@ -33,6 +33,7 @@
 // Includes
 //----------------------------------------------------------------------------
 
+#include "online_service.h"
 #include "stratagus.h"
 
 #include "netconnect.h"
@@ -86,7 +87,7 @@ int NetLocalPlayerNumber;              /// Player number of local client
 
 int NetPlayers;                         /// How many network players
 std::string NetworkMapName;             /// Name of the map received with ICMMap
-static int NoRandomPlacementMultiplayer = 0; /// Disable the random placement of players in muliplayer mode
+int NoRandomPlacementMultiplayer = 0; /// Disable the random placement of players in muliplayer mode
 
 CServerSetup ServerSetupState; // Server selection state for Multiplayer clients
 CServerSetup LocalSetupState;  // Local selection state for Multiplayer clients
@@ -781,6 +782,8 @@ void CClient::Parse_Welcome(const unsigned char *buf)
 	CNetworkParameter::Instance.NetworkLag = msg.Lag;
 	CNetworkParameter::Instance.gameCyclesPerUpdate = msg.gameCyclesPerUpdate;
 
+	OnlineContextHandler->joinGame(msg.hosts[0].PlyName, "");
+
 	Hosts[0].Host = serverHost.getIp();
 	Hosts[0].Port = serverHost.getPort();
 	for (int i = 1; i < PlayerMax; ++i) {
@@ -1705,6 +1708,10 @@ breakout:
 	}
 
 	DebugPrint("DONE: All configs acked - Now starting..\n");
+
+	// advertise online that we're in progress
+	OnlineContextHandler->startAdvertising(true);
+
 	// Give clients a quick-start kick..
 	const CInitMessage_Header message_go(MessageInit_FromServer, ICMGo);
 	for (int i = 0; i < HostsCount; ++i) {
diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 07514ea5a..450d67200 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1,5 +1,5 @@
 #include "online_service.h"
-#include "master.h"
+#include "results.h"
 
 #include <arpa/inet.h>
 #include <clocale>
@@ -7,6 +7,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <ctime>
+#include <ios>
 #include <iostream>
 #include <map>
 #include <ostream>
@@ -25,6 +26,13 @@
 #include <netinet/in.h>
 #endif
 
+#include "stratagus.h"
+#include "lua.h"
+#include "map.h"
+#include "netconnect.h"
+#include "script.h"
+#include "settings.h"
+#include "tileset.h"
 #include "cursor.h"
 #include "font.h"
 #include "input.h"
@@ -42,7 +50,6 @@
 
 #include "./xsha1.h"
 
-
 class BNCSInputStream {
 public:
     BNCSInputStream(CTCPSocket *socket) {
@@ -677,16 +684,134 @@ public:
         msg.flush(getTCPSocket());
     }
 
-    void joinGame(std::string name, std::string pw) {
+    virtual void joinGame(std::string username, std::string pw) {
+        if (!isConnected()) {
+            return;
+        }
         // C>S 0x22 SID_NOTIFYJOIN
         BNCSOutputStream msg(0x09);
         msg.serializeC32("W2BN");
         msg.serialize32(0x4f);
-        msg.serialize(name.c_str());
+        msg.serialize(gameNameFromUsername(getUsername()).c_str());
         msg.serialize(pw.c_str());
         msg.flush(getTCPSocket());
     }
 
+    virtual void leaveGame() {
+        // TODO: ?
+    }
+
+    virtual void startAdvertising(bool isStarted = false) {
+        if (!isConnected()) {
+            return;
+        }
+        BNCSOutputStream msg(0x1c);
+        int maxSlots = 0;
+        for (int i = 0; i < PlayerMax; i++) {
+            if (ServerSetupState.CompOpt[i] == 0) { // available
+                maxSlots++;
+            }
+        }
+        int joinedPlayers = 0;
+        for (int i = 1; i < PlayerMax; i++) { // skip server host
+            if (Hosts[i].PlyNr) {
+                joinedPlayers++;
+            }
+        }
+        uint32_t state = 0x10; // disconnect always counts as loss
+        if (joinedPlayers) {
+            state |= 0x04; // has players other than creator
+        }
+        if (joinedPlayers + 1 == maxSlots) {
+            state |= 0x02; // game is full
+        }
+        if (isStarted) {
+            state |= 0x08; // game in progress
+        }
+        msg.serialize32(state);
+        msg.serialize32(0x00); // uptime
+        msg.serialize16(0x0a); // game type - map settings
+        msg.serialize16(0x01); // sub game type
+        msg.serialize32(0xff); // provider version constant
+        msg.serialize32(0x00); // not ladder
+        msg.serialize((gameNameFromUsername(getUsername())).c_str()); // game name
+        msg.serialize(""); // password. TODO: add support
+
+        std::stringstream statstring;
+        statstring << ","; // this game is not saved. TODO: add support
+        int w = Map.Info.MapWidth;
+        int h = Map.Info.MapWidth;
+        if (w == 128 && h == 128) {
+            statstring << ",";
+        } else {
+            statstring << std::dec << w / 32 << h / 32 << ",";
+        }
+        if (maxSlots == 8) {
+            statstring << ",";
+        } else {
+            statstring << std::dec << maxSlots + 10 << ",";
+        }
+        statstring << "0x04,"; // speed - normal (can be changed in-game anyway)
+        statstring << "0x00,"; // not an approved game
+        statstring << "0x0a,"; // game type uses map settings
+        statstring << "0x01,"; // game settings parameter - none
+        statstring << std::hex << FileChecksums << ","; // cd key checksum - we use lua files checksum
+
+        uint32_t game_settings = 0;
+        if (GameSettings.NumUnits == 1) {
+            game_settings |= 0x200;
+        }
+        if (NoRandomPlacementMultiplayer == 1) {
+            game_settings |= 0x400;
+        }
+        switch (GameSettings.Resources) {
+        case -1:
+            break;
+        case 1:
+            game_settings |= 0x1000;
+            break;
+        case 2:
+            game_settings |= 0x2000;
+            break;
+        case 3:
+            game_settings |= 0x3000;
+            break;
+        default:
+            game_settings |= 0x20000;
+            break;
+        }
+        if (Map.Tileset->Name == "Forest") {
+            game_settings |= 0x4000;
+        } else if (Map.Tileset->Name == "Winter") {
+            game_settings |= 0x8000;
+        } else if (Map.Tileset->Name == "Wasteland") {
+            game_settings |= 0xC000;
+        } else if (Map.Tileset->Name == "Orc Swamp") {
+            game_settings |= 0x1C000;
+        }
+        statstring << std::hex << game_settings << ",";
+
+        statstring << getUsername();
+        statstring.put(0x0d);
+        statstring << Map.Info.Filename;
+        statstring.put(0x0d);
+
+        msg.serialize(statstring.str().c_str());
+        msg.flush(getTCPSocket());
+    }
+
+    virtual void stopAdvertising() {
+        if (!isConnected()) {
+            return;
+        }
+        BNCSOutputStream msg(0x02);
+        msg.flush(getTCPSocket());
+    }
+
+    virtual void reportGameResult() {
+        GameResult
+    }
+
     // UI information
     void setCurrentChannel(std::string name) {
         this->currentChannel = name;
@@ -795,6 +920,10 @@ public:
     uint32_t serverToken;
 
 private:
+    std::string gameNameFromUsername(std::string username) {
+        return username + "'s game";
+    }
+
     NetworkState *state;
     CHost *host;
     CUDPSocket *udpSocket;
@@ -1507,9 +1636,9 @@ class ConnectState : public NetworkState {
         // Connect
 
         std::string localHost = CNetworkParameter::Instance.localHost;
-	if (!localHost.compare("127.0.0.1")) {
+        if (!localHost.compare("127.0.0.1")) {
             localHost = "0.0.0.0";
-	}
+        }
         if (!ctx->getTCPSocket()->Open(CHost(localHost.c_str(), CNetworkParameter::Instance.localPort))) {
             ctx->setState(new DisconnectedState("TCP open failed"));
             return;
@@ -1640,7 +1769,7 @@ class PasswordInputListener : public gcn::ActionListener {
     }
 };
 
-void GoOnline() {
+static int CclGoOnline(lua_State* l) {
     std::string nc, rc;
     GetDefaultTextColors(nc, rc);
 
@@ -1801,4 +1930,20 @@ void GoOnline() {
     delete loginWindowContainer;
     delete username;
     delete password;
+
+    return 0;
+}
+
+static int CclStopAdvertisingOnlineGame(lua_State* l) {
+    return 0;
+}
+
+static int CclStartAdvertisingOnlineGame(lua_State* l) {
+    return 0;
+}
+
+void OnlineServiceCclRegister() {
+    lua_register(Lua, "StartAdvertisingOnlineGame", CclStartAdvertisingOnlineGame);
+    lua_register(Lua, "StopAdvertisingOnlineGame", CclStopAdvertisingOnlineGame);
+    lua_register(Lua, "GoOnline", CclGoOnline);
 }
diff --git a/src/stratagus/mainloop.cpp b/src/stratagus/mainloop.cpp
index 934f4f0a3..c9f248d76 100644
--- a/src/stratagus/mainloop.cpp
+++ b/src/stratagus/mainloop.cpp
@@ -33,6 +33,7 @@
 //  Includes
 //----------------------------------------------------------------------------
 
+#include "online_service.h"
 #include "stratagus.h"
 
 #include "actions.h"
@@ -418,6 +419,10 @@ void GameMainLoop()
 	//
 	// Game over
 	//
+	if (ThisPlayer && IsNetworkGame()) {
+		OnlineContextHandler->reportGameResult();
+	}
+
 	if (GameResult == GameExit) {
 		Exit(0);
 		return;
diff --git a/src/tolua/online_service.pkg b/src/tolua/online_service.pkg
deleted file mode 100644
index 831419847..000000000
--- a/src/tolua/online_service.pkg
+++ /dev/null
@@ -1,3 +0,0 @@
-$#include "online_service.h"
-
-void GoOnline();

From 7ecf62cdd6dab2c50ac9ddf17d9873c53b450fb2 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 13 Aug 2020 16:28:43 +0200
Subject: [PATCH 21/70] implement reporting game result

---
 src/include/netconnect.h       |  2 ++
 src/network/online_service.cpp | 41 ++++++++++++++++++++++++----------
 src/tolua/stratagus.pkg        |  1 -
 3 files changed, 31 insertions(+), 13 deletions(-)

diff --git a/src/include/netconnect.h b/src/include/netconnect.h
index 5902c6e8c..7191dbbda 100644
--- a/src/include/netconnect.h
+++ b/src/include/netconnect.h
@@ -97,6 +97,8 @@ extern int NetConnectType;              /// Network menu: Setup mode active
 extern int NetLocalHostsSlot;              /// Network menu: Slot # in Hosts array of local client
 extern int NetLocalPlayerNumber;           /// Player number of local client
 
+extern std::string NetworkMapName;         /// Name of the map received with ICMMap
+
 extern CServerSetup ServerSetupState;      /// Network menu: Multiplayer Server Menu selections state
 extern CServerSetup LocalSetupState;       /// Network menu: Multiplayer Client Menu selections local state
 
diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 450d67200..516dbb25f 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -809,7 +809,35 @@ public:
     }
 
     virtual void reportGameResult() {
-        GameResult
+        BNCSOutputStream msg(0x2c);
+        msg.serialize32(8); // number of results
+        for (int i = 0; i < 8; i++) {
+            if (NetLocalPlayerNumber == i) {
+                switch (GameResult) {
+                case GameVictory:
+                    msg.serialize32(0x01);
+                    break;
+                case GameDefeat:
+                    msg.serialize32(0x02);
+                    break;
+                case GameDraw:
+                    msg.serialize32(0x03);
+                    break;
+                default:
+                    msg.serialize32(0x04);
+                    break;
+                }
+            } else {
+                // it's annoying to tease out the other results, we ignore it and let the server merge
+                msg.serialize32(0x00);
+            }
+        }
+        for (int i = 0; i < 8; i++) {
+            msg.serialize(Hosts[i].PlyName);
+        }
+        msg.serialize(NetworkMapName.c_str());
+        msg.serialize(""); // TODO: transmit player scores
+        msg.flush(getTCPSocket());
     }
 
     // UI information
@@ -969,15 +997,6 @@ private:
     std::string message;
 };
 
-class C2S_GAMERESULT_OR_STOPADV : public NetworkState {
-    virtual void doOneStep(Context *ctx) {
-        // TODO - wait until the game lobby is left or the game is over and then send the result
-        // C>S 0x02 SID_STOPADV
-        // C>S 0x2C SID_GAMERESULT
-        // C>S 0x22 SID_NOTIFYJOIN
-    }
-};
-
 class S2C_CHATEVENT : public NetworkState {
 public:
     S2C_CHATEVENT() {
@@ -1019,8 +1038,6 @@ public:
                 // S>C 0x1C SID_STARTADVEX3
                 if (ctx->getMsgIStream()->read32()) {
                     ctx->showError("Game creation failed");
-                } else {
-                    ctx->setState(new C2S_GAMERESULT_OR_STOPADV());
                 }
                 break;
             case 0x65:
diff --git a/src/tolua/stratagus.pkg b/src/tolua/stratagus.pkg
index 9f70c7f69..dbb1f4443 100644
--- a/src/tolua/stratagus.pkg
+++ b/src/tolua/stratagus.pkg
@@ -73,7 +73,6 @@ $pfile "editor.pkg"
 $pfile "font.pkg"
 $pfile "game.pkg"
 $pfile "map.pkg"
-$pfile "master.pkg"
 $pfile "minimap.pkg"
 $pfile "network.pkg"
 $pfile "particle.pkg"

From d24c9f2535978cdc387901d6b27c577fd3e4f6d0 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 13 Aug 2020 16:37:52 +0200
Subject: [PATCH 22/70] fix compilation

---
 src/include/online_service.h   | 14 +++++++------
 src/network/online_service.cpp | 36 +++++++++++++++++-----------------
 2 files changed, 26 insertions(+), 24 deletions(-)

diff --git a/src/include/online_service.h b/src/include/online_service.h
index 944cfb8e2..3eedda892 100644
--- a/src/include/online_service.h
+++ b/src/include/online_service.h
@@ -5,23 +5,25 @@
 
 class OnlineContext {
 public:
+    virtual ~OnlineContext() { };
+
     // called in the sdl event loop
-    virtual void doOneStep();
+    virtual void doOneStep() = 0;
 
     // called when joining a network game
-    virtual void joinGame(std::string hostPlayerName, std::string pw);
+    virtual void joinGame(std::string hostPlayerName, std::string pw) = 0;
 
     // called when leaving a network game
-    virtual void leaveGame();
+    virtual void leaveGame() = 0;
 
     // called when advertised game is starting (just reports the game as in-progress)
-    virtual void startAdvertising(bool isStarted = false);
+    virtual void startAdvertising(bool isStarted = false) = 0;
 
     // called when advertised game is left by the server
-    virtual void stopAdvertising();
+    virtual void stopAdvertising() = 0;
 
     // called when network game ends
-    virtual void reportGameResult();
+    virtual void reportGameResult() = 0;
 };
 
 extern OnlineContext *OnlineContextHandler;
diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 516dbb25f..2ae7f0223 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1761,12 +1761,12 @@ static gcn::TextField *username;
 static gcn::TextField *password;
 
 static Context _ctx;
-Context *OnlineContext = &_ctx;
+OnlineContext *OnlineContextHandler = &_ctx;
 
 class ChatInputListener : public gcn::ActionListener {
     virtual void action(const std::string &) {
-        if (!OnlineContext->getCurrentChannel().empty()) {
-            OnlineContext->sendText(chatInput->getText());
+        if (!_ctx.getCurrentChannel().empty()) {
+            _ctx.sendText(chatInput->getText());
             chatInput->setText("");
         }
     }
@@ -1774,15 +1774,15 @@ class ChatInputListener : public gcn::ActionListener {
 
 class UsernameInputListener : public gcn::ActionListener {
     virtual void action(const std::string &) {
-        OnlineContext->setUsername(username->getText());
-        OnlineContext->setPassword(password->getText());
+        _ctx.setUsername(username->getText());
+        _ctx.setPassword(password->getText());
     }
 };
 
 class PasswordInputListener : public gcn::ActionListener {
     virtual void action(const std::string &) {
-        OnlineContext->setUsername(username->getText());
-        OnlineContext->setPassword(password->getText());
+        _ctx.setUsername(username->getText());
+        _ctx.setPassword(password->getText());
     }
 };
 
@@ -1795,7 +1795,7 @@ static int CclGoOnline(lua_State* l) {
     gcn::Widget *oldTop = Gui->getTop();
     Gui->setUseDirtyDrawing(false);
 
-    OnlineContext->setState(new ConnectState());
+    _ctx.setState(new ConnectState());
 
     onlineServiceContainer = new gcn::Container();
     onlineServiceContainer->setDimension(gcn::Rectangle(0, 0, Video.Width, Video.Height));
@@ -1869,17 +1869,17 @@ static int CclGoOnline(lua_State* l) {
     InterfaceState = IfaceStateNormal;
     UI.SelectedViewport = UI.Viewports;
     while (1) {
-        if (OnlineContext->isConnected()) {
+        if (_ctx.isConnected()) {
             loginWindow->setVisible(false);
 
             if ((FrameCounter % (FRAMES_PER_SECOND * 5)) == 0) {
-                OnlineContext->refreshGames();
-                OnlineContext->refreshFriends();
+                _ctx.refreshGames();
+                _ctx.refreshFriends();
             }
 
             if ((FrameCounter % (FRAMES_PER_SECOND * 1)) == 0) {
                 static_cast<gcn::TextBox*>(gamelistArea->getContent())->setText("");
-                for (auto g : OnlineContext->getGames()) {
+                for (auto g : _ctx.getGames()) {
                     static_cast<gcn::TextBox*>(gamelistArea->getContent())->addRow(g->getMap() + " " +
                                                                                    g->getCreator() + " " +
                                                                                    g->getGameType() + " " +
@@ -1888,25 +1888,25 @@ static int CclGoOnline(lua_State* l) {
                 }
 
                 static_cast<gcn::TextBox*>(usersArea->getContent())->setText("");
-                for (auto u : OnlineContext->getUsers()) {
+                for (auto u : _ctx.getUsers()) {
                     static_cast<gcn::TextBox*>(usersArea->getContent())->addRow(u);
                 }
 
                 static_cast<gcn::TextBox*>(channelsArea->getContent())->setText("");
-                for (auto u : OnlineContext->getChannels()) {
+                for (auto u : _ctx.getChannels()) {
                     static_cast<gcn::TextBox*>(channelsArea->getContent())->addRow(u);
                 }
 
                 static_cast<gcn::TextBox*>(friendsArea->getContent())->setText("");
-                for (auto u : OnlineContext->getFriends()) {
+                for (auto u : _ctx.getFriends()) {
                     static_cast<gcn::TextBox*>(friendsArea->getContent())->addRow(u->getName() + ", " + u->getStatus() + ", " + u->getProduct());
                 }
             }
         }
 
-        while (!OnlineContext->getInfo()->empty()) {
-            static_cast<gcn::TextBox*>(messageArea->getContent())->addRow(OnlineContext->getInfo()->front());
-            OnlineContext->getInfo()->pop();
+        while (!_ctx.getInfo()->empty()) {
+            static_cast<gcn::TextBox*>(messageArea->getContent())->addRow(_ctx.getInfo()->front());
+            _ctx.getInfo()->pop();
         }
 
         Gui->draw();

From 46049c2bdcc7d5a7663417cf987ca3f146e07321 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 24 Sep 2020 09:44:21 +0200
Subject: [PATCH 23/70] Fix warnings and build on Windows

---
 src/network/online_service.cpp | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 2ae7f0223..e5490fd63 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1,7 +1,11 @@
 #include "online_service.h"
 #include "results.h"
 
+#ifdef USE_WIN32
+#include <winsock2.h>
+#else
 #include <arpa/inet.h>
+#endif
 #include <clocale>
 #include <cstdio>
 #include <cstdlib>
@@ -17,7 +21,9 @@
 #include <stdexcept>
 #include <string>
 #include <tuple>
+#ifndef USE_WIN32
 #include <unistd.h>
+#endif
 #include <vector>
 
 #ifdef USE_WIN32
@@ -275,8 +281,8 @@ private:
     }
 
     uint8_t *buf;
-    int sz;
-    int pos;
+    unsigned int sz;
+    unsigned int pos;
     int length_pos;
 };
 
@@ -639,7 +645,7 @@ public:
     void sendText(std::string txt) {
         // C>S 0x0E SID_CHATCOMMAND
         int pos = 0;
-        for (int pos = 0; pos < txt.size(); pos += 220) {
+        for (unsigned int pos = 0; pos < txt.size(); pos += 220) {
             std::string text = txt.substr(pos, pos + 220);
             if (pos + 220 < txt.size()) {
                 text += "...";
@@ -1693,9 +1699,14 @@ class ConnectState : public NetworkState {
         // (UINT32) Local IP
         if (CNetworkParameter::Instance.localHost.compare("127.0.0.1")) {
             // user set a custom local ip, use that
+#ifdef USE_WIN32
+            uint32_t addr = inet_addr(CNetworkParameter::Instance.localHost.c_str());
+            buffer.serialize32(addr);
+#else
             struct in_addr addr;
             inet_aton(CNetworkParameter::Instance.localHost.c_str(), &addr);
             buffer.serialize32(addr.s_addr);
+#endif
         } else {
             unsigned long ips[20];
             int networkNumInterfaces = NetworkFildes.GetSocketAddresses(ips, 20);

From f80b13093b0887d75e62987bca36d4d40b7f7565 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Wed, 30 Sep 2020 17:57:10 +0200
Subject: [PATCH 24/70] rework GoOnline to leave gui control to lua

---
 src/network/online_service.cpp | 305 +++++++++++++--------------------
 1 file changed, 117 insertions(+), 188 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index e5490fd63..918c7a50d 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -623,6 +623,10 @@ public:
         return !getCurrentChannel().empty();
     }
 
+    boolean isDisconnected() {
+        return state == NULL;
+    }
+
     // User and UI actions
     void disconnect() {
         if (isConnected()) {
@@ -1758,206 +1762,131 @@ class ConnectState : public NetworkState {
     }
 };
 
-extern gcn::Gui *Gui;
-static gcn::Container *onlineServiceContainer;
-static gcn::ScrollArea *messageArea;
-static gcn::ScrollArea *usersArea;
-static gcn::ScrollArea *friendsArea;
-static gcn::ScrollArea *channelsArea;
-static gcn::ScrollArea *gamelistArea;
-static gcn::TextField *chatInput;
-static gcn::Window *loginWindow;
-static gcn::Container *loginWindowContainer;
-static gcn::TextField *username;
-static gcn::TextField *password;
-
 static Context _ctx;
 OnlineContext *OnlineContextHandler = &_ctx;
 
-class ChatInputListener : public gcn::ActionListener {
-    virtual void action(const std::string &) {
-        if (!_ctx.getCurrentChannel().empty()) {
-            _ctx.sendText(chatInput->getText());
-            chatInput->setText("");
-        }
-    }
-};
-
-class UsernameInputListener : public gcn::ActionListener {
-    virtual void action(const std::string &) {
-        _ctx.setUsername(username->getText());
-        _ctx.setPassword(password->getText());
-    }
-};
-
-class PasswordInputListener : public gcn::ActionListener {
-    virtual void action(const std::string &) {
-        _ctx.setUsername(username->getText());
-        _ctx.setPassword(password->getText());
-    }
-};
-
+/**
+ ** The assumption is that the lists of users, friends, and games are cleared by
+ ** the caller before calling this.
+ **
+ ** Lua arguments and return values:
+ **
+ ** @param username: login username, only used if not logged in
+ ** @param password: login password, only used if not logged in
+ ** @param message: a message to send, only used if logged in
+ ** @param AddMessage: a callback to add new messages. (str) -> nil
+ ** @param AddUser: a callback to add an online username. (str) -> nil
+ ** @param AddFriend: a callback to add an online friend. (name, product, status) -> nil
+ ** @param AddChannel: a callback to add a channel. (str) -> nil
+ ** @param ActiveChannel: a callback to report the currently active channel. (str) -> nil
+ ** @param AddGame: a callback to add an advertised game. (map, creator, gametype, settings, max-players) -> nil
+ **
+ ** @return status - "online" if successfully logged in, "pending" if waiting for connection, or error string
+ */
 static int CclGoOnline(lua_State* l) {
-    std::string nc, rc;
-    GetDefaultTextColors(nc, rc);
+    std::string username = "";
+    std::string password = "";
+    std::string message = "";
 
-    const EventCallback *old_callbacks = GetCallbacks();
+    LuaCallback *AddMessage = NULL;
+    LuaCallback *AddUser = NULL;
+    LuaCallback *AddFriend = NULL;
+    LuaCallback *AddGame = NULL;
+    LuaCallback *AddChannel = NULL;
+    LuaCallback *ActiveChannel = NULL;
 
-    gcn::Widget *oldTop = Gui->getTop();
-    Gui->setUseDirtyDrawing(false);
-
-    _ctx.setState(new ConnectState());
-
-    onlineServiceContainer = new gcn::Container();
-    onlineServiceContainer->setDimension(gcn::Rectangle(0, 0, Video.Width, Video.Height));
-    onlineServiceContainer->setOpaque(false);
-    Gui->setTop(onlineServiceContainer);
-
-    static int chatInputHeight = 24;
-    messageArea = new gcn::ScrollArea(new gcn::TextBox());
-    messageArea->setBackgroundColor(gcn::Color(200, 200, 120));
-    messageArea->setSize(Video.Width * 0.7, Video.Height * 0.7 - chatInputHeight);
-    static_cast<gcn::TextBox*>(messageArea->getContent())->setEditable(false);
-    onlineServiceContainer->add(messageArea, Video.Width * 0.3, Video.Height * 0.3);
-
-    chatInput = new gcn::TextField();
-    chatInput->setSize(Video.Width * 0.7, chatInputHeight);
-    onlineServiceContainer->add(chatInput, Video.Width * 0.3, Video.Height * 0.7 - chatInputHeight);
-    chatInput->addActionListener(new ChatInputListener());
-
-    usersArea = new gcn::ScrollArea(new gcn::TextBox());
-    usersArea->setBackgroundColor(gcn::Color(120, 200, 200));
-    usersArea->setSize(Video.Width * 0.1, Video.Height * 0.7);
-    static_cast<gcn::TextBox*>(usersArea->getContent())->setEditable(false);
-    onlineServiceContainer->add(usersArea, Video.Width * 0.2, Video.Height * 0.3);
-
-    friendsArea = new gcn::ScrollArea(new gcn::TextBox());
-    friendsArea->setBackgroundColor(gcn::Color(200, 120, 200));
-    friendsArea->setSize(Video.Width * 0.1, Video.Height * 0.5);
-    static_cast<gcn::TextBox*>(friendsArea->getContent())->setEditable(false);
-    onlineServiceContainer->add(friendsArea, 0, 0);
-
-    channelsArea = new gcn::ScrollArea(new gcn::TextBox());
-    channelsArea->setBackgroundColor(gcn::Color(255, 255, 255));
-    channelsArea->setSize(Video.Width * 0.1, Video.Height * 0.5);
-    static_cast<gcn::TextBox*>(channelsArea->getContent())->setEditable(false);
-    onlineServiceContainer->add(channelsArea, 0, Video.Height * 0.5);
-
-    gamelistArea = new gcn::ScrollArea(new gcn::TextBox());
-    gamelistArea->setBackgroundColor(gcn::Color(120, 120, 120));
-    gamelistArea->setSize(Video.Width * 0.8, Video.Height * 0.3);
-    static_cast<gcn::TextBox*>(gamelistArea->getContent())->setEditable(false);
-    onlineServiceContainer->add(gamelistArea, Video.Width * 0.2, 0);
-
-    loginWindow = new gcn::Window();
-    loginWindow->setBaseColor(gcn::Color(120, 120, 120, 120));
-    loginWindow->setCaption("Username / Password");
-    loginWindow->setWidth(Video.Width / 2);
-    loginWindow->setHeight(chatInputHeight * 8);
-
-    loginWindowContainer = new gcn::Container();
-    loginWindowContainer->setWidth(loginWindow->getWidth());
-    loginWindowContainer->setHeight(loginWindow->getHeight());
-    loginWindowContainer->setOpaque(false);
-    loginWindow->setContent(loginWindowContainer);
-
-    username = new gcn::TextField("");
-    username->setSize(Video.Width * 0.3, chatInputHeight);
-    username->addActionListener(new UsernameInputListener());
-    loginWindowContainer->add(username, Video.Width * 0.1, chatInputHeight);
-
-    password = new gcn::TextField("");
-    password->setSize(Video.Width * 0.3, chatInputHeight);
-    password->addActionListener(new PasswordInputListener());
-    loginWindowContainer->add(password, Video.Width * 0.1, chatInputHeight * 4);
-
-    onlineServiceContainer->add(loginWindow, Video.Width / 4, Video.Height / 2 - chatInputHeight * 4);
-
-    username->requestFocus();
-
-    SetVideoSync();
-    GameCursor = UI.Point.Cursor;
-    InterfaceState = IfaceStateNormal;
-    UI.SelectedViewport = UI.Viewports;
-    while (1) {
-        if (_ctx.isConnected()) {
-            loginWindow->setVisible(false);
-
-            if ((FrameCounter % (FRAMES_PER_SECOND * 5)) == 0) {
-                _ctx.refreshGames();
-                _ctx.refreshFriends();
-            }
-
-            if ((FrameCounter % (FRAMES_PER_SECOND * 1)) == 0) {
-                static_cast<gcn::TextBox*>(gamelistArea->getContent())->setText("");
-                for (auto g : _ctx.getGames()) {
-                    static_cast<gcn::TextBox*>(gamelistArea->getContent())->addRow(g->getMap() + " " +
-                                                                                   g->getCreator() + " " +
-                                                                                   g->getGameType() + " " +
-                                                                                   g->getGameSettings() + " " +
-                                                                                   std::to_string(g->maxPlayers()));
-                }
-
-                static_cast<gcn::TextBox*>(usersArea->getContent())->setText("");
-                for (auto u : _ctx.getUsers()) {
-                    static_cast<gcn::TextBox*>(usersArea->getContent())->addRow(u);
-                }
-
-                static_cast<gcn::TextBox*>(channelsArea->getContent())->setText("");
-                for (auto u : _ctx.getChannels()) {
-                    static_cast<gcn::TextBox*>(channelsArea->getContent())->addRow(u);
-                }
-
-                static_cast<gcn::TextBox*>(friendsArea->getContent())->setText("");
-                for (auto u : _ctx.getFriends()) {
-                    static_cast<gcn::TextBox*>(friendsArea->getContent())->addRow(u->getName() + ", " + u->getStatus() + ", " + u->getProduct());
-                }
-            }
-        }
-
-        while (!_ctx.getInfo()->empty()) {
-            static_cast<gcn::TextBox*>(messageArea->getContent())->addRow(_ctx.getInfo()->front());
-            _ctx.getInfo()->pop();
-        }
-
-        Gui->draw();
-        DrawCursor();
-        Invalidate();
-        RealizeVideoMemory();
-
-        WaitEventsOneFrame();
+    if (_ctx.isDisconnected()) {
+        _ctx.setState(new ConnectState());
     }
 
-    CleanModules();
-    LoadCcl(Parameters::Instance.luaStartFilename); // Reload the main config file
-    // PreMenuSetup();
+    for (lua_pushnil(l); lua_next(l, 2); lua_pop(l, 1)) {
+        const char *value = LuaToString(l, -2);
 
-    InterfaceState = IfaceStateMenu;
-    GameCursor = UI.Point.Cursor;
+        if (!strcmp(value, "username")) {
+            username = std::string(LuaToString(l, -1));
+        } else if (!strcmp(value, "password")) {
+            password = std::string(LuaToString(l, -1));
+        } else if (!strcmp(value, "message")) {
+            message = std::string(LuaToString(l, -1));
+        } else if (!strcmp(value, "AddMessage")) {
+            AddMessage = new LuaCallback(l, -1);
+        } else if (!strcmp(value, "AddUser")) {
+            AddUser = new LuaCallback(l, -1);
+        } else if (!strcmp(value, "AddFriend")) {
+            AddFriend = new LuaCallback(l, -1);
+        } else if (!strcmp(value, "AddGame")) {
+            AddGame = new LuaCallback(l, -1);
+        } else if (!strcmp(value, "AddChannel")) {
+            AddChannel = new LuaCallback(l, -1);
+        } else if (!strcmp(value, "ActiveChannel")) {
+            ActiveChannel = new LuaCallback(l, -1);
+        } else {
+            LuaError(l, "Unsupported tag: %s" _C_ value);
+        }
+    }
 
-    SetCallbacks(old_callbacks);
-    Gui->setTop(oldTop);
-    SetDefaultTextColors(nc, rc);
+    if (_ctx.isConnected()) {
+        if (!message.empty() && !_ctx.getCurrentChannel().empty()) {
+            _ctx.sendText(message);
+        }
 
-    Video.ClearScreen();
-    Invalidate();
+        if ((FrameCounter % (FRAMES_PER_SECOND * 5)) == 0) {
+            _ctx.refreshGames();
+            _ctx.refreshFriends();
+        }
 
-    delete onlineServiceContainer;
-    delete messageArea->getContent();
-    delete messageArea;
-    delete usersArea->getContent();
-    delete usersArea;
-    delete friendsArea->getContent();
-    delete friendsArea;
-    delete channelsArea->getContent();
-    delete channelsArea;
-    delete gamelistArea->getContent();
-    delete gamelistArea;
-    delete chatInput;
-    delete loginWindow;
-    delete loginWindowContainer;
-    delete username;
-    delete password;
+        if ((FrameCounter % (FRAMES_PER_SECOND * 1)) == 0) {
+            if (AddGame) {
+                for (auto g : _ctx.getGames()) {
+                    AddGame->pushPreamble();
+                    AddGame->pushString(g->getMap());
+                    AddGame->pushString(g->getCreator());
+                    AddGame->pushString(g->getGameType());
+                    AddGame->pushString(g->getGameSettings());
+                    AddGame->pushInteger(g->maxPlayers());
+                    AddGame->run(0);
+                }
+            }
+            if (AddUser) {
+                for (auto u : _ctx.getUsers()) {
+                    AddUser->pushPreamble();
+                    AddUser->pushString(u);
+                    AddUser->run(0);
+                }
+            }
+            if (AddChannel) {
+                for (auto u : _ctx.getChannels()) {
+                    AddChannel->pushPreamble();
+                    AddChannel->pushString(u);
+                    AddChannel->run(0);
+                }
+            }
+            if (ActiveChannel) {
+                ActiveChannel->pushPreamble();
+                ActiveChannel->pushString(_ctx.getCurrentChannel());
+                ActiveChannel->run(0);
+            }
+            if (AddFriend) {
+                for (auto u : _ctx.getFriends()) {
+                    AddFriend->pushPreamble();
+                    AddFriend->pushString(u->getName());
+                    AddFriend->pushString(u->getProduct());
+                    AddFriend->pushString(u->getStatus());
+                    AddFriend->run(0);
+                }
+            }
+            while (!_ctx.getInfo()->empty()) {
+                if (AddMessage) {
+                    AddMessage->pushPreamble();
+                    AddMessage->pushString(_ctx.getInfo()->front());
+                    _ctx.getInfo()->pop();
+                }
+            }
+        }
+    } else if (!username.empty() && !password.empty()) {
+        _ctx.setUsername(username);
+        _ctx.setPassword(password);
+    }
 
     return 0;
 }

From 964317234b016809b74a18413258c35184dd8abf Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Wed, 30 Sep 2020 18:12:04 +0200
Subject: [PATCH 25/70] return results from GoOnline

---
 src/network/online_service.cpp | 31 ++++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 918c7a50d..6935631cd 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -627,6 +627,10 @@ public:
         return state == NULL;
     }
 
+    std::string getLastError() {
+        return lastError;
+    }
+
     // User and UI actions
     void disconnect() {
         if (isConnected()) {
@@ -883,7 +887,10 @@ public:
 
     void showInfo(std::string arg) { info.push("*** " + arg + " ***"); }
 
-    void showError(std::string arg) { info.push("!!! " + arg + " !!!"); }
+    void showError(std::string arg) {
+        info.push("!!! " + arg + " !!!");
+        lastError = arg;
+    }
 
     void showChat(std::string arg) { info.push(arg); }
 
@@ -971,6 +978,8 @@ private:
     std::string username;
     uint32_t password[5]; // xsha1 hash of password
 
+    std::string lastError;
+
     std::string currentChannel;
     std::set<std::string> userList;
     std::vector<std::string> channelList;
@@ -995,7 +1004,7 @@ public:
     virtual void doOneStep(Context *ctx) {
         if (!hasPrinted) {
             std::cout << message << std::endl;
-            ctx->showInfo(message);
+            ctx->showError(message);
             hasPrinted = true;
             ctx->disconnect();
         }
@@ -1825,7 +1834,16 @@ static int CclGoOnline(lua_State* l) {
         }
     }
 
-    if (_ctx.isConnected()) {
+    _ctx.doOneStep();
+
+    if (!_ctx.getLastError().empty()) {
+        lua_pushstring(l, lastError);
+    } else if (!username.empty() && !password.empty()) {
+        lua_pushstring(l, "pending");
+        _ctx.setUsername(username);
+        _ctx.setPassword(password);
+    } else if (_ctx.isConnected()) {
+        lua_pushstring(l, "online");
         if (!message.empty() && !_ctx.getCurrentChannel().empty()) {
             _ctx.sendText(message);
         }
@@ -1883,12 +1901,11 @@ static int CclGoOnline(lua_State* l) {
                 }
             }
         }
-    } else if (!username.empty() && !password.empty()) {
-        _ctx.setUsername(username);
-        _ctx.setPassword(password);
+    } else {
+        lua_pushstring("unknown");
     }
 
-    return 0;
+    return 1;
 }
 
 static int CclStopAdvertisingOnlineGame(lua_State* l) {

From d421df2e22ddba1ffde6518043ab67bec8c4eaa4 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Wed, 30 Sep 2020 18:22:21 +0200
Subject: [PATCH 26/70] better GoOnline docs and states

---
 src/network/online_service.cpp | 33 +++++++++++++++++++++++++--------
 1 file changed, 25 insertions(+), 8 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 6935631cd..9c9c82840 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -619,11 +619,15 @@ public:
         delete host;
     }
 
-    boolean isConnected() {
+    bool isConnected() {
         return !getCurrentChannel().empty();
     }
 
-    boolean isDisconnected() {
+    bool isConnecting() {
+        return !isDisconnected() && !isConnected() && !username.empty() && hasPassword;
+    }
+
+    bool isDisconnected() {
         return state == NULL;
     }
 
@@ -930,8 +934,10 @@ public:
             for (int i = 0; i < sizeof(password); i++) {
                 this->password[i] = 0;
             }
+            hasPassword = false;
         } else {
             pvpgn::sha1_hash(&password, pw.length(), pw.c_str());
+            hasPassword = true;
         }
     }
 
@@ -977,6 +983,7 @@ private:
 
     std::string username;
     uint32_t password[5]; // xsha1 hash of password
+    bool hasPassword;
 
     std::string lastError;
 
@@ -1778,6 +1785,12 @@ OnlineContext *OnlineContextHandler = &_ctx;
  ** The assumption is that the lists of users, friends, and games are cleared by
  ** the caller before calling this.
  **
+ ** The call order should be this:
+ **  1) call once with username + password
+ **  2) call without args until you get a result that is not "pending"
+ **  2a) if result is "online", call repeatedly with callbacks or messages
+ **  2b) if result is an error, you may go to 1 after fixing the error
+ **
  ** Lua arguments and return values:
  **
  ** @param username: login username, only used if not logged in
@@ -1804,10 +1817,6 @@ static int CclGoOnline(lua_State* l) {
     LuaCallback *AddChannel = NULL;
     LuaCallback *ActiveChannel = NULL;
 
-    if (_ctx.isDisconnected()) {
-        _ctx.setState(new ConnectState());
-    }
-
     for (lua_pushnil(l); lua_next(l, 2); lua_pop(l, 1)) {
         const char *value = LuaToString(l, -2);
 
@@ -1836,8 +1845,16 @@ static int CclGoOnline(lua_State* l) {
 
     _ctx.doOneStep();
 
+    if (!username.empty() && !password.empty()) {
+        if (_ctx.isDisconnected()) {
+            _ctx.setState(new ConnectState());
+        }
+    }
+
     if (!_ctx.getLastError().empty()) {
-        lua_pushstring(l, lastError);
+        lua_pushstring(l, _ctx.getLastError().c_str());
+    } else if (_ctx.isConnecting()) {
+        lua_pushstring(l, "pending");
     } else if (!username.empty() && !password.empty()) {
         lua_pushstring(l, "pending");
         _ctx.setUsername(username);
@@ -1902,7 +1919,7 @@ static int CclGoOnline(lua_State* l) {
             }
         }
     } else {
-        lua_pushstring("unknown");
+        lua_pushstring(l, "unknown");
     }
 
     return 1;

From 9ab1d216d144fe69519eaadf992668f5afdde480 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 1 Oct 2020 19:00:56 +0200
Subject: [PATCH 27/70] get host/port from lua

---
 src/network/online_service.cpp | 39 +++++++++++++++++++++++++++-------
 1 file changed, 31 insertions(+), 8 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 9c9c82840..55054461b 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1793,6 +1793,8 @@ OnlineContext *OnlineContextHandler = &_ctx;
  **
  ** Lua arguments and return values:
  **
+ ** @param host: hostname of the online service. Inititates a new connection.
+ ** @param post: port of the online service. Inititates a new connection.
  ** @param username: login username, only used if not logged in
  ** @param password: login password, only used if not logged in
  ** @param message: a message to send, only used if logged in
@@ -1809,6 +1811,8 @@ static int CclGoOnline(lua_State* l) {
     std::string username = "";
     std::string password = "";
     std::string message = "";
+    std::string host = "";
+    int port = 0;
 
     LuaCallback *AddMessage = NULL;
     LuaCallback *AddUser = NULL;
@@ -1816,14 +1820,24 @@ static int CclGoOnline(lua_State* l) {
     LuaCallback *AddGame = NULL;
     LuaCallback *AddChannel = NULL;
     LuaCallback *ActiveChannel = NULL;
+    std::string newActiveChannel = "";
 
-    for (lua_pushnil(l); lua_next(l, 2); lua_pop(l, 1)) {
+    LuaCheckArgs(l, 1);
+    if (!lua_istable(l, 1)) {
+        LuaError(l, "incorrect argument");
+    }
+
+    for (lua_pushnil(l); lua_next(l, 1); lua_pop(l, 1)) {
         const char *value = LuaToString(l, -2);
 
         if (!strcmp(value, "username")) {
             username = std::string(LuaToString(l, -1));
         } else if (!strcmp(value, "password")) {
             password = std::string(LuaToString(l, -1));
+        } else if (!strcmp(value, "host")) {
+            host = std::string(LuaToString(l, -1));
+        } else if (!strcmp(value, "port")) {
+            port = LuaToNumber(l, -1);
         } else if (!strcmp(value, "message")) {
             message = std::string(LuaToString(l, -1));
         } else if (!strcmp(value, "AddMessage")) {
@@ -1837,22 +1851,29 @@ static int CclGoOnline(lua_State* l) {
         } else if (!strcmp(value, "AddChannel")) {
             AddChannel = new LuaCallback(l, -1);
         } else if (!strcmp(value, "ActiveChannel")) {
-            ActiveChannel = new LuaCallback(l, -1);
+            if (lua_isstring(l, -1)) {
+                newActiveChannel = LuaToString(l, -1);
+            } else {
+                ActiveChannel = new LuaCallback(l, -1);
+            }
         } else {
             LuaError(l, "Unsupported tag: %s" _C_ value);
         }
     }
 
-    _ctx.doOneStep();
-
-    if (!username.empty() && !password.empty()) {
+    if (!host.empty() && port) {
         if (_ctx.isDisconnected()) {
+            _ctx.setHost(new CHost(host.c_str(), port));
             _ctx.setState(new ConnectState());
         }
     }
 
-    if (!_ctx.getLastError().empty()) {
-        lua_pushstring(l, _ctx.getLastError().c_str());
+    _ctx.doOneStep();
+
+    std::string error = _ctx.getLastError();
+
+    if (!error.empty()) {
+        lua_pushstring(l, error.c_str());
     } else if (_ctx.isConnecting()) {
         lua_pushstring(l, "pending");
     } else if (!username.empty() && !password.empty()) {
@@ -1865,7 +1886,7 @@ static int CclGoOnline(lua_State* l) {
             _ctx.sendText(message);
         }
 
-        if ((FrameCounter % (FRAMES_PER_SECOND * 5)) == 0) {
+        if ((FrameCounter % (FRAMES_PER_SECOND * 20)) == 0) {
             _ctx.refreshGames();
             _ctx.refreshFriends();
         }
@@ -1900,6 +1921,8 @@ static int CclGoOnline(lua_State* l) {
                 ActiveChannel->pushPreamble();
                 ActiveChannel->pushString(_ctx.getCurrentChannel());
                 ActiveChannel->run(0);
+            } else if (!newActiveChannel.empty()) {
+                _ctx.setCurrentChannel(newActiveChannel);
             }
             if (AddFriend) {
                 for (auto u : _ctx.getFriends()) {

From 485db94bd039c2c2478512d7053388a08b4f60d9 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Tue, 6 Oct 2020 22:28:03 +0200
Subject: [PATCH 28/70] tweak and fix lua GoOnline

---
 src/network/online_service.cpp | 94 ++++++++++++++++++----------------
 1 file changed, 49 insertions(+), 45 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 55054461b..1a8b9f1a5 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1886,59 +1886,63 @@ static int CclGoOnline(lua_State* l) {
             _ctx.sendText(message);
         }
 
-        if ((FrameCounter % (FRAMES_PER_SECOND * 20)) == 0) {
-            _ctx.refreshGames();
+        // refresh friends every 5 minutes
+        if ((FrameCounter % (FRAMES_PER_SECOND * 300)) == 0) {
             _ctx.refreshFriends();
         }
 
-        if ((FrameCounter % (FRAMES_PER_SECOND * 1)) == 0) {
-            if (AddGame) {
-                for (auto g : _ctx.getGames()) {
-                    AddGame->pushPreamble();
-                    AddGame->pushString(g->getMap());
-                    AddGame->pushString(g->getCreator());
-                    AddGame->pushString(g->getGameType());
-                    AddGame->pushString(g->getGameSettings());
-                    AddGame->pushInteger(g->maxPlayers());
-                    AddGame->run(0);
-                }
+        // refresh games every 30 seconds
+        if ((FrameCounter % (FRAMES_PER_SECOND * 30)) == 0) {
+            _ctx.refreshGames();
+        }
+
+        if (AddGame) {
+            for (auto g : _ctx.getGames()) {
+                AddGame->pushPreamble();
+                AddGame->pushString(g->getMap());
+                AddGame->pushString(g->getCreator());
+                AddGame->pushString(g->getGameType());
+                AddGame->pushString(g->getGameSettings());
+                AddGame->pushInteger(g->maxPlayers());
+                AddGame->run(0);
             }
-            if (AddUser) {
-                for (auto u : _ctx.getUsers()) {
-                    AddUser->pushPreamble();
-                    AddUser->pushString(u);
-                    AddUser->run(0);
-                }
+        }
+        if (AddUser) {
+            for (auto u : _ctx.getUsers()) {
+                AddUser->pushPreamble();
+                AddUser->pushString(u);
+                AddUser->run(0);
             }
-            if (AddChannel) {
-                for (auto u : _ctx.getChannels()) {
-                    AddChannel->pushPreamble();
-                    AddChannel->pushString(u);
-                    AddChannel->run(0);
-                }
+        }
+        if (AddChannel) {
+            for (auto u : _ctx.getChannels()) {
+                AddChannel->pushPreamble();
+                AddChannel->pushString(u);
+                AddChannel->run(0);
             }
-            if (ActiveChannel) {
-                ActiveChannel->pushPreamble();
-                ActiveChannel->pushString(_ctx.getCurrentChannel());
-                ActiveChannel->run(0);
-            } else if (!newActiveChannel.empty()) {
-                _ctx.setCurrentChannel(newActiveChannel);
-            }
-            if (AddFriend) {
-                for (auto u : _ctx.getFriends()) {
-                    AddFriend->pushPreamble();
-                    AddFriend->pushString(u->getName());
-                    AddFriend->pushString(u->getProduct());
-                    AddFriend->pushString(u->getStatus());
-                    AddFriend->run(0);
-                }
+        }
+        if (ActiveChannel) {
+            ActiveChannel->pushPreamble();
+            ActiveChannel->pushString(_ctx.getCurrentChannel());
+            ActiveChannel->run(0);
+        } else if (!newActiveChannel.empty()) {
+            _ctx.setCurrentChannel(newActiveChannel);
+        }
+        if (AddFriend) {
+            for (auto u : _ctx.getFriends()) {
+                AddFriend->pushPreamble();
+                AddFriend->pushString(u->getName());
+                AddFriend->pushString(u->getProduct());
+                AddFriend->pushString(u->getStatus());
+                AddFriend->run(0);
             }
+        }
+        if (AddMessage) {
             while (!_ctx.getInfo()->empty()) {
-                if (AddMessage) {
-                    AddMessage->pushPreamble();
-                    AddMessage->pushString(_ctx.getInfo()->front());
-                    _ctx.getInfo()->pop();
-                }
+                AddMessage->pushPreamble();
+                AddMessage->pushString(_ctx.getInfo()->front());
+                AddMessage->run(0);
+                _ctx.getInfo()->pop();
             }
         }
     } else {

From 404fdc6047706a26e365f0736f101536b2f850ac Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Wed, 7 Oct 2020 12:03:37 +0200
Subject: [PATCH 29/70] fix requesting user info

---
 src/network/online_service.cpp | 41 ++++++++++++++++++++++++++--------
 1 file changed, 32 insertions(+), 9 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 1a8b9f1a5..4c7e8ecf5 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -232,6 +232,12 @@ public:
         *view = htonl(data);
         pos += sizeof(data);
     };
+    void serialize32NativeByteOrder(uint32_t data) {
+        ensureSpace(sizeof(data));
+        uint32_t *view = reinterpret_cast<uint32_t *>(buf + pos);
+        *view = data;
+        pos += sizeof(data);
+    };
     void serialize16(uint16_t data) {
         ensureSpace(sizeof(data));
         uint16_t *view = reinterpret_cast<uint16_t *>(buf + pos);
@@ -608,6 +614,12 @@ public:
         this->clientToken = MyRand();
         this->username = "";
         setPassword("");
+        defaultUserKeys.push_back("record\\GAME\\0\\wins");
+        defaultUserKeys.push_back("record\\GAME\\0\\losses");
+        defaultUserKeys.push_back("record\\GAME\\0\\disconnects");
+        defaultUserKeys.push_back("record\\GAME\\0\\last game");
+        defaultUserKeys.push_back("record\\GAME\\0\\last game result");
+
     }
 
     ~Context() {
@@ -670,15 +682,14 @@ public:
 
     void requestExtraUserInfo(std::string username) {
         BNCSOutputStream msg(0x26);
-        msg.serialize32(1); // num accounts
-        msg.serialize32(5); // num keys
-        msg.serialize32((uint32_t) extendedInfoIdx.size());
+        msg.serialize32NativeByteOrder(1); // num accounts
+        msg.serialize32NativeByteOrder(defaultUserKeys.size()); // num keys
+        msg.serialize32NativeByteOrder((uint32_t) extendedInfoNames.size());
+        extendedInfoNames.push_back(username);
         msg.serialize(username.c_str());
-        msg.serialize("record\\GAME\\0\\wins");
-        msg.serialize("record\\GAME\\0\\losses");
-        msg.serialize("record\\GAME\\0\\disconnects");
-        msg.serialize("record\\GAME\\0\\last game");
-        msg.serialize("record\\GAME\\0\\last game result");
+        for (const auto key : defaultUserKeys) {
+            msg.serialize(key.c_str());
+        }
         msg.flush(getTCPSocket());
     }
 
@@ -885,6 +896,10 @@ public:
 
     void reportUserdata(uint32_t id, std::vector<std::string> values) {
         this->extendedInfoValues[id] = values;
+        showInfo("User Info for " + extendedInfoNames.at(id));
+        for (int i = 0; i < values.size(); i++) {
+            showInfo(defaultUserKeys.at(i) + ": " + values.at(i));
+        }
     }
 
     std::queue<std::string> *getInfo() { return &info; }
@@ -993,8 +1008,9 @@ private:
     std::queue<std::string> info;
     std::vector<Game*> games;
     std::vector<Friend*> friends;
-    std::map<std::string, uint32_t> extendedInfoIdx;
+    std::vector<std::string> extendedInfoNames;
     std::map<uint32_t, std::vector<std::string>> extendedInfoValues;
+    std::vector<std::string> defaultUserKeys;
 };
 
 int NetworkState::send(Context *ctx, BNCSOutputStream *buf) {
@@ -1798,6 +1814,7 @@ OnlineContext *OnlineContextHandler = &_ctx;
  ** @param username: login username, only used if not logged in
  ** @param password: login password, only used if not logged in
  ** @param message: a message to send, only used if logged in
+ ** @param userInfo: a username to request info for
  ** @param AddMessage: a callback to add new messages. (str) -> nil
  ** @param AddUser: a callback to add an online username. (str) -> nil
  ** @param AddFriend: a callback to add an online friend. (name, product, status) -> nil
@@ -1821,6 +1838,7 @@ static int CclGoOnline(lua_State* l) {
     LuaCallback *AddChannel = NULL;
     LuaCallback *ActiveChannel = NULL;
     std::string newActiveChannel = "";
+    std::string userInfo = "";
 
     LuaCheckArgs(l, 1);
     if (!lua_istable(l, 1)) {
@@ -1838,6 +1856,8 @@ static int CclGoOnline(lua_State* l) {
             host = std::string(LuaToString(l, -1));
         } else if (!strcmp(value, "port")) {
             port = LuaToNumber(l, -1);
+        } else if (!strcmp(value, "userInfo")) {
+            userInfo = std::string(LuaToString(l, -1));
         } else if (!strcmp(value, "message")) {
             message = std::string(LuaToString(l, -1));
         } else if (!strcmp(value, "AddMessage")) {
@@ -1945,6 +1965,9 @@ static int CclGoOnline(lua_State* l) {
                 _ctx.getInfo()->pop();
             }
         }
+        if (!userInfo.empty()) {
+            _ctx.requestExtraUserInfo(userInfo);
+        }
     } else {
         lua_pushstring(l, "unknown");
     }

From bf7a825fc50a22d003721892ecc1baba5d10b85a Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Wed, 7 Oct 2020 12:03:48 +0200
Subject: [PATCH 30/70] allow joining channel

---
 src/network/online_service.cpp | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 4c7e8ecf5..015de8e35 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -869,6 +869,15 @@ public:
         msg.flush(getTCPSocket());
     }
 
+    void joinChannel(std::string name) {
+        if (isConnected()) {
+            BNCSOutputStream join(0x0c);
+            join.serialize32(0x02); // forced join
+            join.serialize(name.c_str());
+            join.flush(getTCPSocket());
+        }
+    }
+
     // UI information
     void setCurrentChannel(std::string name) {
         this->currentChannel = name;
@@ -1194,8 +1203,8 @@ private:
             ctx->showChat("[BROADCAST]: " + text);
             break;
         case 0x07: // channel info
-            ctx->setCurrentChannel(username);
-            ctx->showInfo("Joined channel " + username);
+            ctx->setCurrentChannel(text);
+            ctx->showInfo("Joined channel " + text);
             break;
         case 0x09: // user flags update
             break;
@@ -1214,7 +1223,7 @@ private:
             ctx->showInfo("[INFO]: " + text);
             break;
         case 0x13: // error message
-            ctx->showError("[ERROR]: " + text);
+            ctx->showError("[SEVERE]: " + text);
             break;
         case 0x17: // emote
             break;
@@ -1946,7 +1955,7 @@ static int CclGoOnline(lua_State* l) {
             ActiveChannel->pushString(_ctx.getCurrentChannel());
             ActiveChannel->run(0);
         } else if (!newActiveChannel.empty()) {
-            _ctx.setCurrentChannel(newActiveChannel);
+            _ctx.joinChannel(newActiveChannel);
         }
         if (AddFriend) {
             for (auto u : _ctx.getFriends()) {

From 6a6260d03520219748cfacc387546eb01b22ebe7 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Wed, 7 Oct 2020 12:03:56 +0200
Subject: [PATCH 31/70] ignore this

---
 src/network/online_service.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 015de8e35..c4b93df84 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -917,7 +917,7 @@ public:
 
     void showError(std::string arg) {
         info.push("!!! " + arg + " !!!");
-        lastError = arg;
+        // lastError = arg;
     }
 
     void showChat(std::string arg) { info.push(arg); }

From 710bdfca409d8b0df449e29896b8297111d49e61 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Wed, 7 Oct 2020 13:16:39 +0200
Subject: [PATCH 32/70] echo our own chat messages

---
 src/network/online_service.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index c4b93df84..aafafbc8e 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -678,6 +678,7 @@ public:
             msg.serialize(text.c_str());
             msg.flush(getTCPSocket());
         }
+        showChat(username + ": " + txt);
     }
 
     void requestExtraUserInfo(std::string username) {

From 55eb35ebbb481d9902caf8f78f12d2d5513b0eb9 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Wed, 7 Oct 2020 13:16:52 +0200
Subject: [PATCH 33/70] implement the game advertising lua functions

---
 src/network/online_service.cpp | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index aafafbc8e..f71d57b25 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1986,11 +1986,23 @@ static int CclGoOnline(lua_State* l) {
 }
 
 static int CclStopAdvertisingOnlineGame(lua_State* l) {
-    return 0;
+    if (_ctx.isConnected()) {
+        _ctx.stopAdvertising();
+        lua_pushboolean(l, true);
+    } else {
+        lua_pushboolean(l, false);
+    }
+    return 1;
 }
 
 static int CclStartAdvertisingOnlineGame(lua_State* l) {
-    return 0;
+    if (_ctx.isConnected()) {
+        _ctx.startAdvertising();
+        lua_pushboolean(l, true);
+    } else {
+        lua_pushboolean(l, false);
+    }
+    return 1;
 }
 
 void OnlineServiceCclRegister() {

From 475588af99c978d6e5ccfab1207fb9d182c64611 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Wed, 7 Oct 2020 14:25:50 +0200
Subject: [PATCH 34/70] fix game creation message

---
 src/network/online_service.cpp | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index f71d57b25..f1c1f9ae7 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -748,20 +748,20 @@ public:
                 joinedPlayers++;
             }
         }
-        uint32_t state = 0x10; // disconnect always counts as loss
+        uint32_t state = 0x10000000; // disconnect always counts as loss
         if (joinedPlayers) {
-            state |= 0x04; // has players other than creator
+            state |= 0x04000000; // has players other than creator
         }
         if (joinedPlayers + 1 == maxSlots) {
-            state |= 0x02; // game is full
+            state |= 0x02000000; // game is full
         }
         if (isStarted) {
-            state |= 0x08; // game in progress
+            state |= 0x08000000; // game in progress
         }
         msg.serialize32(state);
         msg.serialize32(0x00); // uptime
-        msg.serialize16(0x0a); // game type - map settings
-        msg.serialize16(0x01); // sub game type
+        msg.serialize16(0x0300); // game type - map settings not supported on W2BN, so use FFA
+        msg.serialize16(0x0100); // sub game type
         msg.serialize32(0xff); // provider version constant
         msg.serialize32(0x00); // not ladder
         msg.serialize((gameNameFromUsername(getUsername())).c_str()); // game name
@@ -818,13 +818,22 @@ public:
             game_settings |= 0xC000;
         } else if (Map.Tileset->Name == "Orc Swamp") {
             game_settings |= 0x1C000;
+        } else {
+            game_settings |= 0x4000; // default to forest
         }
         statstring << std::hex << game_settings << ",";
 
         statstring << getUsername();
-        statstring.put(0x0d);
-        statstring << Map.Info.Filename;
-        statstring.put(0x0d);
+        statstring << "\r";
+        if (!Map.Info.Filename.empty()) {
+            statstring << Map.Info.Filename;
+        } else if (!Map.Info.Description.empty()) {
+            statstring << Map.Info.Description;
+        } else {
+            statstring << "unknown map";
+        }
+        statstring << "\r";
+        std::cout << statstring.str() << std::endl;
 
         msg.serialize(statstring.str().c_str());
         msg.flush(getTCPSocket());

From 7680e9773da6f51b5101eb7aa6e4aeaf6c33b186 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Wed, 7 Oct 2020 15:44:17 +0200
Subject: [PATCH 35/70] minor cleanup

---
 src/network/online_service.cpp | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index f1c1f9ae7..20fc60025 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -833,7 +833,6 @@ public:
             statstring << "unknown map";
         }
         statstring << "\r";
-        std::cout << statstring.str() << std::endl;
 
         msg.serialize(statstring.str().c_str());
         msg.flush(getTCPSocket());
@@ -1530,18 +1529,6 @@ void C2S_LOGONRESPONSE2::doOneStep(Context *ctx) {
         pvpgn::hash_to_bnhash((pvpgn::t_hash const *)pw, temp.passhash1);
         pvpgn::bnet_hash(&passhash2, sizeof(temp), &temp);	/* do the double hash */
 
-        // std::cout << std::endl << "Password 1 hash: ";
-        // for (int i = 0; i < 5; i++) {
-        //     std::cout << std::hex << pw[i] << " ";
-        // }
-        // std::cout << std::endl << "Password 2 hash: ";
-        // for (int i = 0; i < 5; i++) {
-        //     std::cout << std::hex << passhash2[i] << " ";
-        // }
-        // std::cout << std::endl;
-        // std::cout << "client token: " << *((uint32_t*)temp.ticks) << std::endl;
-        // std::cout << "server token: " << *((uint32_t*)temp.sessionkey) << std::endl;
-
         for (int i = 0; i < 20; i++) {
             logon.serialize8(reinterpret_cast<uint8_t*>(passhash2)[i]);
         }

From 36a561459c3fb65db3abaf03ff5c43f41da8e369 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Wed, 7 Oct 2020 16:50:24 +0200
Subject: [PATCH 36/70] fix parsing and returning games

---
 src/network/online_service.cpp | 56 ++++++++++++++++++++--------------
 1 file changed, 33 insertions(+), 23 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 20fc60025..4bbd5af5c 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -391,15 +391,23 @@ public:
         }
     };
 
+    int getSpeed() {
+        return std::stol(gameStats[3]);
+    }
+
     std::string getApproval() {
-        return "Not approved";
+        if (std::stol(gameStats[4]) == 0) {
+            return "Not approved";
+        } else {
+            return "Approved";
+        }
     }
 
     std::string getGameSettings() {
-        if (gameStats[9].empty()) {
+        if (gameStats[8].empty()) {
             return "Map default";
         }
-        long settings = std::stol(gameStats[9]);
+        long settings = std::stol(gameStats[8]);
         std::string result;
         if (settings & 0x200) {
             result += " 1 worker";
@@ -442,13 +450,11 @@ public:
     };
 
     std::string getCreator() {
-        int end = gameStats[10].find("\r");
-        return gameStats[10].substr(0, end);
+        return gameStats[9];
     };
 
     std::string getMap() {
-        int begin = gameStats[10].find("\r") + 1;
-        return gameStats[10].substr(begin);
+        return gameStats[10];
     };
 
     std::string getGameStatus() {
@@ -561,7 +567,7 @@ public:
         case 0x10:
             return "Iron Man Ladder";
         default:
-            return "Unknown";
+            return gameStats[5]; // just the code
         }
     }
 
@@ -569,13 +575,19 @@ private:
     void splitStatstring() {
         // statstring is a comma-delimited list of values
         int pos = 0;
+        int newpos = 0;
+        char sep[2] = {',', '\0'};
         while (true) {
-            int newpos = gameStatstring.find(",", pos);
-            gameStats.push_back(gameStatstring.substr(pos, newpos));
-            pos = newpos + 1;
-            if (pos == 0) {
+            newpos = gameStatstring.find(sep, pos);
+            if (newpos < pos && sep[0] == ',') {
+                sep[0] = '\r';
+                continue;
+            } else if (newpos < pos) {
                 break;
+            } else {
+                gameStats.push_back(gameStatstring.substr(pos, newpos - pos));
             }
+            pos = newpos + 1;
         }
         while (gameStats.size() < 10) {
             gameStats.push_back("");
@@ -1067,12 +1079,20 @@ public:
         ticks++;
         if ((ticks % 5000) == 0) {
             // C>S 0x07 PKT_KEEPALIVE
-            // ~5000 frames @ ~50fps == 100 seconds
+            // ~5000 frames @ ~50fps ~= 100 seconds
             BNCSOutputStream keepalive(0x07);
             keepalive.serialize32(ticks);
             keepalive.flush(ctx->getUDPSocket(), ctx->getHost());
         }
 
+        if ((ticks % 5000) == 0) {
+            ctx->refreshFriends();
+        }
+
+        if ((ticks % 100) == 0) {
+            ctx->refreshGames();
+        }
+
         if (ctx->getTCPSocket()->HasDataToRead(0)) {
             uint8_t msg = ctx->getMsgIStream()->readMessageId();
             if (msg == 0xff) {
@@ -1912,16 +1932,6 @@ static int CclGoOnline(lua_State* l) {
             _ctx.sendText(message);
         }
 
-        // refresh friends every 5 minutes
-        if ((FrameCounter % (FRAMES_PER_SECOND * 300)) == 0) {
-            _ctx.refreshFriends();
-        }
-
-        // refresh games every 30 seconds
-        if ((FrameCounter % (FRAMES_PER_SECOND * 30)) == 0) {
-            _ctx.refreshGames();
-        }
-
         if (AddGame) {
             for (auto g : _ctx.getGames()) {
                 AddGame->pushPreamble();

From 60b0c859bb28c1563fd63352452ea4b949f4eca3 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 8 Oct 2020 10:24:13 +0200
Subject: [PATCH 37/70] better debug dump

---
 src/network/online_service.cpp | 35 ++++++++++++++++++++++++++++++++--
 1 file changed, 33 insertions(+), 2 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 4bbd5af5c..fbf8c8501 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -141,6 +141,35 @@ public:
         return received_bytes;
     }
 
+    void dump() {
+        std::cout << "Raw contents >>>" << std::endl;
+        for (int i = 0; i < received_bytes; i += 8) {
+            std::cout << std::hex;
+            int j = i;
+            for (; j < received_bytes && j < (i + 8); j++) {
+                uint8_t byte = buffer[j];
+                if (byte < 0x10) {
+                    std::cout << "0";
+                }
+                std::cout << (unsigned short)byte << " ";
+            }
+            for (; j < (i + 9); j++) {
+                std::cout << "   "; // room for 2 hex digits and one space
+            }
+            j = i;
+            for (; j < received_bytes && j < (i + 8); j++) {
+                char c = buffer[j];
+                if (c >= 32) {
+                    std::cout.write(&c, 1);
+                } else {
+                    std::cout.write(".", 1);
+                }
+            }
+            std::cout << std::endl;
+        }
+        std::cout << "<<<" << std::endl;
+    }
+
     std::string string32() {
         // uint32 encoded (4-byte) string
         uint32_t data = read32();
@@ -734,7 +763,7 @@ public:
         BNCSOutputStream msg(0x09);
         msg.serializeC32("W2BN");
         msg.serialize32(0x4f);
-        msg.serialize(gameNameFromUsername(getUsername()).c_str());
+        msg.serialize(gameNameFromUsername(username).c_str());
         msg.serialize(pw.c_str());
         msg.flush(getTCPSocket());
     }
@@ -1138,6 +1167,7 @@ public:
                 int len = ctx->getMsgIStream()->readAll(&out);
                 std::cout.write(out, len);
                 std::cout << "<<<" << std::endl;
+                free(out);
             }
 
             ctx->getMsgIStream()->finishMessage();
@@ -1846,7 +1876,7 @@ OnlineContext *OnlineContextHandler = &_ctx;
  ** @param AddFriend: a callback to add an online friend. (name, product, status) -> nil
  ** @param AddChannel: a callback to add a channel. (str) -> nil
  ** @param ActiveChannel: a callback to report the currently active channel. (str) -> nil
- ** @param AddGame: a callback to add an advertised game. (map, creator, gametype, settings, max-players) -> nil
+ ** @param AddGame: a callback to add an advertised game. (map, creator, gametype, settings, max-players, ip:port) -> nil
  **
  ** @return status - "online" if successfully logged in, "pending" if waiting for connection, or error string
  */
@@ -1940,6 +1970,7 @@ static int CclGoOnline(lua_State* l) {
                 AddGame->pushString(g->getGameType());
                 AddGame->pushString(g->getGameSettings());
                 AddGame->pushInteger(g->maxPlayers());
+                AddGame->pushString(g->getHost().toString());
                 AddGame->run(0);
             }
         }

From 469af90622ab280a083430bfa8994c62bb0958ee Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 8 Oct 2020 10:24:20 +0200
Subject: [PATCH 38/70] fix reading the port for a game

---
 src/network/online_service.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index fbf8c8501..75719803f 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1194,7 +1194,9 @@ private:
             uint32_t settings = ctx->getMsgIStream()->read32();
             uint32_t lang = ctx->getMsgIStream()->read32();
             uint16_t addr_fam = ctx->getMsgIStream()->read16();
-            uint16_t port = ctx->getMsgIStream()->read16();
+            // the port is not in network byte order, since it's sent to
+            // directly go into a sockaddr_in struct
+            uint16_t port = (ctx->getMsgIStream()->read8() << 8) | ctx->getMsgIStream()->read8();
             uint32_t ip = ctx->getMsgIStream()->read32();
             uint32_t sinzero1 = ctx->getMsgIStream()->read32();
             uint32_t sinzero2 = ctx->getMsgIStream()->read32();

From eff1edb15811bfcc87b16b169998e31d914aa249 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 8 Oct 2020 10:28:53 +0200
Subject: [PATCH 39/70] allow disconnecting by passing empty host

---
 src/network/online_service.cpp | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 75719803f..3a1d54c20 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1867,7 +1867,7 @@ OnlineContext *OnlineContextHandler = &_ctx;
  **
  ** Lua arguments and return values:
  **
- ** @param host: hostname of the online service. Inititates a new connection.
+ ** @param host: hostname of the online service. Inititates a new connection if given, or disconnects if empty.
  ** @param post: port of the online service. Inititates a new connection.
  ** @param username: login username, only used if not logged in
  ** @param password: login password, only used if not logged in
@@ -1886,7 +1886,7 @@ static int CclGoOnline(lua_State* l) {
     std::string username = "";
     std::string password = "";
     std::string message = "";
-    std::string host = "";
+    const char* host = NULL;
     int port = 0;
 
     LuaCallback *AddMessage = NULL;
@@ -1911,7 +1911,7 @@ static int CclGoOnline(lua_State* l) {
         } else if (!strcmp(value, "password")) {
             password = std::string(LuaToString(l, -1));
         } else if (!strcmp(value, "host")) {
-            host = std::string(LuaToString(l, -1));
+            host = LuaToString(l, -1);
         } else if (!strcmp(value, "port")) {
             port = LuaToNumber(l, -1);
         } else if (!strcmp(value, "userInfo")) {
@@ -1939,10 +1939,14 @@ static int CclGoOnline(lua_State* l) {
         }
     }
 
-    if (!host.empty() && port) {
-        if (_ctx.isDisconnected()) {
-            _ctx.setHost(new CHost(host.c_str(), port));
-            _ctx.setState(new ConnectState());
+    if (host && port) {
+        if (strlen(host) == 0) {
+            _ctx.disconnect();
+        } else {
+            if (_ctx.isDisconnected()) {
+                _ctx.setHost(new CHost(host, port));
+                _ctx.setState(new ConnectState());
+            }
         }
     }
 

From f136ef9ffb0a8ae3a51043c9feb9c219da32db5e Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Sat, 10 Oct 2020 13:56:16 +0200
Subject: [PATCH 40/70] disconnect handling

---
 src/network/online_service.cpp | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 3a1d54c20..83feecde8 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -688,6 +688,10 @@ public:
         return lastError;
     }
 
+    void clearError() {
+        lastError = "";
+    }
+
     // User and UI actions
     void disconnect() {
         if (isConnected()) {
@@ -1284,7 +1288,7 @@ private:
             ctx->showInfo("[INFO]: " + text);
             break;
         case 0x13: // error message
-            ctx->showError("[SEVERE]: " + text);
+            ctx->showError(text);
             break;
         case 0x17: // emote
             break;
@@ -1941,7 +1945,9 @@ static int CclGoOnline(lua_State* l) {
 
     if (host && port) {
         if (strlen(host) == 0) {
+            lua_pushstring(l, "disconnected");
             _ctx.disconnect();
+            return 1;
         } else {
             if (_ctx.isDisconnected()) {
                 _ctx.setHost(new CHost(host, port));
@@ -1956,6 +1962,7 @@ static int CclGoOnline(lua_State* l) {
 
     if (!error.empty()) {
         lua_pushstring(l, error.c_str());
+        _ctx.clearError();
     } else if (_ctx.isConnecting()) {
         lua_pushstring(l, "pending");
     } else if (!username.empty() && !password.empty()) {
@@ -2022,7 +2029,7 @@ static int CclGoOnline(lua_State* l) {
             _ctx.requestExtraUserInfo(userInfo);
         }
     } else {
-        lua_pushstring(l, "unknown");
+        lua_pushnil(l);
     }
 
     return 1;

From 3915ba622a6d3b1eb1094227b2385c11c4e87cb3 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <tim.felgentreff@oracle.com>
Date: Sat, 14 Nov 2020 10:53:40 +0100
Subject: [PATCH 41/70] make OnlineContext a global lua value with functions

---
 src/network/online_service.cpp | 471 +++++++++++++++++++--------------
 1 file changed, 275 insertions(+), 196 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 83feecde8..e052a05f6 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -684,14 +684,6 @@ public:
         return state == NULL;
     }
 
-    std::string getLastError() {
-        return lastError;
-    }
-
-    void clearError() {
-        lastError = "";
-    }
-
     // User and UI actions
     void disconnect() {
         if (isConnected()) {
@@ -935,6 +927,11 @@ public:
     // UI information
     void setCurrentChannel(std::string name) {
         this->currentChannel = name;
+        if (SetActiveChannel != NULL) {
+          SetActiveChannel->pushPreamble();
+          SetActiveChannel->pushString(name);
+          SetActiveChannel->run();
+        }
     }
 
     std::string getCurrentChannel() { return currentChannel; }
@@ -944,6 +941,22 @@ public:
             delete value;
         }
         this->games = games;
+        if (SetGames != NULL) {
+          SetGames->pushPreamble();
+          for (const auto value : games) {
+            SetGames->pushTable({{"Creator", value->getCreator()},
+                                 {"Host", value->getHost().toString()},
+                                 {"IsSavedGame", value->isSavedGame()},
+                                 {"Map", value->getMap()},
+                                 {"MaxPlayers", value->maxPlayers()},
+                                 {"Speed", value->getSpeed()},
+                                 {"Approval", value->getApproval()},
+                                 {"Settings", value->getGameSettings()},
+                                 {"Status", value->getGameStatus()},
+                                 {"Type", value->getGameType()}});
+          }
+          SetGames->run();
+        }
     }
 
     std::vector<Game*> getGames() { return games; }
@@ -953,41 +966,91 @@ public:
             delete value;
         }
         this->friends = friends;
+        if (SetFriends != NULL) {            
+            SetFriends->pushPreamble();
+            for (const auto value : friends) {
+                SetFriends->pushTable({ { "Name", value->getName() },
+                                        { "Status", value->getStatus() },
+                                        { "Product", value->getProduct() } });
+            }
+            SetFriends->run();
+        }
     }
 
     std::vector<Friend*> getFriends() { return friends; }
 
     void reportUserdata(uint32_t id, std::vector<std::string> values) {
-        this->extendedInfoValues[id] = values;
-        showInfo("User Info for " + extendedInfoNames.at(id));
-        for (int i = 0; i < values.size(); i++) {
-            showInfo(defaultUserKeys.at(i) + ": " + values.at(i));
+        if (ShowUserInfo != NULL) {
+            ShowUserInfo->pushPreamble();
+            std::map<std::string, std::variant<std::string, int>> m;
+            m["User"] = extendedInfoNames.at(id);            
+            for (int i = 0; i < values.size(); i++) {
+                m[defaultUserKeys.at(i)] = values.at(i);
+            }
+            ShowUserInfo->pushTable(m);
+            ShowUserInfo->run();
         }
     }
 
     std::queue<std::string> *getInfo() { return &info; }
 
-    void showInfo(std::string arg) { info.push("*** " + arg + " ***"); }
+    void showInfo(std::string arg) {
+        std::string infoStr = "*** " + arg + " ***";
+        info.push(infoStr);
+        if (ShowInfo != NULL) {
+            ShowInfo->pushPreamble();
+            ShowInfo->pushString(infoStr);
+            ShowInfo->run();
+        }
+    }
 
     void showError(std::string arg) {
         info.push("!!! " + arg + " !!!");
-        // lastError = arg;
+        if (ShowError != NULL) {
+            ShowError->pushPreamble();
+            ShowError->pushString(arg);
+            ShowError->run();
+        }
     }
 
-    void showChat(std::string arg) { info.push(arg); }
+    void showChat(std::string arg) {
+        info.push(arg);
+        if (ShowChat != NULL) {
+            ShowChat->pushPreamble();
+            ShowChat->pushString(arg);
+            ShowChat->run();
+        }
+    }
 
     void addUser(std::string name) {
         userList.insert(name);
+        if (AddUser != NULL) {
+            AddUser->pushPreamble();
+            AddUser->pushString(name);
+            AddUser->run();
+        }
     }
 
     void removeUser(std::string name) {
         userList.erase(name);
+        if (RemoveUser != NULL) {
+            RemoveUser->pushPreamble();
+            RemoveUser->pushString(name);
+            RemoveUser->run();
+        }
     }
 
     std::set<std::string> getUsers() { return userList; }
 
     void setChannels(std::vector<std::string> channels) {
         this->channelList = channels;
+        if (SetChannels != NULL) {            
+            SetChannels->pushPreamble();
+            for (const auto value : channels) {
+                SetChannels->pushString(value);
+            }
+            SetChannels->run();
+        }
     }
 
     std::vector<std::string> getChannels() { return channelList; }
@@ -1048,6 +1111,17 @@ public:
     uint32_t clientToken;
     uint32_t serverToken;
 
+    LuaCallback *AddUser = NULL;
+    LuaCallback *RemoveUser = NULL;
+    LuaCallback *SetFriends = NULL;
+    LuaCallback *SetGames = NULL;
+    LuaCallback *SetChannels = NULL;
+    LuaCallback *SetActiveChannel = NULL;
+    LuaCallback *ShowError = NULL;
+    LuaCallback *ShowInfo = NULL;
+    LuaCallback *ShowChat = NULL;
+    LuaCallback *ShowUserInfo = NULL;
+
 private:
     std::string gameNameFromUsername(std::string username) {
         return username + "'s game";
@@ -1072,7 +1146,6 @@ private:
     std::vector<Game*> games;
     std::vector<Friend*> friends;
     std::vector<std::string> extendedInfoNames;
-    std::map<uint32_t, std::vector<std::string>> extendedInfoValues;
     std::vector<std::string> defaultUserKeys;
 };
 
@@ -1859,182 +1932,6 @@ class ConnectState : public NetworkState {
 static Context _ctx;
 OnlineContext *OnlineContextHandler = &_ctx;
 
-/**
- ** The assumption is that the lists of users, friends, and games are cleared by
- ** the caller before calling this.
- **
- ** The call order should be this:
- **  1) call once with username + password
- **  2) call without args until you get a result that is not "pending"
- **  2a) if result is "online", call repeatedly with callbacks or messages
- **  2b) if result is an error, you may go to 1 after fixing the error
- **
- ** Lua arguments and return values:
- **
- ** @param host: hostname of the online service. Inititates a new connection if given, or disconnects if empty.
- ** @param post: port of the online service. Inititates a new connection.
- ** @param username: login username, only used if not logged in
- ** @param password: login password, only used if not logged in
- ** @param message: a message to send, only used if logged in
- ** @param userInfo: a username to request info for
- ** @param AddMessage: a callback to add new messages. (str) -> nil
- ** @param AddUser: a callback to add an online username. (str) -> nil
- ** @param AddFriend: a callback to add an online friend. (name, product, status) -> nil
- ** @param AddChannel: a callback to add a channel. (str) -> nil
- ** @param ActiveChannel: a callback to report the currently active channel. (str) -> nil
- ** @param AddGame: a callback to add an advertised game. (map, creator, gametype, settings, max-players, ip:port) -> nil
- **
- ** @return status - "online" if successfully logged in, "pending" if waiting for connection, or error string
- */
-static int CclGoOnline(lua_State* l) {
-    std::string username = "";
-    std::string password = "";
-    std::string message = "";
-    const char* host = NULL;
-    int port = 0;
-
-    LuaCallback *AddMessage = NULL;
-    LuaCallback *AddUser = NULL;
-    LuaCallback *AddFriend = NULL;
-    LuaCallback *AddGame = NULL;
-    LuaCallback *AddChannel = NULL;
-    LuaCallback *ActiveChannel = NULL;
-    std::string newActiveChannel = "";
-    std::string userInfo = "";
-
-    LuaCheckArgs(l, 1);
-    if (!lua_istable(l, 1)) {
-        LuaError(l, "incorrect argument");
-    }
-
-    for (lua_pushnil(l); lua_next(l, 1); lua_pop(l, 1)) {
-        const char *value = LuaToString(l, -2);
-
-        if (!strcmp(value, "username")) {
-            username = std::string(LuaToString(l, -1));
-        } else if (!strcmp(value, "password")) {
-            password = std::string(LuaToString(l, -1));
-        } else if (!strcmp(value, "host")) {
-            host = LuaToString(l, -1);
-        } else if (!strcmp(value, "port")) {
-            port = LuaToNumber(l, -1);
-        } else if (!strcmp(value, "userInfo")) {
-            userInfo = std::string(LuaToString(l, -1));
-        } else if (!strcmp(value, "message")) {
-            message = std::string(LuaToString(l, -1));
-        } else if (!strcmp(value, "AddMessage")) {
-            AddMessage = new LuaCallback(l, -1);
-        } else if (!strcmp(value, "AddUser")) {
-            AddUser = new LuaCallback(l, -1);
-        } else if (!strcmp(value, "AddFriend")) {
-            AddFriend = new LuaCallback(l, -1);
-        } else if (!strcmp(value, "AddGame")) {
-            AddGame = new LuaCallback(l, -1);
-        } else if (!strcmp(value, "AddChannel")) {
-            AddChannel = new LuaCallback(l, -1);
-        } else if (!strcmp(value, "ActiveChannel")) {
-            if (lua_isstring(l, -1)) {
-                newActiveChannel = LuaToString(l, -1);
-            } else {
-                ActiveChannel = new LuaCallback(l, -1);
-            }
-        } else {
-            LuaError(l, "Unsupported tag: %s" _C_ value);
-        }
-    }
-
-    if (host && port) {
-        if (strlen(host) == 0) {
-            lua_pushstring(l, "disconnected");
-            _ctx.disconnect();
-            return 1;
-        } else {
-            if (_ctx.isDisconnected()) {
-                _ctx.setHost(new CHost(host, port));
-                _ctx.setState(new ConnectState());
-            }
-        }
-    }
-
-    _ctx.doOneStep();
-
-    std::string error = _ctx.getLastError();
-
-    if (!error.empty()) {
-        lua_pushstring(l, error.c_str());
-        _ctx.clearError();
-    } else if (_ctx.isConnecting()) {
-        lua_pushstring(l, "pending");
-    } else if (!username.empty() && !password.empty()) {
-        lua_pushstring(l, "pending");
-        _ctx.setUsername(username);
-        _ctx.setPassword(password);
-    } else if (_ctx.isConnected()) {
-        lua_pushstring(l, "online");
-        if (!message.empty() && !_ctx.getCurrentChannel().empty()) {
-            _ctx.sendText(message);
-        }
-
-        if (AddGame) {
-            for (auto g : _ctx.getGames()) {
-                AddGame->pushPreamble();
-                AddGame->pushString(g->getMap());
-                AddGame->pushString(g->getCreator());
-                AddGame->pushString(g->getGameType());
-                AddGame->pushString(g->getGameSettings());
-                AddGame->pushInteger(g->maxPlayers());
-                AddGame->pushString(g->getHost().toString());
-                AddGame->run(0);
-            }
-        }
-        if (AddUser) {
-            for (auto u : _ctx.getUsers()) {
-                AddUser->pushPreamble();
-                AddUser->pushString(u);
-                AddUser->run(0);
-            }
-        }
-        if (AddChannel) {
-            for (auto u : _ctx.getChannels()) {
-                AddChannel->pushPreamble();
-                AddChannel->pushString(u);
-                AddChannel->run(0);
-            }
-        }
-        if (ActiveChannel) {
-            ActiveChannel->pushPreamble();
-            ActiveChannel->pushString(_ctx.getCurrentChannel());
-            ActiveChannel->run(0);
-        } else if (!newActiveChannel.empty()) {
-            _ctx.joinChannel(newActiveChannel);
-        }
-        if (AddFriend) {
-            for (auto u : _ctx.getFriends()) {
-                AddFriend->pushPreamble();
-                AddFriend->pushString(u->getName());
-                AddFriend->pushString(u->getProduct());
-                AddFriend->pushString(u->getStatus());
-                AddFriend->run(0);
-            }
-        }
-        if (AddMessage) {
-            while (!_ctx.getInfo()->empty()) {
-                AddMessage->pushPreamble();
-                AddMessage->pushString(_ctx.getInfo()->front());
-                AddMessage->run(0);
-                _ctx.getInfo()->pop();
-            }
-        }
-        if (!userInfo.empty()) {
-            _ctx.requestExtraUserInfo(userInfo);
-        }
-    } else {
-        lua_pushnil(l);
-    }
-
-    return 1;
-}
-
 static int CclStopAdvertisingOnlineGame(lua_State* l) {
     if (_ctx.isConnected()) {
         _ctx.stopAdvertising();
@@ -2055,8 +1952,190 @@ static int CclStartAdvertisingOnlineGame(lua_State* l) {
     return 1;
 }
 
-void OnlineServiceCclRegister() {
-    lua_register(Lua, "StartAdvertisingOnlineGame", CclStartAdvertisingOnlineGame);
-    lua_register(Lua, "StopAdvertisingOnlineGame", CclStopAdvertisingOnlineGame);
-    lua_register(Lua, "GoOnline", CclGoOnline);
+static int CclSetup(lua_State *l) {
+    LuaCheckArgs(l, 1);
+    if (!lua_istable(l, 1)) {
+        LuaError(l, "incorrect argument");
+    }
+
+    for (lua_pushnil(l); lua_next(l, 1); lua_pop(l, 1)) {
+        const char *value = LuaToString(l, -2);
+        if (!strcmp(value, "AddUser")) {
+            if (_ctx.AddUser) {
+                delete _ctx.AddUser;
+            }
+            _ctx.AddUser = new LuaCallback(l, -1);
+        } else if (!strcmp(value, "RemoveUser")) {
+            if (_ctx.RemoveUser) {
+                delete _ctx.RemoveUser;
+            }
+            _ctx.RemoveUser = new LuaCallback(l, -1);
+        } else if (!strcmp(value, "SetFriends")) {
+            if (_ctx.SetFriends) {
+                delete _ctx.SetFriends;
+            }
+            _ctx.SetFriends = new LuaCallback(l, -1);
+        } else if (!strcmp(value, "SetGames")) {
+          if (_ctx.SetGames) {
+            delete _ctx.SetGames;
+          }
+          _ctx.SetGames = new LuaCallback(l, -1);
+        } else if (!strcmp(value, "SetChannels")) {
+            if (_ctx.SetChannels) {
+                delete _ctx.SetChannels;
+            }
+            _ctx.SetChannels = new LuaCallback(l, -1);
+        } else if (!strcmp(value, "SetActiveChannel")) {
+            if (_ctx.SetActiveChannel) {
+                delete _ctx.SetActiveChannel;
+            }
+            _ctx.SetActiveChannel = new LuaCallback(l, -1);
+        } else if (!strcmp(value, "ShowChat")) {
+            if (_ctx.ShowChat) {
+                delete _ctx.ShowChat;
+            }
+            _ctx.ShowChat = new LuaCallback(l, -1);
+        } else if (!strcmp(value, "ShowInfo")) {
+            if (_ctx.ShowInfo) {
+                delete _ctx.ShowInfo;
+            }
+            _ctx.ShowInfo = new LuaCallback(l, -1);
+        } else if (!strcmp(value, "ShowError")) {
+            if (_ctx.ShowError) {
+                delete _ctx.ShowError;
+            }
+            _ctx.ShowError = new LuaCallback(l, -1);
+        } else if (!strcmp(value, "ShowUserInfo")) {
+            if (_ctx.ShowUserInfo) {
+                delete _ctx.ShowUserInfo;
+            }
+            _ctx.ShowUserInfo = new LuaCallback(l, -1);
+        } else {
+            LuaError(l, "Unsupported callback: %s" _C_ value);
+        }
+    }
+    return 0;
+}
+
+static int CclDisconnect(lua_State *l) {
+    LuaCheckArgs(l, 0);
+    _ctx.disconnect();
+    return 0;
+}
+
+static int CclConnect(lua_State *l) {
+    _ctx.disconnect();
+    LuaCheckArgs(l, 2);
+    const char *host = LuaToString(l, 1);
+    int port = LuaToNumber(l, 2);
+
+    _ctx.setHost(new CHost(host, port));
+    _ctx.setState(new ConnectState());
+    return 0;
+}
+
+static int CclLogin(lua_State *l) {
+    LuaCheckArgs(l, 2);
+    _ctx.setUsername(LuaToString(l, 1));
+    _ctx.setPassword(LuaToString(l, 2));
+    return 0;
+}
+
+static int CclStep(lua_State *l) {
+    LuaCheckArgs(l, 0);
+    if (_ctx.isDisconnected()) {
+        LuaError(l, "disconnected service cannot step");
+    } else {
+        _ctx.doOneStep();
+        if (_ctx.isConnecting()) {
+            lua_pushstring(l, "connecting");
+        } else if (_ctx.isConnected()) {
+            lua_pushstring(l, "online");
+        } else {
+            LuaError(l, "unknown online state");
+        }
+    }
+    return 1;
+}
+
+static int CclSendMessage(lua_State *l) {
+    LuaCheckArgs(l, 1);
+    if (!_ctx.isConnected()) {
+        LuaError(l, "connect first");
+    }
+    _ctx.sendText(LuaToString(l, 1));
+    return 0;
+}
+
+static int CclJoinChannel(lua_State *l) {
+    LuaCheckArgs(l, 1);
+    if (!_ctx.isConnected()) {
+        LuaError(l, "connect first");
+    }
+    _ctx.joinChannel(LuaToString(l, 1));
+    return 0;
+}
+
+static int CclJoinGame(lua_State *l) {
+    LuaCheckArgs(l, 2);
+    if (!_ctx.isConnected()) {
+        LuaError(l, "connect first");
+    }
+    _ctx.joinGame(LuaToString(l, 1), LuaToString(l, 2));
+    return 0;
+}
+
+static int CclStatus(lua_State *l) {
+    LuaCheckArgs(l, 0);
+    if (_ctx.isConnected()) {
+        lua_pushstring(l, "connected");
+    } else if (_ctx.isConnecting()) {
+        lua_pushstring(l, "connecting");
+    } else if (_ctx.isDisconnected()) {
+        lua_pushstring(l, "disconnected");
+    }
+    return 1;
+}
+
+static int CclRequestUserInfo(lua_State *l) {
+    LuaCheckArgs(l, 1);
+    if (!_ctx.isConnected()) {
+        LuaError(l, "not connected");
+    }
+    _ctx.requestExtraUserInfo(LuaToString(l, 1));
+    return 0;
+}
+
+void OnlineServiceCclRegister() {
+    lua_createtable(Lua, 0, 3);
+
+    lua_pushcfunction(Lua, CclSetup);
+    lua_setfield(Lua, -2, "setupcallbacks");
+
+    lua_pushcfunction(Lua, CclConnect);
+    lua_setfield(Lua, -2, "connect");
+    lua_pushcfunction(Lua, CclDisconnect);
+    lua_setfield(Lua, -2, "disconnect");
+
+    lua_pushcfunction(Lua, CclStatus);
+    lua_setfield(Lua, -2, "status");
+
+    // step is called in the SDL event loop, I don't think we need to expose it
+    // lua_pushcfunction(Lua, CclStep);
+    // lua_setfield(Lua, -2, "step");
+    lua_pushcfunction(Lua, CclSendMessage);
+    lua_setfield(Lua, -2, "sendmessage");
+    lua_pushcfunction(Lua, CclJoinChannel);
+    lua_setfield(Lua, -2, "joinchannel");
+    lua_pushcfunction(Lua, CclRequestUserInfo);
+    lua_setfield(Lua, -2, "requestuserinfo");
+    // join game is called from the netconnect code, I don't think we need to expose it
+    // lua_pushcfunction(Lua, CclJoinGame);
+    // lua_setfield(Lua, -2, "joingame");
+    lua_pushcfunction(Lua, CclStartAdvertisingOnlineGame);
+    lua_setfield(Lua, -2, "startadvertising");
+    lua_pushcfunction(Lua, CclStopAdvertisingOnlineGame);
+    lua_setfield(Lua, -2, "stopadvertising");
+
+    lua_setglobal(Lua, "OnlineService");
 }

From e6887ef0cf7019e088233772ff698b34aeb8a6e0 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Sat, 21 Nov 2020 10:53:11 +0100
Subject: [PATCH 42/70] include our script.h instead of lua.h directly

---
 src/network/online_service.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index e052a05f6..2098f3d9c 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -33,7 +33,7 @@
 #endif
 
 #include "stratagus.h"
-#include "lua.h"
+#include "script.h"
 #include "map.h"
 #include "netconnect.h"
 #include "script.h"

From 750b96c573a06140a8e48359bca8b1f8b286f45c Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Mon, 23 Nov 2020 11:08:48 +0100
Subject: [PATCH 43/70] add login lua function, immediately execute stateless
 callbacks when they are set

---
 src/network/online_service.cpp | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 2098f3d9c..08db5ff52 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1975,6 +1975,9 @@ static int CclSetup(lua_State *l) {
                 delete _ctx.SetFriends;
             }
             _ctx.SetFriends = new LuaCallback(l, -1);
+            if (_ctx.isConnected()) {
+                _ctx.setFriendslist(_ctx.getFriends());
+            }
         } else if (!strcmp(value, "SetGames")) {
           if (_ctx.SetGames) {
             delete _ctx.SetGames;
@@ -1984,12 +1987,18 @@ static int CclSetup(lua_State *l) {
             if (_ctx.SetChannels) {
                 delete _ctx.SetChannels;
             }
+            if (_ctx.isConnected()) {
+                _ctx.setChannels(_ctx.getChannels());
+            }
             _ctx.SetChannels = new LuaCallback(l, -1);
         } else if (!strcmp(value, "SetActiveChannel")) {
             if (_ctx.SetActiveChannel) {
                 delete _ctx.SetActiveChannel;
             }
             _ctx.SetActiveChannel = new LuaCallback(l, -1);
+            if (_ctx.isConnected()) {
+                _ctx.setCurrentChannel(_ctx.getCurrentChannel());
+            }
         } else if (!strcmp(value, "ShowChat")) {
             if (_ctx.ShowChat) {
                 delete _ctx.ShowChat;
@@ -2110,10 +2119,12 @@ void OnlineServiceCclRegister() {
     lua_createtable(Lua, 0, 3);
 
     lua_pushcfunction(Lua, CclSetup);
-    lua_setfield(Lua, -2, "setupcallbacks");
+    lua_setfield(Lua, -2, "setup");
 
     lua_pushcfunction(Lua, CclConnect);
     lua_setfield(Lua, -2, "connect");
+    lua_pushcfunction(Lua, CclLogin);
+    lua_setfield(Lua, -2, "login");
     lua_pushcfunction(Lua, CclDisconnect);
     lua_setfield(Lua, -2, "disconnect");
 

From 2a1f8c644026cc32b1a84d1bd185c621cf8a2b6f Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Mon, 23 Nov 2020 11:15:43 +0100
Subject: [PATCH 44/70] return the last info on unknown state

---
 src/network/online_service.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 08db5ff52..558769e6c 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -2102,6 +2102,12 @@ static int CclStatus(lua_State *l) {
         lua_pushstring(l, "connecting");
     } else if (_ctx.isDisconnected()) {
         lua_pushstring(l, "disconnected");
+    } else {
+        if (!_ctx.getInfo()->empty()) {
+            lua_pushstring(l, _ctx.getInfo()->back().c_str());
+        } else {
+            lua_pushstring(l, "unknown error");
+        }
     }
     return 1;
 }

From c6982ac07a2741a0c5ee99f46e543b8bc714a3b9 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Mon, 23 Nov 2020 11:22:59 +0100
Subject: [PATCH 45/70] do not modify info string

---
 src/network/online_service.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 558769e6c..8ccfa54e7 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -995,7 +995,7 @@ public:
     std::queue<std::string> *getInfo() { return &info; }
 
     void showInfo(std::string arg) {
-        std::string infoStr = "*** " + arg + " ***";
+        std::string infoStr = arg;
         info.push(infoStr);
         if (ShowInfo != NULL) {
             ShowInfo->pushPreamble();

From eeb89e9729a2738958502986ea4e8f81c42bacba Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Mon, 23 Nov 2020 15:19:25 +0100
Subject: [PATCH 46/70] do not create account automatically, expose lua
 function to do it

---
 src/network/online_service.cpp | 35 ++++++++++++++++++++++++++++++----
 1 file changed, 31 insertions(+), 4 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 8ccfa54e7..cdb512cc4 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1082,6 +1082,14 @@ public:
         }
     }
 
+    void setCreateAccount(bool flag) {
+        createAccount = flag;
+    }
+
+    bool shouldCreateAccount() {
+        return createAccount;
+    }
+
     // Protocol
     CHost *getHost() { return host; }
 
@@ -1136,6 +1144,7 @@ private:
     std::string username;
     uint32_t password[5]; // xsha1 hash of password
     bool hasPassword;
+    bool createAccount;
 
     std::string lastError;
 
@@ -1597,18 +1606,25 @@ class S2C_LOGONRESPONSE2 : public NetworkState {
                 return;
             case 0x01:
             case 0x010000:
-                ctx->showInfo("Account does not exist, creating it...");
-                createAccount(ctx);
+                if (ctx->shouldCreateAccount()) {
+                    ctx->showInfo("Account does not exist, creating it...");
+                    createAccount(ctx);
+                } else {
+                    ctx->showError("Account does not exist");
+                    ctx->setUsername("");
+                    ctx->setPassword("");
+                    ctx->setState(new C2S_LOGONRESPONSE2());
+                }
                 return;
             case 0x02:
             case 0x020000:
-                ctx->showInfo("Incorrect password");
+                ctx->showError("Incorrect password");
                 ctx->setPassword("");
                 ctx->setState(new C2S_LOGONRESPONSE2());
                 return;
             case 0x06:
             case 0x060000:
-                ctx->showInfo("Account closed: " + ctx->getMsgIStream()->readString());
+                ctx->showError("Account closed: " + ctx->getMsgIStream()->readString());
                 ctx->setPassword("");
                 ctx->setState(new C2S_LOGONRESPONSE2());
                 return;
@@ -2045,6 +2061,15 @@ static int CclConnect(lua_State *l) {
 
 static int CclLogin(lua_State *l) {
     LuaCheckArgs(l, 2);
+    _ctx.setCreateAccount(false);
+    _ctx.setUsername(LuaToString(l, 1));
+    _ctx.setPassword(LuaToString(l, 2));
+    return 0;
+}
+
+static int CclSignUp(lua_State *l) {
+    LuaCheckArgs(l, 2);
+    _ctx.setCreateAccount(true);
     _ctx.setUsername(LuaToString(l, 1));
     _ctx.setPassword(LuaToString(l, 2));
     return 0;
@@ -2131,6 +2156,8 @@ void OnlineServiceCclRegister() {
     lua_setfield(Lua, -2, "connect");
     lua_pushcfunction(Lua, CclLogin);
     lua_setfield(Lua, -2, "login");
+    lua_pushcfunction(Lua, CclSignUp);
+    lua_setfield(Lua, -2, "signup");
     lua_pushcfunction(Lua, CclDisconnect);
     lua_setfield(Lua, -2, "disconnect");
 

From 8f9d2af3033a2822e13d65946ad2415df05e7214 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Tue, 24 Nov 2020 08:46:13 +0100
Subject: [PATCH 47/70] minor tweaks for channel refreshes, info text color,
 disconnecting

---
 src/network/online_service.cpp | 153 +++++++++++++++++----------------
 1 file changed, 81 insertions(+), 72 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index cdb512cc4..0e4c61e1c 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -655,6 +655,8 @@ public:
         this->clientToken = MyRand();
         this->username = "";
         setPassword("");
+        defaultUserKeys.push_back("profile\\location");
+        defaultUserKeys.push_back("profile\\description");
         defaultUserKeys.push_back("record\\GAME\\0\\wins");
         defaultUserKeys.push_back("record\\GAME\\0\\losses");
         defaultUserKeys.push_back("record\\GAME\\0\\disconnects");
@@ -731,6 +733,12 @@ public:
         msg.flush(getTCPSocket());
     }
 
+    void refreshChannels() {
+        BNCSOutputStream getlist(0x0b);
+        getlist.serialize32(0x00);
+        getlist.flush(getTCPSocket());
+    }
+
     void refreshGames() {
         // C>S 0x09 SID_GETADVLISTEX
         BNCSOutputStream getadvlistex(0x09);
@@ -983,7 +991,7 @@ public:
         if (ShowUserInfo != NULL) {
             ShowUserInfo->pushPreamble();
             std::map<std::string, std::variant<std::string, int>> m;
-            m["User"] = extendedInfoNames.at(id);            
+            m["User"] = extendedInfoNames.at(id);
             for (int i = 0; i < values.size(); i++) {
                 m[defaultUserKeys.at(i)] = values.at(i);
             }
@@ -1166,17 +1174,12 @@ class DisconnectedState : public NetworkState {
 public:
     DisconnectedState(std::string message) {
         this->message = message;
-        this->hasPrinted = false;
     };
 
     virtual void doOneStep(Context *ctx) {
-        if (!hasPrinted) {
-            std::cout << message << std::endl;
-            ctx->showError(message);
-            hasPrinted = true;
-            ctx->disconnect();
-        }
-        // the end
+        std::cout << message << std::endl;
+        ctx->disconnect();
+        ctx->showError(message);
     }
 
 private:
@@ -1202,9 +1205,10 @@ public:
 
         if ((ticks % 5000) == 0) {
             ctx->refreshFriends();
+            ctx->refreshChannels();
         }
 
-        if ((ticks % 100) == 0) {
+        if ((ticks % 300) == 0) {
             ctx->refreshGames();
         }
 
@@ -1367,7 +1371,7 @@ private:
             ctx->showInfo("Channel restricted");
             break;
         case 0x12: // general info text
-            ctx->showInfo("[INFO]: " + text);
+            ctx->showInfo("~<" + text + "~>");
             break;
         case 0x13: // error message
             ctx->showError(text);
@@ -1451,9 +1455,7 @@ class C2S_ENTERCHAT : public NetworkState {
         enterchat.serialize("");
         enterchat.flush(ctx->getTCPSocket());
 
-        BNCSOutputStream getlist(0x0b);
-        getlist.serialize32(0x00);
-        getlist.flush(ctx->getTCPSocket());
+        ctx->refreshChannels();
 
         BNCSOutputStream join(0x0c);
         join.serialize32(0x01); // first-join
@@ -1500,8 +1502,8 @@ private:
     int retries;
 };
 
-class C2S_LOGONRESPONSE2 : public NetworkState {
-    virtual void doOneStep(Context *ctx);
+class C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT : public NetworkState {
+  virtual void doOneStep(Context *ctx);
 };
 
 class S2C_CREATEACCOUNT2 : public NetworkState {
@@ -1529,52 +1531,52 @@ class S2C_CREATEACCOUNT2 : public NetworkState {
             switch (status) {
             case 0x00:
                 // login into created account
-                ctx->setState(new C2S_LOGONRESPONSE2());
+                ctx->setState(new C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT());
                 return;
             case 0x01:
                 ctx->showError("Name too short" + nameSugg);
                 ctx->setUsername("");
-                ctx->setState(new C2S_LOGONRESPONSE2());
+                ctx->setState(new C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT());
                 return;
             case 0x02:
                 ctx->showError("Name contains invalid character(s)" + nameSugg);
                 ctx->setUsername("");
-                ctx->setState(new C2S_LOGONRESPONSE2());
+                ctx->setState(new C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT());
                 return;
             case 0x03:
                 ctx->showError("Name contains banned word(s)" + nameSugg);
                 ctx->setUsername("");
-                ctx->setState(new C2S_LOGONRESPONSE2());
+                ctx->setState(new C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT());
                 return;
             case 0x04:
                 ctx->showError("Account already exists" + nameSugg);
                 ctx->setUsername("");
-                ctx->setState(new C2S_LOGONRESPONSE2());
+                ctx->setState(new C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT());
                 return;
             case 0x05:
                 ctx->showError("Account is still being created" + nameSugg);
                 ctx->setUsername("");
-                ctx->setState(new C2S_LOGONRESPONSE2());
+                ctx->setState(new C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT());
                 return;
             case 0x06:
                 ctx->showError("Name does not contain enough alphanumeric characters" + nameSugg);
                 ctx->setUsername("");
-                ctx->setState(new C2S_LOGONRESPONSE2());
+                ctx->setState(new C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT());
                 return;
             case 0x07:
                 ctx->showError("Name contained adjacent punctuation characters" + nameSugg);
                 ctx->setUsername("");
-                ctx->setState(new C2S_LOGONRESPONSE2());
+                ctx->setState(new C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT());
                 return;
             case 0x08:
                 ctx->showError("Name contained too many punctuation characters" + nameSugg);
                 ctx->setUsername("");
-                ctx->setState(new C2S_LOGONRESPONSE2());
+                ctx->setState(new C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT());
                 return;
             default:
                 ctx->showError("Unknown error creating account" + nameSugg);
                 ctx->setUsername("");
-                ctx->setState(new C2S_LOGONRESPONSE2());
+                ctx->setState(new C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT());
                 return;
             }
 
@@ -1606,27 +1608,22 @@ class S2C_LOGONRESPONSE2 : public NetworkState {
                 return;
             case 0x01:
             case 0x010000:
-                if (ctx->shouldCreateAccount()) {
-                    ctx->showInfo("Account does not exist, creating it...");
-                    createAccount(ctx);
-                } else {
-                    ctx->showError("Account does not exist");
-                    ctx->setUsername("");
-                    ctx->setPassword("");
-                    ctx->setState(new C2S_LOGONRESPONSE2());
-                }
+                ctx->showError("Account does not exist");
+                ctx->setUsername("");
+                ctx->setPassword("");
+                ctx->setState(new C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT());
                 return;
             case 0x02:
             case 0x020000:
                 ctx->showError("Incorrect password");
                 ctx->setPassword("");
-                ctx->setState(new C2S_LOGONRESPONSE2());
+                ctx->setState(new C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT());
                 return;
             case 0x06:
             case 0x060000:
                 ctx->showError("Account closed: " + ctx->getMsgIStream()->readString());
                 ctx->setPassword("");
-                ctx->setState(new C2S_LOGONRESPONSE2());
+                ctx->setState(new C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT());
                 return;
             default:
                 ctx->setState(new DisconnectedState("unknown logon response"));
@@ -1647,41 +1644,53 @@ private:
     }
 };
 
-void C2S_LOGONRESPONSE2::doOneStep(Context *ctx) {
-    std::string user = ctx->getUsername();
-    uint32_t *pw = ctx->getPassword1(); // single-hashed for SID_LOGONRESPONSE2
-    if (!user.empty() && pw) {
-        BNCSOutputStream logon(0x3a);
-        logon.serialize32(ctx->clientToken);
-        logon.serialize32(ctx->serverToken);
-
-        // Battle.net password hashes are hashed twice using XSHA-1. First, the
-        // password is hashed by itself, then the following data is hashed again
-        // and sent to Battle.net:
-        // (UINT32) Client Token
-        // (UINT32) Server Token
-        // (UINT32)[5] First password hash
-        // The logic below is taken straight from pvpgn
-        struct {
-            pvpgn::bn_int ticks;
-            pvpgn::bn_int sessionkey;
-            pvpgn::bn_int passhash1[5];
-        } temp;
-        uint32_t passhash2[5];
-
-        pvpgn::bn_int_set(&temp.ticks, ntohl(ctx->clientToken));
-        pvpgn::bn_int_set(&temp.sessionkey, ntohl(ctx->serverToken));
-        pvpgn::hash_to_bnhash((pvpgn::t_hash const *)pw, temp.passhash1);
-        pvpgn::bnet_hash(&passhash2, sizeof(temp), &temp);	/* do the double hash */
-
-        for (int i = 0; i < 20; i++) {
-            logon.serialize8(reinterpret_cast<uint8_t*>(passhash2)[i]);
-        }
-        logon.serialize(user.c_str());
-        logon.flush(ctx->getTCPSocket());
-
-        ctx->setState(new S2C_LOGONRESPONSE2());
+void C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT::doOneStep(Context *ctx) {
+  std::string user = ctx->getUsername();
+  uint32_t *pw = ctx->getPassword1(); // single-hashed for SID_LOGONRESPONSE2
+  if (!user.empty() && pw) {
+    if (ctx->shouldCreateAccount()) {
+      BNCSOutputStream msg(0x3d);
+      uint32_t *pw = ctx->getPassword1();
+      for (int i = 0; i < 20; i++) {
+        msg.serialize8(reinterpret_cast<uint8_t *>(pw)[i]);
+      }
+      msg.serialize(ctx->getUsername().c_str());
+      msg.flush(ctx->getTCPSocket());
+      ctx->setState(new S2C_CREATEACCOUNT2());
+      return;
     }
+
+    BNCSOutputStream logon(0x3a);
+    logon.serialize32(ctx->clientToken);
+    logon.serialize32(ctx->serverToken);
+
+    // Battle.net password hashes are hashed twice using XSHA-1. First, the
+    // password is hashed by itself, then the following data is hashed again
+    // and sent to Battle.net:
+    // (UINT32) Client Token
+    // (UINT32) Server Token
+    // (UINT32)[5] First password hash
+    // The logic below is taken straight from pvpgn
+    struct {
+      pvpgn::bn_int ticks;
+      pvpgn::bn_int sessionkey;
+      pvpgn::bn_int passhash1[5];
+    } temp;
+    uint32_t passhash2[5];
+
+    pvpgn::bn_int_set(&temp.ticks, ntohl(ctx->clientToken));
+    pvpgn::bn_int_set(&temp.sessionkey, ntohl(ctx->serverToken));
+    pvpgn::hash_to_bnhash((pvpgn::t_hash const *)pw, temp.passhash1);
+    pvpgn::bnet_hash(&passhash2, sizeof(temp), &temp); /* do the double hash */
+
+    for (int i = 0; i < 20; i++) {
+      logon.serialize8(reinterpret_cast<uint8_t *>(passhash2)[i]);
+    }
+    logon.serialize(user.c_str());
+    logon.flush(ctx->getTCPSocket());
+
+    ctx->setState(new S2C_LOGONRESPONSE2());
+  }
 };
 
 class S2C_SID_AUTH_CHECK : public NetworkState {
@@ -1705,7 +1714,7 @@ class S2C_SID_AUTH_CHECK : public NetworkState {
             switch (result) {
             case 0x000:
                 // Passed challenge
-                ctx->setState(new C2S_LOGONRESPONSE2());
+                ctx->setState(new C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT());
                 return;
             case 0x100:
                 // Old game version

From d529d9b0a9fe1f4b73df019661392aacd9e96ff1 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Tue, 24 Nov 2020 14:36:18 +0100
Subject: [PATCH 48/70] use our game udp socket for bnet hole punching

---
 src/network/network.cpp        |  3 ++
 src/network/online_service.cpp | 69 +++++++++++++++++++---------------
 2 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/src/network/network.cpp b/src/network/network.cpp
index eabc20fdd..6133f4c98 100644
--- a/src/network/network.cpp
+++ b/src/network/network.cpp
@@ -403,6 +403,9 @@ static void NetworkSendPacket(const CNetworkCommandQueue(&ncq)[MaxNetworkCommand
 */
 void InitNetwork1()
 {
+	if (NetworkFildes.IsValid()) {
+		return;
+	}
 	CNetworkParameter::Instance.FixValues();
 
 	NetInit(); // machine dependent setup
diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 0e4c61e1c..f126f4420 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -296,7 +296,9 @@ public:
     };
 
     void flush(CUDPSocket *sock, CHost *host) {
-        sock->Send(*host, getBuffer(), pos);
+        if (sock->IsValid()) {
+            sock->Send(*host, getBuffer(), pos);
+        }
     };
 
 private:
@@ -647,7 +649,6 @@ protected:
 class Context : public OnlineContext {
 public:
     Context() {
-        this->udpSocket = new CUDPSocket();
         this->tcpSocket = new CTCPSocket();
         this->istream = new BNCSInputStream(tcpSocket);
         this->state = NULL;
@@ -669,7 +670,6 @@ public:
         if (state != NULL) {
             delete state;
         }
-        delete udpSocket;
         delete tcpSocket;
         delete host;
     }
@@ -697,7 +697,6 @@ public:
             BNCSOutputStream leave(0x10);
             leave.flush(tcpSocket);
         }
-        udpSocket->Close();
         tcpSocket->Close();
         state = NULL;
         clientToken = MyRand();
@@ -1108,7 +1107,18 @@ public:
         host = arg;
     }
 
-    CUDPSocket *getUDPSocket() { return udpSocket; }
+    CUDPSocket *getUDPSocket() {
+        if (!NetworkFildes.IsValid()) {
+            if (canDoUdp) {
+                InitNetwork1();
+                if (!NetworkFildes.IsValid()) {
+                    // do not try again
+                    canDoUdp = false;
+                }
+            }
+        }
+        return &NetworkFildes;
+    }
 
     CTCPSocket *getTCPSocket() { return tcpSocket; }
 
@@ -1145,7 +1155,7 @@ private:
 
     NetworkState *state;
     CHost *host;
-    CUDPSocket *udpSocket;
+    bool canDoUdp;
     CTCPSocket *tcpSocket;
     BNCSInputStream *istream;
 
@@ -1473,27 +1483,29 @@ public:
     };
 
     virtual void doOneStep(Context *ctx) {
-        if (ctx->getUDPSocket()->HasDataToRead(1)) {
-            // PKT_SERVERPING
-            //  (UINT8) 0xFF
-            //  (UINT8) 0x05
-            // (UINT16) 8
-            // (UINT32) UDP Code
-            char buf[8];
-            int received = ctx->getUDPSocket()->Recv(buf, 8, ctx->getHost());
-            if (received == 8) {
-                uint32_t udpCode = reinterpret_cast<uint32_t*>(buf)[1];
-                BNCSOutputStream udppingresponse(0x14);
-                udppingresponse.serialize32(udpCode);
-                udppingresponse.flush(ctx->getTCPSocket());
+        if (ctx->getUDPSocket()->IsValid()) {
+            if (ctx->getUDPSocket()->HasDataToRead(1)) {
+                // PKT_SERVERPING
+                //  (UINT8) 0xFF
+                //  (UINT8) 0x05
+                // (UINT16) 8
+                // (UINT32) UDP Code
+                char buf[8];
+                int received = ctx->getUDPSocket()->Recv(buf, 8, ctx->getHost());
+                if (received == 8) {
+                    uint32_t udpCode = reinterpret_cast<uint32_t*>(buf)[1];
+                    BNCSOutputStream udppingresponse(0x14);
+                    udppingresponse.serialize32(udpCode);
+                    udppingresponse.flush(ctx->getTCPSocket());
+                }
+            } else {
+                retries++;
+                if (retries < 50) {
+                    return;
+                }
+                // we're using a timeout of 1ms, so now we've been waiting at
+                // the very least for 5 seconds... let's skip UDP then
             }
-        } else {
-            retries++;
-            if (retries < 50) {
-                return;
-            }
-            // we're using a timeout of 1ms, so now we've been waiting at
-            // the very least for 5 seconds... let's skip UDP then
         }
         ctx->setState(new C2S_ENTERCHAT());
     }
@@ -1870,11 +1882,6 @@ class ConnectState : public NetworkState {
             ctx->setState(new DisconnectedState("TCP connect failed for server " + ctx->getHost()->toString()));
             return;
         }
-        if (!ctx->getUDPSocket()->Open(*ctx->getHost())) {
-            std::cerr << "UDP open failed" << std::endl;
-            // ctx->setState(new DisconnectedState("UDP open failed"));
-            // return;
-        }
 
         // Send proto byte
         ctx->getTCPSocket()->Send("\x1", 1);

From bd4169420dbb121c09bb4205f05444fd4c77740c Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Tue, 24 Nov 2020 15:19:14 +0100
Subject: [PATCH 49/70] simplify login procedure

---
 src/network/online_service.cpp | 70 ++++++++++++++--------------------
 1 file changed, 28 insertions(+), 42 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index f126f4420..9acb1d188 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -323,6 +323,18 @@ private:
     int length_pos;
 };
 
+static std::string gameName() {
+    if (!FullGameName.empty()) {
+        return FullGameName;
+    } else {
+        if (!GameName.empty()) {
+            return GameName;
+        } else {
+            return Parameters::Instance.applicationName;
+        }
+    }
+}
+
 class Friend {
 public:
     Friend(std::string name, uint8_t status, uint8_t location, uint32_t product, std::string locationName) {
@@ -734,7 +746,8 @@ public:
 
     void refreshChannels() {
         BNCSOutputStream getlist(0x0b);
-        getlist.serialize32(0x00);
+        // identify as W2BN
+        getlist.serialize32(0x4f);
         getlist.flush(getTCPSocket());
     }
 
@@ -1204,7 +1217,6 @@ public:
     }
 
     virtual void doOneStep(Context *ctx) {
-        ticks++;
         if ((ticks % 5000) == 0) {
             // C>S 0x07 PKT_KEEPALIVE
             // ~5000 frames @ ~50fps ~= 100 seconds
@@ -1222,6 +1234,8 @@ public:
             ctx->refreshGames();
         }
 
+        ticks++;
+
         if (ctx->getTCPSocket()->HasDataToRead(0)) {
             uint8_t msg = ctx->getMsgIStream()->readMessageId();
             if (msg == 0xff) {
@@ -1236,6 +1250,9 @@ public:
             case 0x25: // SID_PING
                 handlePing(ctx);
                 break;
+            case 0x0b: // SID_CHANNELLIST
+                ctx->setChannels(ctx->getMsgIStream()->readStringlist());
+                break;
             case 0x0f: // CHATEVENT
                 handleChatevent(ctx);
                 break;
@@ -1246,7 +1263,7 @@ public:
             case 0x1c:
                 // S>C 0x1C SID_STARTADVEX3
                 if (ctx->getMsgIStream()->read32()) {
-                    ctx->showError("Game creation failed");
+                    ctx->showError("Online game creation failed");
                 }
                 break;
             case 0x65:
@@ -1391,34 +1408,7 @@ private:
         }
     }
 
-    uint32_t ticks;
-};
-
-class S2C_GETCHANNELLIST : public NetworkState {
-    virtual void doOneStep(Context *ctx) {
-        if (ctx->getTCPSocket()->HasDataToRead(0)) {
-            uint8_t msg = ctx->getMsgIStream()->readMessageId();
-            if (msg == 0xff) {
-                // try again next time
-                return;
-            }
-            if (msg != 0x0b) {
-                std::string error = std::string("Expected SID_GETCHANNELLIST, got msg id ");
-                error += std::to_string(msg);
-                ctx->setState(new DisconnectedState(error));
-            }
-
-            std::vector<std::string> channels = ctx->getMsgIStream()->readStringlist();
-            ctx->getMsgIStream()->finishMessage();
-            ctx->setChannels(channels);
-
-            // request our user info and refresh the active games list
-            ctx->requestExtraUserInfo(ctx->getUsername());
-            ctx->refreshGames();
-
-            ctx->setState(new S2C_CHATEVENT());
-        }
-    }
+    uint64_t ticks;
 };
 
 class S2C_ENTERCHAT : public NetworkState {
@@ -1452,7 +1442,9 @@ class S2C_ENTERCHAT : public NetworkState {
                 ctx->showInfo("Statstring after logon: " + statString);
             }
 
-            ctx->setState(new S2C_GETCHANNELLIST());
+            ctx->requestExtraUserInfo(ctx->getUsername());
+
+            ctx->setState(new S2C_CHATEVENT());
         }
     }
 };
@@ -1469,9 +1461,11 @@ class C2S_ENTERCHAT : public NetworkState {
 
         BNCSOutputStream join(0x0c);
         join.serialize32(0x01); // first-join
-        join.serialize("ignored");
+        join.serialize(gameName().c_str());
         join.flush(ctx->getTCPSocket());
 
+        ctx->refreshChannels();
+
         ctx->setState(new S2C_ENTERCHAT());
     }
 };
@@ -1810,15 +1804,7 @@ class S2C_SID_AUTH_INFO : public NetworkState {
             check.serialize32(0);
             // EXE information
             std::string exeInfo("");
-            if (!FullGameName.empty()) {
-                exeInfo += FullGameName;
-            } else {
-                if (!GameName.empty()) {
-                    exeInfo += GameName;
-                } else {
-                    exeInfo += Parameters::Instance.applicationName;
-                }
-            }
+            exeInfo += gameName();
             exeInfo += " ";
             exeInfo += StratagusLastModifiedDate;
             exeInfo += " ";

From 91b2eb51e911dc666fb3a7773fe9c069d0608e4c Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Tue, 24 Nov 2020 15:34:05 +0100
Subject: [PATCH 50/70] add joined channel to list if it is unlisted

---
 src/network/online_service.cpp | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 9acb1d188..8d5a57a24 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -947,6 +947,17 @@ public:
     // UI information
     void setCurrentChannel(std::string name) {
         this->currentChannel = name;
+        bool unlisted = true;
+        for (const auto c : channelList) {
+            if (c == name) {
+                unlisted = false;
+                break;
+            }
+        }
+        if (unlisted) {
+            channelList.push_back(name);
+            setChannels(channelList);
+        }
         if (SetActiveChannel != NULL) {
           SetActiveChannel->pushPreamble();
           SetActiveChannel->pushString(name);

From 817121f8ab9546d29c3fb6d83f7e840e4c6b6243 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Wed, 25 Nov 2020 17:45:47 +0100
Subject: [PATCH 51/70] minor tweak

---
 src/network/online_service.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 8d5a57a24..20c78cfb6 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1383,13 +1383,13 @@ private:
             ctx->removeUser(username);
             ctx->showInfo(username + " left");
         case 0x04: // recv whisper
-            ctx->showChat(username + " whispers: " + text);
+            ctx->showChat(username + " whispers " + text);
             break;
         case 0x05: // recv chat
             ctx->showChat(username + ": " + text);
             break;
         case 0x06: // recv broadcast
-            ctx->showChat("[BROADCAST]: " + text);
+            ctx->showInfo("[BROADCAST]: " + text);
             break;
         case 0x07: // channel info
             ctx->setCurrentChannel(text);
@@ -1477,6 +1477,9 @@ class C2S_ENTERCHAT : public NetworkState {
 
         ctx->refreshChannels();
 
+        // TODO: maybe send 0x45 SID_NETGAMEPORT to report our gameport on pvpgn
+        // to whatever the user specified on the cmdline?
+
         ctx->setState(new S2C_ENTERCHAT());
     }
 };

From f4502393da4db0f2b3522c65759b7aa80ec5c61f Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 26 Nov 2020 07:43:02 +0100
Subject: [PATCH 52/70] make sure we close the UDP network when departing the
 online server

---
 src/network/online_service.cpp | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 20c78cfb6..8ce76e631 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -709,6 +709,9 @@ public:
             BNCSOutputStream leave(0x10);
             leave.flush(tcpSocket);
         }
+        if (canDoUdp == 1) {
+            ExitNetwork1();
+        }
         tcpSocket->Close();
         state = NULL;
         clientToken = MyRand();
@@ -1133,11 +1136,14 @@ public:
 
     CUDPSocket *getUDPSocket() {
         if (!NetworkFildes.IsValid()) {
-            if (canDoUdp) {
+            if (canDoUdp == -1) {
                 InitNetwork1();
-                if (!NetworkFildes.IsValid()) {
+                if (NetworkFildes.IsValid()) {
+                    // I started it, so I'll shut it down
+                    canDoUdp = 1;
+                } else {
                     // do not try again
-                    canDoUdp = false;
+                    canDoUdp = 0;
                 }
             }
         }
@@ -1179,7 +1185,7 @@ private:
 
     NetworkState *state;
     CHost *host;
-    bool canDoUdp;
+    int8_t canDoUdp = -1; // -1,0,1 --- not tried, doesn't work, I created it
     CTCPSocket *tcpSocket;
     BNCSInputStream *istream;
 

From a02f20caca930d7c11fd90fc78232515d4cb7145 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 26 Nov 2020 09:14:26 +0100
Subject: [PATCH 53/70] support custom /udppunch whisper message to help with
 NAT

---
 src/include/netconnect.h       |  3 +++
 src/network/online_service.cpp | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/src/include/netconnect.h b/src/include/netconnect.h
index 7191dbbda..083e9569f 100644
--- a/src/include/netconnect.h
+++ b/src/include/netconnect.h
@@ -32,6 +32,7 @@
 //@{
 
 #include "net_message.h"
+#include "network/netsockets.h"
 
 class CHost;
 
@@ -123,6 +124,8 @@ extern void NetworkProcessServerRequest();  /// Menu Loop: Send out server reque
 extern void NetworkServerResyncClients();   /// Menu Loop: Server: Mark clients state to send stateinfo message
 extern void NetworkDetachFromServer();      /// Menu Loop: Client: Send GoodBye to the server and detach
 
+extern void NetworkSendICMessage(CUDPSocket &socket, const CHost &host, const CInitMessage_Header &msg);
+
 //@}
 
 #endif // !__NETCONNECT_H__
diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 8ce76e631..445633015 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -747,6 +747,10 @@ public:
         msg.flush(getTCPSocket());
     }
 
+    void punchNAT(std::string username) {
+        sendText("/udppunch " + username);
+    }
+
     void refreshChannels() {
         BNCSOutputStream getlist(0x0b);
         // identify as W2BN
@@ -1389,6 +1393,26 @@ private:
             ctx->removeUser(username);
             ctx->showInfo(username + " left");
         case 0x04: // recv whisper
+            if (!text.empty()) {
+                const char *prefix = "/udppunch ";
+                unsigned int a, b, c, d, ip, port;
+                if (text.rfind(prefix, 0)) {
+                    int res = sscanf(text.substr(strlen(prefix)).c_str(), "%uhh.%uhh.%uhh.%uhh:%d", &d, &c, &b, &a, &port);
+                    if (res == 5) {
+                         ip = a | b << 8 | c << 16 | d << 24;
+                         if (NetConnectType == 1 && !GameRunning) { // the server, waiting for clients
+                             const CInitMessage_Header message(MessageInit_FromServer, ICMAYT);
+                             NetworkSendICMessage(*(ctx->getUDPSocket()), CHost(ip, port), message);
+                         } else {
+                             // the client will connect now and send packages, anyway.
+                             // any other state shouldn't try to udp hole punch at this stage
+                         }
+                         return;
+                    } else {
+                        // incorrect format, fall through and treat as normal whisper;
+                    }
+                }
+            }
             ctx->showChat(username + " whispers " + text);
             break;
         case 0x05: // recv chat
@@ -2168,6 +2192,12 @@ static int CclRequestUserInfo(lua_State *l) {
     return 0;
 }
 
+static int CclPunchNAT(lua_State *l) {
+    LuaCheckArgs(l, 1);
+    _ctx.punchNAT(LuaToString(l, 1));
+    return 0;
+}
+
 void OnlineServiceCclRegister() {
     lua_createtable(Lua, 0, 3);
 
@@ -2202,6 +2232,8 @@ void OnlineServiceCclRegister() {
     lua_setfield(Lua, -2, "startadvertising");
     lua_pushcfunction(Lua, CclStopAdvertisingOnlineGame);
     lua_setfield(Lua, -2, "stopadvertising");
+    lua_pushcfunction(Lua, CclPunchNAT);
+    lua_setfield(Lua, -2, "punchNAT");
 
     lua_setglobal(Lua, "OnlineService");
 }

From 9f125d71410f978a6b9b83fecc89ab5fa13044ae Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 26 Nov 2020 10:18:34 +0100
Subject: [PATCH 54/70] tweak refresh ticks

---
 src/network/online_service.cpp | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 445633015..3f819c373 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1238,20 +1238,22 @@ public:
     }
 
     virtual void doOneStep(Context *ctx) {
-        if ((ticks % 5000) == 0) {
+        if ((ticks % 500) == 0) {
             // C>S 0x07 PKT_KEEPALIVE
-            // ~5000 frames @ ~50fps ~= 100 seconds
+            // ~500 frames @ ~50fps ~= 10 seconds
             BNCSOutputStream keepalive(0x07);
             keepalive.serialize32(ticks);
             keepalive.flush(ctx->getUDPSocket(), ctx->getHost());
         }
 
-        if ((ticks % 5000) == 0) {
+        if ((ticks % 2000) == 0) {
+            // ~2000 frames @ ~50fps ~= 40 seconds
             ctx->refreshFriends();
             ctx->refreshChannels();
         }
 
-        if ((ticks % 300) == 0) {
+        if ((ticks % 500) == 0) {
+            // ~300 frames @ ~50fps ~= 10 seconds
             ctx->refreshGames();
         }
 

From 81b497d17b913d61967bb2abc2e1afa5c75c307a Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 26 Nov 2020 11:05:21 +0100
Subject: [PATCH 55/70] move dump helper into static function

---
 src/network/online_service.cpp | 58 +++++++++++++++++-----------------
 1 file changed, 29 insertions(+), 29 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 3f819c373..697b0bf2d 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -56,6 +56,35 @@
 
 #include "./xsha1.h"
 
+static void dump(uint8_t* buffer, int received_bytes) {
+    std::cout << "Raw contents >>>" << std::endl;
+    for (int i = 0; i < received_bytes; i += 8) {
+        std::cout << std::hex;
+        int j = i;
+        for (; j < received_bytes && j < (i + 8); j++) {
+            uint8_t byte = buffer[j];
+            if (byte < 0x10) {
+                std::cout << "0";
+            }
+            std::cout << (unsigned short)byte << " ";
+        }
+        for (; j < (i + 9); j++) {
+            std::cout << "   "; // room for 2 hex digits and one space
+        }
+        j = i;
+        for (; j < received_bytes && j < (i + 8); j++) {
+            char c = buffer[j];
+            if (c >= 32) {
+                std::cout.write(&c, 1);
+            } else {
+                std::cout.write(".", 1);
+            }
+        }
+        std::cout << std::endl;
+    }
+    std::cout << "<<<" << std::endl;
+}
+
 class BNCSInputStream {
 public:
     BNCSInputStream(CTCPSocket *socket) {
@@ -141,35 +170,6 @@ public:
         return received_bytes;
     }
 
-    void dump() {
-        std::cout << "Raw contents >>>" << std::endl;
-        for (int i = 0; i < received_bytes; i += 8) {
-            std::cout << std::hex;
-            int j = i;
-            for (; j < received_bytes && j < (i + 8); j++) {
-                uint8_t byte = buffer[j];
-                if (byte < 0x10) {
-                    std::cout << "0";
-                }
-                std::cout << (unsigned short)byte << " ";
-            }
-            for (; j < (i + 9); j++) {
-                std::cout << "   "; // room for 2 hex digits and one space
-            }
-            j = i;
-            for (; j < received_bytes && j < (i + 8); j++) {
-                char c = buffer[j];
-                if (c >= 32) {
-                    std::cout.write(&c, 1);
-                } else {
-                    std::cout.write(".", 1);
-                }
-            }
-            std::cout << std::endl;
-        }
-        std::cout << "<<<" << std::endl;
-    }
-
     std::string string32() {
         // uint32 encoded (4-byte) string
         uint32_t data = read32();

From 37ee7c69fb246a224ff05f00d35dfd3161e49aad Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 26 Nov 2020 11:05:40 +0100
Subject: [PATCH 56/70] udp packages do not have a length

---
 src/network/online_service.cpp | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 697b0bf2d..dde2276ed 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -237,18 +237,25 @@ private:
 
 class BNCSOutputStream {
 public:
-    BNCSOutputStream(uint8_t id) {
+    BNCSOutputStream(uint8_t id, bool udp = false) {
         // Every BNCS message has the same header:
         //  (UINT8) Always 0xFF
         //  (UINT8) Message ID
         // (UINT16) Message length, including this header
         //   (VOID) Message data
+        // The UDP messages are instead 4 bytes for the id, then the content
         this->sz = 16;
         this->buf = (uint8_t*) calloc(sizeof(uint8_t), this->sz);
         this->pos = 0;
-        serialize8(0xff);
-        serialize8(id);
-        this->length_pos = pos;
+        if (udp) {
+            serialize8(id);
+            serialize8(0);
+            this->length_pos = -1;
+        } else {
+            serialize8(0xff);
+            serialize8(id);
+            this->length_pos = pos;
+        }
         serialize16((uint16_t)0);
     };
     ~BNCSOutputStream() {
@@ -303,9 +310,11 @@ public:
 
 private:
     uint8_t *getBuffer() {
-        // insert length to make it a valid buffer
-        uint16_t *view = reinterpret_cast<uint16_t *>(buf + length_pos);
-        *view = pos;
+        // if needed, insert length to make it a valid buffer
+        if (length_pos >= 0) {
+            uint16_t *view = reinterpret_cast<uint16_t *>(buf + length_pos);
+            *view = pos;
+        }
         return buf;
     };
 

From d04351345334cd55286c07e41dd08142bd35d155 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 26 Nov 2020 11:06:39 +0100
Subject: [PATCH 57/70] fix setting udp connection info on the server

---
 src/network/online_service.cpp | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index dde2276ed..1315d6f38 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -960,6 +960,13 @@ public:
         }
     }
 
+    void sendUdpConnectionInfo() {
+        BNCSOutputStream conntest(0x09, true);
+        conntest.serialize32(serverToken);
+        conntest.serialize32(udpToken);
+        conntest.flush(getUDPSocket(), getHost());
+    }
+
     // UI information
     void setCurrentChannel(std::string name) {
         this->currentChannel = name;
@@ -1179,6 +1186,7 @@ public:
 
     uint32_t clientToken;
     uint32_t serverToken;
+    uint32_t udpToken;
 
     LuaCallback *AddUser = NULL;
     LuaCallback *RemoveUser = NULL;
@@ -1250,7 +1258,7 @@ public:
         if ((ticks % 500) == 0) {
             // C>S 0x07 PKT_KEEPALIVE
             // ~500 frames @ ~50fps ~= 10 seconds
-            BNCSOutputStream keepalive(0x07);
+            BNCSOutputStream keepalive(0x07, true);
             keepalive.serialize32(ticks);
             keepalive.flush(ctx->getUDPSocket(), ctx->getHost());
         }
@@ -1495,6 +1503,8 @@ class S2C_ENTERCHAT : public NetworkState {
             }
 
             ctx->requestExtraUserInfo(ctx->getUsername());
+            // send again
+            ctx->sendUdpConnectionInfo();
 
             ctx->setState(new S2C_CHATEVENT());
         }
@@ -1832,17 +1842,15 @@ class S2C_SID_AUTH_INFO : public NetworkState {
             assert(logonType == 0x00); // only support Broken SHA-1 logon for now
             uint32_t serverToken = ctx->getMsgIStream()->read32();
             ctx->serverToken = htonl(serverToken); // keep in network order
-            uint32_t udpValue = ctx->getMsgIStream()->read32();
+            uint32_t udpToken = ctx->getMsgIStream()->read32();
+            ctx->udpToken = htonl(udpToken);
             uint64_t mpqFiletime = ctx->getMsgIStream()->readFiletime();
             std::string mpqFilename = ctx->getMsgIStream()->readString();
             std::string formula = ctx->getMsgIStream()->readString();
             ctx->getMsgIStream()->finishMessage();
 
             // immediately respond with pkt_conntest2 udp msg
-            BNCSOutputStream conntest(0x09);
-            conntest.serialize32(serverToken);
-            conntest.serialize32(udpValue);
-            conntest.flush(ctx->getUDPSocket(), ctx->getHost());
+            ctx->sendUdpConnectionInfo();
 
             // immediately respond with SID_AUTH_CHECK
             BNCSOutputStream check(0x51);

From 0b1f3157aa28c82c9e6722ccc7ec4f2d17775aef Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 26 Nov 2020 11:09:08 +0100
Subject: [PATCH 58/70] fix echoreply response

---
 src/network/online_service.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 1315d6f38..93097f333 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1340,7 +1340,7 @@ private:
     void handlePing(Context *ctx) {
         uint32_t pingValue = ctx->getMsgIStream()->read32();
         BNCSOutputStream buffer(0x25);
-        buffer.serialize32(pingValue);
+        buffer.serialize32(htonl(pingValue));
         send(ctx, &buffer);
     }
 
@@ -1903,7 +1903,7 @@ class S2C_SID_PING : public NetworkState {
 
             // immediately respond with C2S_SID_PING
             BNCSOutputStream buffer(0x25);
-            buffer.serialize32(pingValue);
+            buffer.serialize32(htonl(pingValue));
             send(ctx, &buffer);
 
             ctx->setState(new S2C_SID_AUTH_INFO());

From 4bf454058b341972d1b52edc897136ec33f705b0 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 26 Nov 2020 13:22:04 +0100
Subject: [PATCH 59/70] since UDP port is shared with other network events, we
 need to handle incoming UDP packages differently

---
 src/include/online_service.h   |  5 +++
 src/network/network.cpp        |  5 +++
 src/network/online_service.cpp | 70 +++++++++++++++-------------------
 3 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/src/include/online_service.h b/src/include/online_service.h
index 3eedda892..16157c4ed 100644
--- a/src/include/online_service.h
+++ b/src/include/online_service.h
@@ -3,10 +3,15 @@
 
 #include <string>
 
+#include "network/netsockets.h"
+
 class OnlineContext {
 public:
     virtual ~OnlineContext() { };
 
+    // called in the sdl event loop
+    virtual bool handleUDP(const unsigned char *buffer, int len, CHost host) = 0;
+
     // called in the sdl event loop
     virtual void doOneStep() = 0;
 
diff --git a/src/network/network.cpp b/src/network/network.cpp
index 6133f4c98..3fa084d41 100644
--- a/src/network/network.cpp
+++ b/src/network/network.cpp
@@ -219,6 +219,7 @@
 //  Includes
 //----------------------------------------------------------------------------
 
+#include "online_service.h"
 #include "stratagus.h"
 
 #include <stddef.h>
@@ -866,6 +867,10 @@ void NetworkEvent()
 		return;
 	}
 
+	if (OnlineContextHandler->handleUDP(buf, len, host)) {
+		return;
+	}
+
 	// Setup messages
 	if (NetConnectRunning) {
 		if (NetworkParseSetupEvent(buf, len, host)) {
diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 93097f333..709966945 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1176,6 +1176,34 @@ public:
 
     virtual void doOneStep() { if (this->state != NULL) this->state->doOneStep(this); }
 
+    virtual bool handleUDP(const unsigned char *buffer, int len, CHost host) {
+        if (host.getIp() != getHost()->getIp() || host.getPort() != getHost()->getPort()) {
+            return false;
+        }
+
+        uint8_t id = 0;
+        if (len >= 4) {
+            uint8_t id = buffer[0];
+        }
+        switch (id) {
+        case 0x05:
+            // PKT_SERVERPING
+            // (UINT32) 0x05 0x00 0x00 0x00
+            // (UINT32) UDP Code
+            {
+                const uint32_t udpCode = reinterpret_cast<const uint32_t*>(buffer)[1];
+                BNCSOutputStream udppingresponse(0x14);
+                udppingresponse.serialize32(udpCode);
+                udppingresponse.flush(getTCPSocket());
+            }
+            break;
+        default:
+            // unknown package, ignore
+            break;
+        }
+        return true;
+    }
+
     void setState(NetworkState* newState) {
         assert (newState != this->state);
         if (this->state != NULL) {
@@ -1535,44 +1563,6 @@ class C2S_ENTERCHAT : public NetworkState {
     }
 };
 
-class S2C_PKT_SERVERPING : public NetworkState {
-public:
-    S2C_PKT_SERVERPING() {
-        this->retries = 0;
-    };
-
-    virtual void doOneStep(Context *ctx) {
-        if (ctx->getUDPSocket()->IsValid()) {
-            if (ctx->getUDPSocket()->HasDataToRead(1)) {
-                // PKT_SERVERPING
-                //  (UINT8) 0xFF
-                //  (UINT8) 0x05
-                // (UINT16) 8
-                // (UINT32) UDP Code
-                char buf[8];
-                int received = ctx->getUDPSocket()->Recv(buf, 8, ctx->getHost());
-                if (received == 8) {
-                    uint32_t udpCode = reinterpret_cast<uint32_t*>(buf)[1];
-                    BNCSOutputStream udppingresponse(0x14);
-                    udppingresponse.serialize32(udpCode);
-                    udppingresponse.flush(ctx->getTCPSocket());
-                }
-            } else {
-                retries++;
-                if (retries < 50) {
-                    return;
-                }
-                // we're using a timeout of 1ms, so now we've been waiting at
-                // the very least for 5 seconds... let's skip UDP then
-            }
-        }
-        ctx->setState(new C2S_ENTERCHAT());
-    }
-
-private:
-    int retries;
-};
-
 class C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT : public NetworkState {
   virtual void doOneStep(Context *ctx);
 };
@@ -1674,8 +1664,8 @@ class S2C_LOGONRESPONSE2 : public NetworkState {
 
             switch (status) {
             case 0x00:
-                // success. we need to send SID_UDPPINGRESPONSE before entering chat
-                ctx->setState(new S2C_PKT_SERVERPING());
+                // success. we will send SID_UDPPINGRESPONSE before entering chat
+                ctx->setState(new C2S_ENTERCHAT());
                 return;
             case 0x01:
             case 0x010000:

From ab4f322c2d7d472b232688683b7e22fee8da3d4d Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 26 Nov 2020 14:11:22 +0100
Subject: [PATCH 60/70] fix array index oob

---
 src/network/online_service.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 709966945..0fbbb47c0 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1441,10 +1441,10 @@ private:
             ctx->showInfo(username + " left");
         case 0x04: // recv whisper
             if (!text.empty()) {
-                const char *prefix = "/udppunch ";
+                std::string prefix = "/udppunch ";
                 unsigned int a, b, c, d, ip, port;
-                if (text.rfind(prefix, 0)) {
-                    int res = sscanf(text.substr(strlen(prefix)).c_str(), "%uhh.%uhh.%uhh.%uhh:%d", &d, &c, &b, &a, &port);
+                if (text.size() > prefix.size() && text.rfind(prefix, 0)) {
+                    int res = sscanf(text.substr(prefix.size()).c_str(), "%uhh.%uhh.%uhh.%uhh:%d", &d, &c, &b, &a, &port);
                     if (res == 5) {
                          ip = a | b << 8 | c << 16 | d << 24;
                          if (NetConnectType == 1 && !GameRunning) { // the server, waiting for clients

From 63f6a416780d3f8f8c698862449b23c3168c9873 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 26 Nov 2020 14:42:21 +0100
Subject: [PATCH 61/70] fix udppunch whisper parsing

---
 src/network/online_service.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 0fbbb47c0..8a59a5215 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1443,9 +1443,9 @@ private:
             if (!text.empty()) {
                 std::string prefix = "/udppunch ";
                 unsigned int a, b, c, d, ip, port;
-                if (text.size() > prefix.size() && text.rfind(prefix, 0)) {
-                    int res = sscanf(text.substr(prefix.size()).c_str(), "%uhh.%uhh.%uhh.%uhh:%d", &d, &c, &b, &a, &port);
-                    if (res == 5) {
+                if (text.size() > prefix.size() && text.rfind(prefix, 0) != std::string::npos) {
+                    int res = sscanf(text.substr(prefix.size()).c_str(), "%d.%d.%d.%d:%d", &a, &b, &c, &d, &port);
+                    if (res == 5 && a < 255 && b < 255 && c < 255 && d < 255 && port > 1024) {
                          ip = a | b << 8 | c << 16 | d << 24;
                          if (NetConnectType == 1 && !GameRunning) { // the server, waiting for clients
                              const CInitMessage_Header message(MessageInit_FromServer, ICMAYT);

From 83ef84815689632de732f35c55c398999e2a18e6 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 26 Nov 2020 16:19:55 +0100
Subject: [PATCH 62/70] request our own external address in the beginning so we
 can udp punch without help

---
 src/network/online_service.cpp | 61 ++++++++++++++++++++++++----------
 1 file changed, 44 insertions(+), 17 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 8a59a5215..d64568e39 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -728,7 +728,7 @@ public:
         setPassword("");
     }
 
-    void sendText(std::string txt) {
+    void sendText(std::string txt, bool silent = false) {
         // C>S 0x0E SID_CHATCOMMAND
         int pos = 0;
         for (unsigned int pos = 0; pos < txt.size(); pos += 220) {
@@ -740,7 +740,15 @@ public:
             msg.serialize(text.c_str());
             msg.flush(getTCPSocket());
         }
-        showChat(username + ": " + txt);
+        if (!silent) {
+            showChat(username + ": " + txt);
+        }
+    }
+
+    void requestExternalAddress() {
+        // uses the /netinfo command to get which ip:port the server sees from us
+        sendText("/netinfo", true);
+        requestedAddress = true;
     }
 
     void requestExtraUserInfo(std::string username) {
@@ -757,7 +765,9 @@ public:
     }
 
     void punchNAT(std::string username) {
-        sendText("/udppunch " + username);
+        if (externalAddress.isValid()) {
+            sendText("/whisper " + username + " /udppunch " + externalAddress.toString());
+        }
     }
 
     void refreshChannels() {
@@ -1049,6 +1059,30 @@ public:
     std::queue<std::string> *getInfo() { return &info; }
 
     void showInfo(std::string arg) {
+        if (requestedAddress) {
+            // we have requested our external address from the server
+            DebugPrint("Requested Address Info: %s\n" _C_ arg.c_str());
+            if (arg.find("Server TCP: ") != std::string::npos || arg.find("Client TCP: ") != std::string::npos) {
+                // ignore
+                return;
+            }
+            if (arg.find("Client UDP: ") != std::string::npos) {
+                unsigned int a, b, c, d, ip, port;
+                unsigned char prefix[256]; // longer than any BNet message can be
+                int res = sscanf(arg.c_str(), "%s %s %d.%d.%d.%d:%d", prefix, prefix, &a, &b, &c, &d, &port);
+                if (res == 7 && a < 255 && b < 255 && c < 255 && d < 255 && port > 1024) {
+                    ip = a | b << 8 | c << 16 | d << 24;
+                    externalAddress = CHost(ip, port);
+                    DebugPrint("My external address is %s\n" _C_ externalAddress.toString().c_str());
+                }
+                return;
+            }
+            if (arg.find("Game UDP: ") != std::string::npos) {
+                // this is the last line in the /netinfo response
+                requestedAddress = false;
+                return;
+            }
+        }
         std::string infoStr = arg;
         info.push(infoStr);
         if (ShowInfo != NULL) {
@@ -1243,6 +1277,9 @@ private:
     bool hasPassword;
     bool createAccount;
 
+    bool requestedAddress = false;
+    CHost externalAddress;
+
     std::string lastError;
 
     std::string currentChannel;
@@ -1302,6 +1339,10 @@ public:
             ctx->refreshGames();
         }
 
+        if (ticks == 50) {
+            ctx->requestExternalAddress();
+        }
+
         ticks++;
 
         if (ctx->getTCPSocket()->HasDataToRead(0)) {
@@ -1531,8 +1572,6 @@ class S2C_ENTERCHAT : public NetworkState {
             }
 
             ctx->requestExtraUserInfo(ctx->getUsername());
-            // send again
-            ctx->sendUdpConnectionInfo();
 
             ctx->setState(new S2C_CHATEVENT());
         }
@@ -1691,18 +1730,6 @@ class S2C_LOGONRESPONSE2 : public NetworkState {
             }
         }
     }
-
-private:
-    void createAccount(Context *ctx) {
-        BNCSOutputStream msg(0x3d);
-        uint32_t *pw = ctx->getPassword1();
-        for (int i = 0; i < 20; i++) {
-            msg.serialize8(reinterpret_cast<uint8_t*>(pw)[i]);
-        }
-        msg.serialize(ctx->getUsername().c_str());
-        msg.flush(ctx->getTCPSocket());
-        ctx->setState(new S2C_CREATEACCOUNT2());
-    }
 };
 
 void C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT::doOneStep(Context *ctx) {

From 0da52223d857d7d37caf52e9d3ef733f5dc39888 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 26 Nov 2020 16:24:24 +0100
Subject: [PATCH 63/70] debug printing for online service

---
 src/network/online_service.cpp | 34 +++++++++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index d64568e39..1ed6f1e43 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -739,6 +739,7 @@ public:
             BNCSOutputStream msg(0x0e);
             msg.serialize(text.c_str());
             msg.flush(getTCPSocket());
+            DebugPrint("TCP Sent: 0x0e CHATCOMMAND\n");
         }
         if (!silent) {
             showChat(username + ": " + txt);
@@ -762,6 +763,7 @@ public:
             msg.serialize(key.c_str());
         }
         msg.flush(getTCPSocket());
+        DebugPrint("TCP Sent: 0x26 USERINFO\n");
     }
 
     void punchNAT(std::string username) {
@@ -775,6 +777,7 @@ public:
         // identify as W2BN
         getlist.serialize32(0x4f);
         getlist.flush(getTCPSocket());
+        DebugPrint("TCP Sent: 0x0b CHANNELLIST\n");
     }
 
     void refreshGames() {
@@ -789,12 +792,14 @@ public:
         getadvlistex.serialize(""); // no game pw
         getadvlistex.serialize(""); // no game statstring
         getadvlistex.flush(getTCPSocket());
+        DebugPrint("TCP Sent: 0x09 GAMELIST\n");
     }
 
     void refreshFriends() {
         // C>S 0x65 SID_FRIENDSLIST
         BNCSOutputStream msg(0x65);
         msg.flush(getTCPSocket());
+        DebugPrint("TCP Sent: 0x65 FRIENDSLIST\n");
     }
 
     virtual void joinGame(std::string username, std::string pw) {
@@ -808,6 +813,7 @@ public:
         msg.serialize(gameNameFromUsername(username).c_str());
         msg.serialize(pw.c_str());
         msg.flush(getTCPSocket());
+        DebugPrint("TCP Sent: 0x09 NOTIFYJOIN\n");
     }
 
     virtual void leaveGame() {
@@ -919,6 +925,7 @@ public:
 
         msg.serialize(statstring.str().c_str());
         msg.flush(getTCPSocket());
+        DebugPrint("TCP Sent: 0x1c STARTADVEX\n");
     }
 
     virtual void stopAdvertising() {
@@ -927,6 +934,7 @@ public:
         }
         BNCSOutputStream msg(0x02);
         msg.flush(getTCPSocket());
+        DebugPrint("TCP Sent: 0x02 STOPADVEX\n");
     }
 
     virtual void reportGameResult() {
@@ -959,6 +967,7 @@ public:
         msg.serialize(NetworkMapName.c_str());
         msg.serialize(""); // TODO: transmit player scores
         msg.flush(getTCPSocket());
+        DebugPrint("TCP Sent: 0x2c REPORTRESULT\n");
     }
 
     void joinChannel(std::string name) {
@@ -967,6 +976,7 @@ public:
             join.serialize32(0x02); // forced join
             join.serialize(name.c_str());
             join.flush(getTCPSocket());
+            DebugPrint("TCP Sent: 0x0c JOINCHANNEL\n");
         }
     }
 
@@ -975,6 +985,7 @@ public:
         conntest.serialize32(serverToken);
         conntest.serialize32(udpToken);
         conntest.flush(getUDPSocket(), getHost());
+        DebugPrint("UDP Sent: 0x09 connection info\n");
     }
 
     // UI information
@@ -1217,8 +1228,9 @@ public:
 
         uint8_t id = 0;
         if (len >= 4) {
-            uint8_t id = buffer[0];
+            id = buffer[0];
         }
+        DebugPrint("UDP Recv: 0x%x\n" _C_ id);
         switch (id) {
         case 0x05:
             // PKT_SERVERPING
@@ -1229,6 +1241,7 @@ public:
                 BNCSOutputStream udppingresponse(0x14);
                 udppingresponse.serialize32(udpCode);
                 udppingresponse.flush(getTCPSocket());
+                DebugPrint("TCP Sent: 0x14 UDPPINGRESPONSE\n");
             }
             break;
         default:
@@ -1326,6 +1339,7 @@ public:
             BNCSOutputStream keepalive(0x07, true);
             keepalive.serialize32(ticks);
             keepalive.flush(ctx->getUDPSocket(), ctx->getHost());
+            DebugPrint("UDP Sent: 0x07 PKT_KEEPALIVE\n");
         }
 
         if ((ticks % 2000) == 0) {
@@ -1351,6 +1365,7 @@ public:
                 // try again next time
                 return;
             }
+            DebugPrint("TCP Recv: 0x%x\n" _C_ msg);
 
             switch (msg) {
             case 0x00: // SID_NULL
@@ -1404,6 +1419,7 @@ private:
     void handleNull(Context *ctx) {
         BNCSOutputStream buffer(0x00);
         send(ctx, &buffer);
+        DebugPrint("TCP Sent: 0x00 NULL\n");
     }
 
     void handlePing(Context *ctx) {
@@ -1411,6 +1427,7 @@ private:
         BNCSOutputStream buffer(0x25);
         buffer.serialize32(htonl(pingValue));
         send(ctx, &buffer);
+        DebugPrint("TCP Sent: 0x25 PING\n");
     }
 
     void handleGamelist(Context *ctx) {
@@ -1491,6 +1508,7 @@ private:
                          if (NetConnectType == 1 && !GameRunning) { // the server, waiting for clients
                              const CInitMessage_Header message(MessageInit_FromServer, ICMAYT);
                              NetworkSendICMessage(*(ctx->getUDPSocket()), CHost(ip, port), message);
+                             DebugPrint("UDP Sent: UDP punch\n");
                          } else {
                              // the client will connect now and send packages, anyway.
                              // any other state shouldn't try to udp hole punch at this stage
@@ -1549,6 +1567,7 @@ class S2C_ENTERCHAT : public NetworkState {
                 return;
             }
             if (msg == 0x3a) {
+                DebugPrint("TCP Recv: 0x3a LOGONRESPONSE\n");
                 // pvpgn seems to send a successful logonresponse again
                 uint32_t status = ctx->getMsgIStream()->read32();
                 assert(status == 0);
@@ -1560,6 +1579,7 @@ class S2C_ENTERCHAT : public NetworkState {
                 error += std::to_string(msg);
                 ctx->setState(new DisconnectedState(error));
             }
+            DebugPrint("TCP Recv: 0x0a ENTERCHAT\n");
 
             std::string uniqueName = ctx->getMsgIStream()->readString();
             std::string statString = ctx->getMsgIStream()->readString();
@@ -1585,6 +1605,7 @@ class C2S_ENTERCHAT : public NetworkState {
         enterchat.serialize(ctx->getUsername().c_str());
         enterchat.serialize("");
         enterchat.flush(ctx->getTCPSocket());
+        DebugPrint("TCP Sent: 0x0a ENTERCHAT\n");
 
         ctx->refreshChannels();
 
@@ -1592,6 +1613,7 @@ class C2S_ENTERCHAT : public NetworkState {
         join.serialize32(0x01); // first-join
         join.serialize(gameName().c_str());
         join.flush(ctx->getTCPSocket());
+        DebugPrint("TCP Sent: 0x0c JOINCHANNEL\n");
 
         ctx->refreshChannels();
 
@@ -1619,6 +1641,7 @@ class S2C_CREATEACCOUNT2 : public NetworkState {
                 error += std::to_string(msg);
                 ctx->setState(new DisconnectedState(error));
             }
+            DebugPrint("TCP Recv: 0x3d CREATEACCOUNT\n");
 
             uint32_t status = ctx->getMsgIStream()->read32();
             std::string nameSugg = ctx->getMsgIStream()->readString();
@@ -1697,6 +1720,7 @@ class S2C_LOGONRESPONSE2 : public NetworkState {
                 error += std::to_string(msg);
                 ctx->setState(new DisconnectedState(error));
             }
+            DebugPrint("TCP Sent: 0x3a LOGONRESPONSE\n");
 
             uint32_t status = ctx->getMsgIStream()->read32();
             ctx->getMsgIStream()->finishMessage();
@@ -1744,6 +1768,7 @@ void C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT::doOneStep(Context *ctx) {
       }
       msg.serialize(ctx->getUsername().c_str());
       msg.flush(ctx->getTCPSocket());
+      DebugPrint("TCP Sent: 0x3d CREATEACOUNT\n");
       ctx->setState(new S2C_CREATEACCOUNT2());
       return;
     }
@@ -1776,6 +1801,7 @@ void C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT::doOneStep(Context *ctx) {
     }
     logon.serialize(user.c_str());
     logon.flush(ctx->getTCPSocket());
+    DebugPrint("TCP Sent: 0x3a LOGIN\n");
 
     ctx->setState(new S2C_LOGONRESPONSE2());
   }
@@ -1794,6 +1820,7 @@ class S2C_SID_AUTH_CHECK : public NetworkState {
                 error += std::to_string(msg);
                 ctx->setState(new DisconnectedState(error));
             }
+            DebugPrint("TCP Recv: 0x51 AUTH_CHECK\n");
 
             uint32_t result = ctx->getMsgIStream()->read32();
             std::string info = ctx->getMsgIStream()->readString();
@@ -1854,6 +1881,7 @@ class S2C_SID_AUTH_INFO : public NetworkState {
                 error += std::to_string(msg);
                 ctx->setState(new DisconnectedState(error));
             }
+            DebugPrint("TCP Recv: 0x50 AUTH_INFO\n");
 
             uint32_t logonType = ctx->getMsgIStream()->read32();
             assert(logonType == 0x00); // only support Broken SHA-1 logon for now
@@ -1895,6 +1923,7 @@ class S2C_SID_AUTH_INFO : public NetworkState {
             // Key owner name
             check.serialize(DESCRIPTION);
             check.flush(ctx->getTCPSocket());
+            DebugPrint("TCP Sent: 0x51 AUTH_CHECK\n");
 
             ctx->setState(new S2C_SID_AUTH_CHECK());
         }
@@ -1915,6 +1944,7 @@ class S2C_SID_PING : public NetworkState {
                 error += std::to_string(msg);
                 ctx->setState(new DisconnectedState(error));
             }
+            DebugPrint("TCP Recv: 0x25 PING\n");
             uint32_t pingValue = ctx->getMsgIStream()->read32();
             ctx->getMsgIStream()->finishMessage();
 
@@ -1922,6 +1952,7 @@ class S2C_SID_PING : public NetworkState {
             BNCSOutputStream buffer(0x25);
             buffer.serialize32(htonl(pingValue));
             send(ctx, &buffer);
+            DebugPrint("TCP Sent: 0x25 PING\n");
 
             ctx->setState(new S2C_SID_AUTH_INFO());
         }
@@ -2023,6 +2054,7 @@ class ConnectState : public NetworkState {
         buffer.serialize("United States");
 
         send(ctx, &buffer);
+        DebugPrint("TCP Sent: 0x50 AUTH_INFO\n");
         ctx->setState(new S2C_SID_PING());
     }
 };

From a77a0468f9a04eccc1d4349f8c6042cc86eb7457 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Thu, 26 Nov 2020 20:04:00 +0100
Subject: [PATCH 64/70] request external address in a better place

---
 src/network/online_service.cpp | 31 +++++++++++++------------------
 1 file changed, 13 insertions(+), 18 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 1ed6f1e43..8047d45ce 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -748,8 +748,10 @@ public:
 
     void requestExternalAddress() {
         // uses the /netinfo command to get which ip:port the server sees from us
-        sendText("/netinfo", true);
-        requestedAddress = true;
+        if (!externalAddress.isValid()) {
+            sendText("/netinfo", true);
+            requestedAddress = true;
+        }
     }
 
     void requestExtraUserInfo(std::string username) {
@@ -1012,6 +1014,9 @@ public:
     std::string getCurrentChannel() { return currentChannel; }
 
     void setGamelist(std::vector<Game*> games) {
+        // before we are able to join any game, we should try to get our own
+        // external address to help NAT traversal
+        requestExternalAddress();
         for (const auto value : this->games) {
             delete value;
         }
@@ -1073,10 +1078,6 @@ public:
         if (requestedAddress) {
             // we have requested our external address from the server
             DebugPrint("Requested Address Info: %s\n" _C_ arg.c_str());
-            if (arg.find("Server TCP: ") != std::string::npos || arg.find("Client TCP: ") != std::string::npos) {
-                // ignore
-                return;
-            }
             if (arg.find("Client UDP: ") != std::string::npos) {
                 unsigned int a, b, c, d, ip, port;
                 unsigned char prefix[256]; // longer than any BNet message can be
@@ -1086,12 +1087,10 @@ public:
                     externalAddress = CHost(ip, port);
                     DebugPrint("My external address is %s\n" _C_ externalAddress.toString().c_str());
                 }
-                return;
             }
             if (arg.find("Game UDP: ") != std::string::npos) {
                 // this is the last line in the /netinfo response
                 requestedAddress = false;
-                return;
             }
         }
         std::string infoStr = arg;
@@ -1333,30 +1332,26 @@ public:
     }
 
     virtual void doOneStep(Context *ctx) {
-        if ((ticks % 500) == 0) {
+        if ((ticks % 1000) == 0) {
             // C>S 0x07 PKT_KEEPALIVE
-            // ~500 frames @ ~50fps ~= 10 seconds
+            // ~1000 frames @ ~50fps ~= 20 seconds
             BNCSOutputStream keepalive(0x07, true);
             keepalive.serialize32(ticks);
             keepalive.flush(ctx->getUDPSocket(), ctx->getHost());
             DebugPrint("UDP Sent: 0x07 PKT_KEEPALIVE\n");
         }
 
-        if ((ticks % 2000) == 0) {
-            // ~2000 frames @ ~50fps ~= 40 seconds
+        if ((ticks % 10000) == 0) {
+            // ~10000 frames @ ~50fps ~= 200 seconds
             ctx->refreshFriends();
             ctx->refreshChannels();
         }
 
-        if ((ticks % 500) == 0) {
-            // ~300 frames @ ~50fps ~= 10 seconds
+        if ((ticks % 1000) == 0) {
+            // ~1000 frames @ ~50fps ~= 20 seconds
             ctx->refreshGames();
         }
 
-        if (ticks == 50) {
-            ctx->requestExternalAddress();
-        }
-
         ticks++;
 
         if (ctx->getTCPSocket()->HasDataToRead(0)) {

From 17632c1baa3b914c047c9bfecf92608649dcf899 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Fri, 27 Nov 2020 12:32:39 +0100
Subject: [PATCH 65/70] remove assert for compatibility with Atlas

---
 src/network/online_service.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 8047d45ce..1c6700e41 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1879,7 +1879,8 @@ class S2C_SID_AUTH_INFO : public NetworkState {
             DebugPrint("TCP Recv: 0x50 AUTH_INFO\n");
 
             uint32_t logonType = ctx->getMsgIStream()->read32();
-            assert(logonType == 0x00); // only support Broken SHA-1 logon for now
+            // assert(logonType == 0x00); // only support Broken SHA-1 logon for now
+            DebugPrint("logonType: 0x%x\n" _C_ logonType);
             uint32_t serverToken = ctx->getMsgIStream()->read32();
             ctx->serverToken = htonl(serverToken); // keep in network order
             uint32_t udpToken = ctx->getMsgIStream()->read32();

From 0a8ed97c6043bfb23c2813cc0983284b225e7390 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Fri, 27 Nov 2020 14:36:54 +0100
Subject: [PATCH 66/70] use a more compatible method to get my own address

This was after discussion with other developers on the BNetDocs chat. Since the
/netinfo command is a pvpgn specific thing, we instead advertise a fake game
when logging in and check what the server says our ip:port are.
---
 src/network/online_service.cpp | 62 +++++++++++++++++++---------------
 1 file changed, 34 insertions(+), 28 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 1c6700e41..4061632c6 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -747,10 +747,23 @@ public:
     }
 
     void requestExternalAddress() {
-        // uses the /netinfo command to get which ip:port the server sees from us
-        if (!externalAddress.isValid()) {
-            sendText("/netinfo", true);
+        // start advertising a fake game so we can see our external address in the gamelist
+        if (!requestedAddress) {
             requestedAddress = true;
+            BNCSOutputStream msg(0x1c);
+            msg.serialize32(0x10000000);
+            msg.serialize32(0x00); // uptime
+            msg.serialize16(0x0300); // game type
+            msg.serialize16(0x0100); // sub game type
+            msg.serialize32(0xff); // provider version constant
+            msg.serialize32(0x00); // not ladder
+            msg.serialize(""); // game name
+            msg.serialize(""); // password
+            std::string statstring = ",,,0x04,0x00,0x0a,0x01,0x1234,0x4000,";
+            statstring += getUsername() + "\rudp\r";
+            msg.serialize(statstring.c_str());
+            msg.flush(getTCPSocket());
+            DebugPrint("TCP Sent: 0x1c STARTADVEX\n");
         }
     }
 
@@ -1016,10 +1029,22 @@ public:
     void setGamelist(std::vector<Game*> games) {
         // before we are able to join any game, we should try to get our own
         // external address to help NAT traversal
-        requestExternalAddress();
         for (const auto value : this->games) {
             delete value;
         }
+        if (requestedAddress && !externalAddress.isValid()) {
+            for (int i = 0; i < games.size(); i++) {
+                const auto game = games[i];
+                if (game->getCreator() == getUsername() && game->getMap() == "udp") {
+                    // our fake game, remove and break;
+                    games.erase(games.begin() + i);
+                    externalAddress = game->getHost();
+                    DebugPrint("My external address is %s\n" _C_ externalAddress.toString().c_str());
+                    stopAdvertising();
+                    break;
+                }
+            }
+        }
         this->games = games;
         if (SetGames != NULL) {
           SetGames->pushPreamble();
@@ -1075,24 +1100,6 @@ public:
     std::queue<std::string> *getInfo() { return &info; }
 
     void showInfo(std::string arg) {
-        if (requestedAddress) {
-            // we have requested our external address from the server
-            DebugPrint("Requested Address Info: %s\n" _C_ arg.c_str());
-            if (arg.find("Client UDP: ") != std::string::npos) {
-                unsigned int a, b, c, d, ip, port;
-                unsigned char prefix[256]; // longer than any BNet message can be
-                int res = sscanf(arg.c_str(), "%s %s %d.%d.%d.%d:%d", prefix, prefix, &a, &b, &c, &d, &port);
-                if (res == 7 && a < 255 && b < 255 && c < 255 && d < 255 && port > 1024) {
-                    ip = a | b << 8 | c << 16 | d << 24;
-                    externalAddress = CHost(ip, port);
-                    DebugPrint("My external address is %s\n" _C_ externalAddress.toString().c_str());
-                }
-            }
-            if (arg.find("Game UDP: ") != std::string::npos) {
-                // this is the last line in the /netinfo response
-                requestedAddress = false;
-            }
-        }
         std::string infoStr = arg;
         info.push(infoStr);
         if (ShowInfo != NULL) {
@@ -1347,8 +1354,8 @@ public:
             ctx->refreshChannels();
         }
 
-        if ((ticks % 1000) == 0) {
-            // ~1000 frames @ ~50fps ~= 20 seconds
+        if ((ticks % 500) == 0) {
+            // ~1000 frames @ ~50fps ~= 10 seconds
             ctx->refreshGames();
         }
 
@@ -1450,7 +1457,7 @@ private:
 
 
     void handleFriendlist(Context *ctx) {
-        uint32_t cnt = ctx->getMsgIStream()->read32();
+        uint8_t cnt = ctx->getMsgIStream()->read8();
         std::vector<Friend*> friends;
         while (cnt--) {
             std::string user = ctx->getMsgIStream()->readString();
@@ -1540,7 +1547,7 @@ private:
             ctx->showInfo("Channel restricted");
             break;
         case 0x12: // general info text
-            ctx->showInfo("~<" + text + "~>");
+            ctx->showInfo(text);
             break;
         case 0x13: // error message
             ctx->showError(text);
@@ -1587,6 +1594,7 @@ class S2C_ENTERCHAT : public NetworkState {
             }
 
             ctx->requestExtraUserInfo(ctx->getUsername());
+            ctx->requestExternalAddress();
 
             ctx->setState(new S2C_CHATEVENT());
         }
@@ -1610,8 +1618,6 @@ class C2S_ENTERCHAT : public NetworkState {
         join.flush(ctx->getTCPSocket());
         DebugPrint("TCP Sent: 0x0c JOINCHANNEL\n");
 
-        ctx->refreshChannels();
-
         // TODO: maybe send 0x45 SID_NETGAMEPORT to report our gameport on pvpgn
         // to whatever the user specified on the cmdline?
 

From a7e93459fa9725bba027f85715d3511d6e85f17b Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Fri, 27 Nov 2020 22:23:16 +0100
Subject: [PATCH 67/70] use static message buffer

---
 src/network/online_service.cpp | 21 ++++++++-------------
 1 file changed, 8 insertions(+), 13 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 4061632c6..a35bffe62 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -244,8 +244,6 @@ public:
         // (UINT16) Message length, including this header
         //   (VOID) Message data
         // The UDP messages are instead 4 bytes for the id, then the content
-        this->sz = 16;
-        this->buf = (uint8_t*) calloc(sizeof(uint8_t), this->sz);
         this->pos = 0;
         if (udp) {
             serialize8(id);
@@ -258,8 +256,8 @@ public:
         }
         serialize16((uint16_t)0);
     };
+
     ~BNCSOutputStream() {
-        free(buf);
     };
 
     void serialize32(uint32_t data) {
@@ -319,16 +317,12 @@ private:
     };
 
     void ensureSpace(size_t required) {
-        if (pos + required >= sz) {
-            sz = sz * 2;
-            buf = (uint8_t*) realloc(buf, sz);
-            assert(buf != NULL);
-        }
+        assert(pos + required < MAX_MSG_SIZE);
     }
 
-    uint8_t *buf;
-    unsigned int sz;
-    unsigned int pos;
+    static const uint32_t MAX_MSG_SIZE = 1024; // 1kb for an outgoing message should be plenty
+    uint8_t buf[MAX_MSG_SIZE];
+    unsigned int pos = 0;
     int length_pos;
 };
 
@@ -1033,13 +1027,14 @@ public:
             delete value;
         }
         if (requestedAddress && !externalAddress.isValid()) {
-            for (int i = 0; i < games.size(); i++) {
+            for (unsigned int i = 0; i < games.size(); i++) {
                 const auto game = games[i];
                 if (game->getCreator() == getUsername() && game->getMap() == "udp") {
                     // our fake game, remove and break;
                     games.erase(games.begin() + i);
                     externalAddress = game->getHost();
                     DebugPrint("My external address is %s\n" _C_ externalAddress.toString().c_str());
+                    showInfo("Your external route is " + externalAddress.toString());
                     stopAdvertising();
                     break;
                 }
@@ -1089,7 +1084,7 @@ public:
             ShowUserInfo->pushPreamble();
             std::map<std::string, std::variant<std::string, int>> m;
             m["User"] = extendedInfoNames.at(id);
-            for (int i = 0; i < values.size(); i++) {
+            for (unsigned int i = 0; i < values.size(); i++) {
                 m[defaultUserKeys.at(i)] = values.at(i);
             }
             ShowUserInfo->pushTable(m);

From 4e5fd6773ce7a4aa8da2e051016d268e6f9eab50 Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Sat, 28 Nov 2020 14:44:29 +0100
Subject: [PATCH 68/70] do not attempt to create account multiple times

---
 src/network/online_service.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index a35bffe62..7e99308ab 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1757,6 +1757,7 @@ void C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT::doOneStep(Context *ctx) {
   uint32_t *pw = ctx->getPassword1(); // single-hashed for SID_LOGONRESPONSE2
   if (!user.empty() && pw) {
     if (ctx->shouldCreateAccount()) {
+      ctx->setCreateAccount(false);
       BNCSOutputStream msg(0x3d);
       uint32_t *pw = ctx->getPassword1();
       for (int i = 0; i < 20; i++) {

From 7797fab297bc330fc986603154dc9197f4b0b0ac Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Sat, 28 Nov 2020 14:45:16 +0100
Subject: [PATCH 69/70] fix indentation and whitespace

---
 src/network/online_service.cpp | 164 ++++++++++++++++-----------------
 1 file changed, 82 insertions(+), 82 deletions(-)

diff --git a/src/network/online_service.cpp b/src/network/online_service.cpp
index 7e99308ab..03547860d 100644
--- a/src/network/online_service.cpp
+++ b/src/network/online_service.cpp
@@ -1012,9 +1012,9 @@ public:
             setChannels(channelList);
         }
         if (SetActiveChannel != NULL) {
-          SetActiveChannel->pushPreamble();
-          SetActiveChannel->pushString(name);
-          SetActiveChannel->run();
+            SetActiveChannel->pushPreamble();
+            SetActiveChannel->pushString(name);
+            SetActiveChannel->run();
         }
     }
 
@@ -1042,20 +1042,20 @@ public:
         }
         this->games = games;
         if (SetGames != NULL) {
-          SetGames->pushPreamble();
-          for (const auto value : games) {
-            SetGames->pushTable({{"Creator", value->getCreator()},
-                                 {"Host", value->getHost().toString()},
-                                 {"IsSavedGame", value->isSavedGame()},
-                                 {"Map", value->getMap()},
-                                 {"MaxPlayers", value->maxPlayers()},
-                                 {"Speed", value->getSpeed()},
-                                 {"Approval", value->getApproval()},
-                                 {"Settings", value->getGameSettings()},
-                                 {"Status", value->getGameStatus()},
-                                 {"Type", value->getGameType()}});
-          }
-          SetGames->run();
+            SetGames->pushPreamble();
+            for (const auto value : games) {
+                SetGames->pushTable({{"Creator", value->getCreator()},
+                                     {"Host", value->getHost().toString()},
+                                     {"IsSavedGame", value->isSavedGame()},
+                                     {"Map", value->getMap()},
+                                     {"MaxPlayers", value->maxPlayers()},
+                                     {"Speed", value->getSpeed()},
+                                     {"Approval", value->getApproval()},
+                                     {"Settings", value->getGameSettings()},
+                                     {"Status", value->getGameStatus()},
+                                     {"Type", value->getGameType()}});
+            }
+            SetGames->run();
         }
     }
 
@@ -1066,7 +1066,7 @@ public:
             delete value;
         }
         this->friends = friends;
-        if (SetFriends != NULL) {            
+        if (SetFriends != NULL) {
             SetFriends->pushPreamble();
             for (const auto value : friends) {
                 SetFriends->pushTable({ { "Name", value->getName() },
@@ -1144,7 +1144,7 @@ public:
 
     void setChannels(std::vector<std::string> channels) {
         this->channelList = channels;
-        if (SetChannels != NULL) {            
+        if (SetChannels != NULL) {
             SetChannels->pushPreamble();
             for (const auto value : channels) {
                 SetChannels->pushString(value);
@@ -1501,16 +1501,16 @@ private:
                 if (text.size() > prefix.size() && text.rfind(prefix, 0) != std::string::npos) {
                     int res = sscanf(text.substr(prefix.size()).c_str(), "%d.%d.%d.%d:%d", &a, &b, &c, &d, &port);
                     if (res == 5 && a < 255 && b < 255 && c < 255 && d < 255 && port > 1024) {
-                         ip = a | b << 8 | c << 16 | d << 24;
-                         if (NetConnectType == 1 && !GameRunning) { // the server, waiting for clients
-                             const CInitMessage_Header message(MessageInit_FromServer, ICMAYT);
-                             NetworkSendICMessage(*(ctx->getUDPSocket()), CHost(ip, port), message);
-                             DebugPrint("UDP Sent: UDP punch\n");
-                         } else {
-                             // the client will connect now and send packages, anyway.
-                             // any other state shouldn't try to udp hole punch at this stage
-                         }
-                         return;
+                        ip = a | b << 8 | c << 16 | d << 24;
+                        if (NetConnectType == 1 && !GameRunning) { // the server, waiting for clients
+                            const CInitMessage_Header message(MessageInit_FromServer, ICMAYT);
+                            NetworkSendICMessage(*(ctx->getUDPSocket()), CHost(ip, port), message);
+                            DebugPrint("UDP Sent: UDP punch\n");
+                        } else {
+                            // the client will connect now and send packages, anyway.
+                            // any other state shouldn't try to udp hole punch at this stage
+                        }
+                        return;
                     } else {
                         // incorrect format, fall through and treat as normal whisper;
                     }
@@ -1621,7 +1621,7 @@ class C2S_ENTERCHAT : public NetworkState {
 };
 
 class C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT : public NetworkState {
-  virtual void doOneStep(Context *ctx);
+    virtual void doOneStep(Context *ctx);
 };
 
 class S2C_CREATEACCOUNT2 : public NetworkState {
@@ -1753,55 +1753,55 @@ class S2C_LOGONRESPONSE2 : public NetworkState {
 };
 
 void C2S_LOGONRESPONSE2_OR_C2S_CREATEACCOUNT::doOneStep(Context *ctx) {
-  std::string user = ctx->getUsername();
-  uint32_t *pw = ctx->getPassword1(); // single-hashed for SID_LOGONRESPONSE2
-  if (!user.empty() && pw) {
-    if (ctx->shouldCreateAccount()) {
-      ctx->setCreateAccount(false);
-      BNCSOutputStream msg(0x3d);
-      uint32_t *pw = ctx->getPassword1();
-      for (int i = 0; i < 20; i++) {
-        msg.serialize8(reinterpret_cast<uint8_t *>(pw)[i]);
-      }
-      msg.serialize(ctx->getUsername().c_str());
-      msg.flush(ctx->getTCPSocket());
-      DebugPrint("TCP Sent: 0x3d CREATEACOUNT\n");
-      ctx->setState(new S2C_CREATEACCOUNT2());
-      return;
+    std::string user = ctx->getUsername();
+    uint32_t *pw = ctx->getPassword1(); // single-hashed for SID_LOGONRESPONSE2
+    if (!user.empty() && pw) {
+        if (ctx->shouldCreateAccount()) {
+            ctx->setCreateAccount(false);
+            BNCSOutputStream msg(0x3d);
+            uint32_t *pw = ctx->getPassword1();
+            for (int i = 0; i < 20; i++) {
+                msg.serialize8(reinterpret_cast<uint8_t *>(pw)[i]);
+            }
+            msg.serialize(ctx->getUsername().c_str());
+            msg.flush(ctx->getTCPSocket());
+            DebugPrint("TCP Sent: 0x3d CREATEACOUNT\n");
+            ctx->setState(new S2C_CREATEACCOUNT2());
+            return;
+        }
+
+        BNCSOutputStream logon(0x3a);
+        logon.serialize32(ctx->clientToken);
+        logon.serialize32(ctx->serverToken);
+
+        // Battle.net password hashes are hashed twice using XSHA-1. First, the
+        // password is hashed by itself, then the following data is hashed again
+        // and sent to Battle.net:
+        // (UINT32) Client Token
+        // (UINT32) Server Token
+        // (UINT32)[5] First password hash
+        // The logic below is taken straight from pvpgn
+        struct {
+            pvpgn::bn_int ticks;
+            pvpgn::bn_int sessionkey;
+            pvpgn::bn_int passhash1[5];
+        } temp;
+        uint32_t passhash2[5];
+
+        pvpgn::bn_int_set(&temp.ticks, ntohl(ctx->clientToken));
+        pvpgn::bn_int_set(&temp.sessionkey, ntohl(ctx->serverToken));
+        pvpgn::hash_to_bnhash((pvpgn::t_hash const *)pw, temp.passhash1);
+        pvpgn::bnet_hash(&passhash2, sizeof(temp), &temp); /* do the double hash */
+
+        for (int i = 0; i < 20; i++) {
+            logon.serialize8(reinterpret_cast<uint8_t *>(passhash2)[i]);
+        }
+        logon.serialize(user.c_str());
+        logon.flush(ctx->getTCPSocket());
+        DebugPrint("TCP Sent: 0x3a LOGIN\n");
+
+        ctx->setState(new S2C_LOGONRESPONSE2());
     }
-
-    BNCSOutputStream logon(0x3a);
-    logon.serialize32(ctx->clientToken);
-    logon.serialize32(ctx->serverToken);
-
-    // Battle.net password hashes are hashed twice using XSHA-1. First, the
-    // password is hashed by itself, then the following data is hashed again
-    // and sent to Battle.net:
-    // (UINT32) Client Token
-    // (UINT32) Server Token
-    // (UINT32)[5] First password hash
-    // The logic below is taken straight from pvpgn
-    struct {
-      pvpgn::bn_int ticks;
-      pvpgn::bn_int sessionkey;
-      pvpgn::bn_int passhash1[5];
-    } temp;
-    uint32_t passhash2[5];
-
-    pvpgn::bn_int_set(&temp.ticks, ntohl(ctx->clientToken));
-    pvpgn::bn_int_set(&temp.sessionkey, ntohl(ctx->serverToken));
-    pvpgn::hash_to_bnhash((pvpgn::t_hash const *)pw, temp.passhash1);
-    pvpgn::bnet_hash(&passhash2, sizeof(temp), &temp); /* do the double hash */
-
-    for (int i = 0; i < 20; i++) {
-      logon.serialize8(reinterpret_cast<uint8_t *>(passhash2)[i]);
-    }
-    logon.serialize(user.c_str());
-    logon.flush(ctx->getTCPSocket());
-    DebugPrint("TCP Sent: 0x3a LOGIN\n");
-
-    ctx->setState(new S2C_LOGONRESPONSE2());
-  }
 };
 
 class S2C_SID_AUTH_CHECK : public NetworkState {
@@ -2107,10 +2107,10 @@ static int CclSetup(lua_State *l) {
                 _ctx.setFriendslist(_ctx.getFriends());
             }
         } else if (!strcmp(value, "SetGames")) {
-          if (_ctx.SetGames) {
-            delete _ctx.SetGames;
-          }
-          _ctx.SetGames = new LuaCallback(l, -1);
+            if (_ctx.SetGames) {
+                delete _ctx.SetGames;
+            }
+            _ctx.SetGames = new LuaCallback(l, -1);
         } else if (!strcmp(value, "SetChannels")) {
             if (_ctx.SetChannels) {
                 delete _ctx.SetChannels;

From 9a3624c99c8ae56fb38b2735f7d79001a92873ea Mon Sep 17 00:00:00 2001
From: Tim Felgentreff <timfelgentreff@gmail.com>
Date: Fri, 11 Dec 2020 11:14:34 +0100
Subject: [PATCH 70/70] make many shaders from libretro/glsl repo work directly

---
 src/video/shaders.cpp        |  19 +-
 src/video/shaders/cg2glsl.py | 728 +++++++++++++++++++++++++++++++++++
 2 files changed, 746 insertions(+), 1 deletion(-)
 create mode 100644 src/video/shaders/cg2glsl.py

diff --git a/src/video/shaders.cpp b/src/video/shaders.cpp
index 9c137501f..689f56c4f 100644
--- a/src/video/shaders.cpp
+++ b/src/video/shaders.cpp
@@ -24,6 +24,7 @@
  */
 
 #include "shaders.h"
+#include <stdint.h>
 
 #ifndef __APPLE__
 #include <SDL.h>
@@ -31,7 +32,7 @@
 #include <SDL_opengl_glext.h>
 
 #include <stdlib.h>
-
+#include <regex>
 #include <iostream>
 #include <fstream>
 
@@ -89,6 +90,8 @@ static const char* shaderNames[MAX_SHADERS + 1] = { NULL };
 static char shadersLoaded = -1;
 static int currentShaderIdx = 0;
 
+static std::regex invalidQuoteReplaceRegex("\"([a-zA-Z0-9 -\\.]+)\"");
+
 const char* none =
 #include "./shaders/noshader.glsl"
 ;
@@ -135,6 +138,20 @@ static GLuint compileProgramSource(std::string source) {
 	GLuint programId = 0;
 	GLuint vtxShaderId, fragShaderId;
 
+	uint32_t offset = 0;
+	std::smatch m;
+	while (std::regex_search(source.cbegin() + offset, source.cend(), m, invalidQuoteReplaceRegex)) {
+		uint32_t next_offset = offset + m.position() + m.length();
+		for (std::string::iterator it = source.begin() + offset + m.position(); it < source.begin() + next_offset; it++) {
+			if (*it == '"') {
+				source.replace(it, it + 1, " ");
+			} else if (*it == ' ') {
+				source.replace(it, it + 1, "_");
+			}
+		}
+		offset = next_offset;
+	}
+
 	vtxShaderId = compileShader((std::string("#define VERTEX\n") + source).c_str(), GL_VERTEX_SHADER);
 	fragShaderId = compileShader((std::string("#define FRAGMENT\n") + source).c_str(), GL_FRAGMENT_SHADER);
 
diff --git a/src/video/shaders/cg2glsl.py b/src/video/shaders/cg2glsl.py
new file mode 100644
index 000000000..af8205f09
--- /dev/null
+++ b/src/video/shaders/cg2glsl.py
@@ -0,0 +1,728 @@
+#!/usr/bin/env python3
+"""
+Python 3 script which converts simple RetroArch Cg shaders to modern GLSL (ES) format.
+Author: Hans-Kristian Arntzen (Themaister)
+License: Public domain
+
+N.b.: This script works on some shader files from https://github.com/libretro/common-shaders
+Other directly converted shaders can be found here: https://github.com/libretro/glsl-shaders
+"""
+
+import os
+import errno
+import subprocess
+import sys
+
+if sys.version_info < (3, 0, 0):
+    sys.stderr.write("You need python 3.0 or later to run this script\n")
+    exit(1)
+
+batch_mode = False
+
+def log(*arg):
+    if not batch_mode:
+        # FIXME: This causes a syntax error in python2, preventing the version warning from displaying.
+        print(*arg)
+
+def keep_line_if(func, lines):
+    ret = []
+    for line in filter(func, lines):
+        ret.append(line)
+    return ret
+
+def remove_comments(source_lines):
+    lines_without_comments = [line.split('//')[0] for line in source_lines]
+    return keep_line_if(lambda line: line, lines_without_comments)
+
+def defines_var(line):
+    return ('//var' in line) or ('#var' in line)
+
+def replace_by_table(source, table):
+    for orig, new in table:
+        if orig:
+            source = source.replace(orig, new)
+
+    return source
+
+def replace_global_in(source):
+    replace_table = [
+        ('IN.video_size', 'InputSize'),
+        ('IN.texture_size', 'TextureSize'),
+        ('IN.output_size', 'OutputSize'),
+        ('IN.frame_count', 'FrameCount'),
+        ('IN.frame_direction', 'FrameDirection'),
+    ]
+
+    for line in source.splitlines():
+        if defines_var(line):
+            for index, replace in enumerate(replace_table):
+                orig = line.split()[2]
+                if replace[0] == orig:
+                    replace_table[index] = (line.split(':')[2].split(' ')[1], replace_table[index][1])
+
+    log('Replace globals:', replace_table)
+
+    return replace_by_table(source, replace_table)
+
+def replace_global_vertex(source):
+    source = replace_global_in(source)
+
+    replace_table = [
+        ('attribute', 'COMPAT_ATTRIBUTE'),
+        ('varying', 'COMPAT_VARYING'),
+        ('texture2D', 'COMPAT_TEXTURE'),
+        ('POSITION', 'VertexCoord'),
+        ('TEXCOORD1', 'LUTTexCoord'),
+        ('TEXCOORD0', 'TexCoord'),
+        ('TEXCOORD', 'TexCoord'),
+        ('uniform vec4 _modelViewProj1[4];', ''),
+        ('_modelViewProj1', 'MVPMatrix'),
+        ('_IN1._mvp_matrix[0]', 'MVPMatrix[0]'),
+        ('_IN1._mvp_matrix[1]', 'MVPMatrix[1]'),
+        ('_IN1._mvp_matrix[2]', 'MVPMatrix[2]'),
+        ('_IN1._mvp_matrix[3]', 'MVPMatrix[3]'),
+
+        ('FrameCount', 'float(FrameCount)'),
+        ('FrameDirection', 'float(FrameDirection)'),
+        ('input', 'input_dummy'),  # 'input' is reserved in GLSL.
+        ('output', 'output_dummy'),  # 'output' is reserved in GLSL.
+    ]
+
+    return replace_by_table(source, replace_table)
+
+def translate_varyings(varyings, source, direction):
+    dictionary = {}
+    for varying in varyings:
+        for line in source:
+            if defines_var(line) and (varying in line) and (direction in line):
+                log('Found line for', varying + ':', line)
+                dictionary[varying] = 'VAR' + line.split(':')[0].split('.')[-1].strip()
+                break
+
+    return dictionary
+
+def no_uniform(elem):
+    banned = [
+        '_video_size',
+        '_texture_size',
+        '_output_size',
+        '_output_dummy_size',
+        '_frame_count',
+        '_frame_direction',
+        '_mvp_matrix',
+        '_vertex_coord',
+        'sampler2D'
+    ]
+
+    for ban in banned:
+        if ban in elem:
+            return False
+
+    return True
+
+def destructify_varyings(source, direction):
+    # We have to change varying structs that Cg support to single varyings for GL.
+    # Varying structs aren't supported until later versions
+    # of GLSL.
+
+    # Global structs are sometimes used to store temporary data.
+    # Don't try to remove this as it breaks compile.
+    vout_lines = []
+    for line in source:
+        if defines_var(line) and (('$vout.' in line) or ('$vin.' in line)):
+            vout_lines.append(line)
+
+    struct_types = []
+    for line in source[1:]:
+        if 'struct' in line:
+            struct_type = line.split()[1]
+            if struct_type not in struct_types:
+                struct_types.append(struct_type)
+
+    log('Struct types:', struct_types)
+
+    last_struct_decl_line = 0
+    varyings = []
+    varyings_name = []
+    # Find all varyings in structs and make them "global" varyings.
+    for struct in struct_types:
+        for i, line in enumerate(source):
+            if ('struct ' + struct) in line:
+                j = i + 1
+                while (j < len(source)) and ('};' not in source[j]):
+                    j += 1
+
+                lines = ['COMPAT_VARYING ' + string for string in source[(i + 1):j]]
+                varyings.extend(lines)
+                names = [string.split()[1].split(';')[0].strip() for string in source[(i + 1):j]]
+                varyings_name.extend(names)
+                log('Found elements in struct', struct + ':', names)
+                last_struct_decl_line = j
+
+                # Must have explicit uniform sampler2D in struct.
+                for index in range(i, j + 1):
+                    if 'sampler2D' in source[index]:
+                        source[index] = 'float _placeholder{};'.format(index)
+
+    varyings_tmp = varyings
+    varyings = []
+    variables = []
+
+    # Don't include useless varyings like IN.video_size, IN.texture_size, etc as they are not actual varyings ...
+    for i in filter(no_uniform, varyings_tmp):
+        varyings.append(i)
+
+    # Find any global variable struct that is supposed to be the output varying, and redirect all references to it to
+    # the actual varyings we just declared ...
+    # Globals only come before main() ...
+    # Make sure to only look after all struct declarations as there might be overlap.
+    for line in remove_comments(source[last_struct_decl_line:]):
+        if 'void main()' in line:
+            break
+
+        for struct in struct_types:
+            if struct in line:
+                variable = line.split()[1].split(';')[0]
+
+                # Only redirect if the struct is actually used as vertex output.
+                for vout_line in vout_lines:
+                    if variable in vout_line:
+                        log('Found struct variable for', struct + ':', variable, 'in line:', line)
+                        variables.append(variable)
+                        break
+
+    varyings_dict = translate_varyings(varyings_name, source, direction)
+    log('Varyings dict:', varyings_dict)
+
+    # Append all varyings. Keep the structs as they might be used as regular values.
+    for varying in varyings:
+        source.insert(1, varying)
+
+    log('Variables:', variables)
+    log('Varying names:', varyings_name)
+
+    # Replace struct access with global access, e.g. (_co1._c00 => _c00)
+    # Also replace mangled Cg name with 'real' name.
+    for index, _ in enumerate(source):
+        for variable in variables:
+            for varying_name in varyings_dict:
+                trans_from = variable + '.' + varying_name
+                trans_to = varyings_dict[varying_name]
+                source[index] = source[index].replace(trans_from, trans_to)
+
+    for index, _ in enumerate(source):
+        for varying_name in varyings_name:
+            if varying_name in varyings_dict:
+                source[index] = source[index].replace(varying_name, varyings_dict[varying_name])
+
+    # Replace union <struct>. Sometimes we get collision in vertex and fragment.
+    for index, line in enumerate(source):
+        for struct_type in struct_types:
+            line = line.replace('uniform ' + struct_type, struct_type)
+        source[index] = line
+
+    return source
+
+def translate(cg, translations):
+    if cg in translations:
+        return translations[cg]
+    else:
+        return cg
+
+def translate_varying(cg):
+    translations = {
+        'IN.tex_coord': 'TexCoord',
+        'IN.vertex_coord': 'VertexCoord',
+        'IN.lut_tex_coord': 'LUTTexCoord',
+        'ORIG.tex_coord': 'OrigTexCoord',
+        'FEEDBACK.tex_coord': 'FeedbackTexCoord',
+        'PREV.tex_coord': 'PrevTexCoord',
+        'PREV1.tex_coord': 'Prev1TexCoord',
+        'PREV2.tex_coord': 'Prev2TexCoord',
+        'PREV3.tex_coord': 'Prev3TexCoord',
+        'PREV4.tex_coord': 'Prev4TexCoord',
+        'PREV5.tex_coord': 'Prev5TexCoord',
+        'PREV6.tex_coord': 'Prev6TexCoord',
+        'PASS1.tex_coord': 'Pass1TexCoord',
+        'PASS2.tex_coord': 'Pass2TexCoord',
+        'PASS3.tex_coord': 'Pass3TexCoord',
+        'PASS4.tex_coord': 'Pass4TexCoord',
+        'PASS5.tex_coord': 'Pass5TexCoord',
+        'PASS6.tex_coord': 'Pass6TexCoord',
+        'PASS7.tex_coord': 'Pass7TexCoord',
+        'PASS8.tex_coord': 'Pass8TexCoord',
+        'PASSPREV2.tex_coord': 'PassPrev2TexCoord',
+        'PASSPREV3.tex_coord': 'PassPrev3TexCoord',
+        'PASSPREV4.tex_coord': 'PassPrev4TexCoord',
+        'PASSPREV5.tex_coord': 'PassPrev5TexCoord',
+        'PASSPREV6.tex_coord': 'PassPrev6TexCoord',
+        'PASSPREV7.tex_coord': 'PassPrev7TexCoord',
+        'PASSPREV8.tex_coord': 'PassPrev8TexCoord',
+    }
+
+    return translate(cg, translations)
+
+def translate_texture_size(cg):
+    translations = {
+        'ORIG.texture_size': 'OrigTextureSize',
+        'FEEDBACK.texture_size': 'FeedbackTextureSize',
+        'PREV.texture_size': 'PrevTextureSize',
+        'PREV1.texture_size': 'Prev1TextureSize',
+        'PREV2.texture_size': 'Prev2TextureSize',
+        'PREV3.texture_size': 'Prev3TextureSize',
+        'PREV4.texture_size': 'Prev4TextureSize',
+        'PREV5.texture_size': 'Prev5TextureSize',
+        'PREV6.texture_size': 'Prev6TextureSize',
+        'PASS1.texture_size': 'Pass1TextureSize',
+        'PASS2.texture_size': 'Pass2TextureSize',
+        'PASS3.texture_size': 'Pass3TextureSize',
+        'PASS4.texture_size': 'Pass4TextureSize',
+        'PASS5.texture_size': 'Pass5TextureSize',
+        'PASS6.texture_size': 'Pass6TextureSize',
+        'PASS7.texture_size': 'Pass7TextureSize',
+        'PASS8.texture_size': 'Pass8TextureSize',
+        'PASSPREV2.texture_size': 'PassPrev2TextureSize',
+        'PASSPREV3.texture_size': 'PassPrev3TextureSize',
+        'PASSPREV4.texture_size': 'PassPrev4TextureSize',
+        'PASSPREV5.texture_size': 'PassPrev5TextureSize',
+        'PASSPREV6.texture_size': 'PassPrev6TextureSize',
+        'PASSPREV7.texture_size': 'PassPrev7TextureSize',
+        'PASSPREV8.texture_size': 'PassPrev8TextureSize',
+        'ORIG.video_size': 'OrigInputSize',
+        'FEEDBACK.video_size': 'FeedbackInputSize',
+        'PREV.video_size': 'PrevInputSize',
+        'PREV1.video_size': 'Prev1InputSize',
+        'PREV2.video_size': 'Prev2InputSize',
+        'PREV3.video_size': 'Prev3InputSize',
+        'PREV4.video_size': 'Prev4InputSize',
+        'PREV5.video_size': 'Prev5InputSize',
+        'PREV6.video_size': 'Prev6InputSize',
+        'PASS1.video_size': 'Pass1InputSize',
+        'PASS2.video_size': 'Pass2InputSize',
+        'PASS3.video_size': 'Pass3InputSize',
+        'PASS4.video_size': 'Pass4InputSize',
+        'PASS5.video_size': 'Pass5InputSize',
+        'PASS6.video_size': 'Pass6InputSize',
+        'PASS7.video_size': 'Pass7InputSize',
+        'PASS8.video_size': 'Pass8InputSize',
+        'PASSPREV2.video_size': 'PassPrev2InputSize',
+        'PASSPREV3.video_size': 'PassPrev3InputSize',
+        'PASSPREV4.video_size': 'PassPrev4InputSize',
+        'PASSPREV5.video_size': 'PassPrev5InputSize',
+        'PASSPREV6.video_size': 'PassPrev6InputSize',
+        'PASSPREV7.video_size': 'PassPrev7InputSize',
+        'PASSPREV8.video_size': 'PassPrev8InputSize',
+    }
+
+    return translate(cg, translations)
+
+def replace_varyings(source):
+    ret = []
+    translations = []
+    attribs = []
+    uniforms = []
+    for index, line in enumerate(source):
+        if defines_var(line):
+            func = translate_texture_size
+            collection = uniforms
+            if '$vin.' in line:
+                func = translate_varying
+                collection = attribs
+            orig = line.split()[2]
+            translated = func(orig)
+            if translated != orig and translated not in collection:
+                cg_var = line.split(':')[2].split(' ')[1]
+                if cg_var:
+                    translations.append((cg_var, translated))
+                    collection.append(translated)
+
+    for index, line in enumerate(source):
+        if 'void main()' in line:
+            for attrib in attribs:
+                if attrib == 'VertexCoord':
+                    source.insert(index, 'COMPAT_ATTRIBUTE vec4 ' + attrib + ';')
+                else:
+                    source.insert(index, 'COMPAT_ATTRIBUTE vec2 ' + attrib + ';')
+            for uniform in uniforms:
+                source.insert(index, 'uniform COMPAT_PRECISION vec2 ' + uniform + ';')
+            break
+
+    for line in source:
+        for trans in translations:
+            line = line.replace(trans[0], trans[1])
+        ret.append(line)
+
+    return ret
+
+def fix_samplers(log_prefix, ref_index, source):
+    translations = []
+    added_samplers = []
+    translated_samplers = []
+    uniforms = []
+    struct_texunit0 = False  # If True, we have to append uniform sampler2D Texture manually ...
+    for line in source:
+        if ('TEXUNIT0' in line) and ('semantic' not in line):
+            main_sampler = (line.split(':')[2].split(' ')[1], 'Texture')
+            if main_sampler[0]:
+                translations.append(main_sampler)
+                log(log_prefix, 'Sampler:', main_sampler[0], '->', main_sampler[1])
+                struct_texunit0 = '.' in main_sampler[0]
+        elif ('//var sampler2D' in line) or ('#var sampler2D' in line):
+            cg_texture = line.split()[2]
+            translated = translate_texture(cg_texture)
+            orig_name = translated
+            new_name = line.split(':')[2].split(' ')[1]
+            log(log_prefix, 'Sampler:', new_name, '->', orig_name)
+            if new_name:
+                if translated != cg_texture and translated not in translated_samplers:
+                    translated_samplers.append(translated)
+                    added_samplers.append('uniform sampler2D ' + translated + ';')
+                translations.append((new_name, orig_name))
+        elif defines_var(line):
+            orig = line.split()[2]
+            translated = translate_texture_size(orig)
+            if translated != orig and translated not in uniforms:
+                cg_uniform = line.split(':')[2].split(' ')[1]
+                if cg_uniform:
+                    translations.append((cg_uniform, translated))
+                    uniforms.append(translated)
+
+    for sampler in added_samplers:
+        source.insert(ref_index, sampler)
+    if struct_texunit0:
+        source.insert(ref_index, 'uniform sampler2D Texture;')
+    for uniform in uniforms:
+        source.insert(ref_index, 'uniform COMPAT_PRECISION vec2 ' + uniform + ';')
+    for index, line in enumerate(source):
+        for orig, new in translations:
+            log(log_prefix, 'Translation:', orig, '->', new)
+            source[index] = source[index].replace(orig, new)
+
+    return source
+
+def hack_source_vertex(source):
+    ref_index = 0
+    for index, line in enumerate(source):
+        if 'void main()' in line:
+            source.insert(index, 'uniform COMPAT_PRECISION vec2 InputSize;')
+            source.insert(index, 'uniform COMPAT_PRECISION vec2 TextureSize;')
+            source.insert(index, 'uniform COMPAT_PRECISION vec2 OutputSize;')
+            source.insert(index, 'uniform int FrameCount;')
+            source.insert(index, 'uniform int FrameDirection;')
+            source.insert(index, 'uniform mat4 MVPMatrix;')
+
+            ref_index = index
+            break
+
+    # Fix samplers in vertex shader (supported by GLSL).
+    source = fix_samplers('Vertex:', ref_index, source)
+    source = destructify_varyings(source, '$vout.')
+    source = replace_varyings(source)
+
+    return source
+
+def replace_global_fragment(source):
+    source = replace_global_in(source)
+
+    replace_table = [
+        ('varying', 'COMPAT_VARYING'),
+        ('texture2D', 'COMPAT_TEXTURE'),
+        ('FrameCount', 'float(FrameCount)'),
+        ('FrameDirection', 'float(FrameDirection)'),
+        ('input', 'input_dummy'),
+        ('output', 'output_dummy'),  # 'output' is reserved in GLSL.
+        ('gl_FragColor', 'FragColor'),
+    ]
+
+    return replace_by_table(source, replace_table)
+
+def translate_texture(cg):
+    translations = {
+        'ORIG.texture': 'OrigTexture',
+        'FEEDBACK.texture': 'FeedbackTexture',
+        'PREV.texture': 'PrevTexture',
+        'PREV1.texture': 'Prev1Texture',
+        'PREV2.texture': 'Prev2Texture',
+        'PREV3.texture': 'Prev3Texture',
+        'PREV4.texture': 'Prev4Texture',
+        'PREV5.texture': 'Prev5Texture',
+        'PREV6.texture': 'Prev6Texture',
+        'PASS1.texture': 'Pass1Texture',
+        'PASS2.texture': 'Pass2Texture',
+        'PASS3.texture': 'Pass3Texture',
+        'PASS4.texture': 'Pass4Texture',
+        'PASS5.texture': 'Pass5Texture',
+        'PASS6.texture': 'Pass6Texture',
+        'PASS7.texture': 'Pass7Texture',
+        'PASS8.texture': 'Pass8Texture',
+        'PASSPREV2.texture': 'PassPrev2Texture',
+        'PASSPREV3.texture': 'PassPrev3Texture',
+        'PASSPREV4.texture': 'PassPrev4Texture',
+        'PASSPREV5.texture': 'PassPrev5Texture',
+        'PASSPREV6.texture': 'PassPrev6Texture',
+        'PASSPREV7.texture': 'PassPrev7Texture',
+        'PASSPREV8.texture': 'PassPrev8Texture',
+    }
+
+    return translate(cg, translations)
+
+def hack_source_fragment(source):
+    ref_index = 0
+    for index, line in enumerate(source):
+        if 'void main()' in line:
+            source.insert(index, 'uniform COMPAT_PRECISION vec2 InputSize;')
+            source.insert(index, 'uniform COMPAT_PRECISION vec2 TextureSize;')
+            source.insert(index, 'uniform COMPAT_PRECISION vec2 OutputSize;')
+            source.insert(index, 'uniform int FrameCount;')
+            source.insert(index, 'uniform int FrameDirection;')
+            ref_index = index
+            break
+
+    source = fix_samplers('Fragment:', ref_index, source)
+    source = destructify_varyings(source, '$vin.')
+
+    return source
+
+def validate_shader(source, target):
+    log('Shader:')
+    log('===')
+    log(source)
+    log('===')
+
+    command = ['cgc', '-noentry', '-ogles']
+    p = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout_ret, stderr_ret = p.communicate(source.encode())
+
+    log('CGC:', stderr_ret.decode())
+
+    return p.returncode == 0
+
+def preprocess_vertex(source_data):
+    input_data = source_data.splitlines()
+    ret = []
+    for line in input_data:
+        if ('uniform' in line) and (('float4x4' in line) or ('half4x4' in line)):
+            ret.append('#pragma pack_matrix(column_major)\n')
+            ret.append(line)
+            ret.append('#pragma pack_matrix(row_major)\n')
+        else:
+            ret.append(line)
+    return '\n'.join(ret)
+
+def convert(source, dest):
+    # Have to preprocess first to resolve #includes so we can hack potential vertex shaders.
+    inc_dir = os.path.split(source)[0]
+    vert_cmd_preprocess = ['cgc', '-E', '-I', '.' if inc_dir == '' else inc_dir, source]
+    p = subprocess.Popen(vert_cmd_preprocess, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
+    source_data, stderr_ret = p.communicate()
+    log(stderr_ret.decode())
+
+    if p.returncode != 0:
+        log('Vertex preprocessing failed ...')
+
+    source_data = preprocess_vertex(source_data.decode())
+
+    vert_cmd = ['cgc', '-profile', 'glesv', '-entry', 'main_vertex', '-quiet']
+    p = subprocess.Popen(vert_cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
+    vertex_source, stderr_ret = p.communicate(input=source_data.encode())
+    log(stderr_ret.decode())
+    vertex_source = vertex_source.decode()
+
+    if p.returncode != 0:
+        log('Vertex compilation failed ...')
+        return 1
+
+    frag_cmd = ['cgc', '-profile', 'glesf', '-entry', 'main_fragment', '-quiet']
+    p = subprocess.Popen(frag_cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
+    fragment_source, stderr_ret = p.communicate(input=source_data.encode())
+    log(stderr_ret.decode())
+    fragment_source = fragment_source.decode()
+
+    if p.returncode != 0:
+        log('Vertex compilation failed ...')
+        return 1
+
+    vertex_source = replace_global_vertex(vertex_source)
+    fragment_source = replace_global_fragment(fragment_source)
+
+    vertex_source = vertex_source.splitlines()
+    fragment_source = fragment_source.splitlines()
+
+    # Cg think we're using row-major matrices, but we're using column major.
+    # Also, Cg tends to compile matrix multiplications as dot products in GLSL.
+    # Hack in a fix for this.
+    log('Hacking vertex')
+    vertex_source = hack_source_vertex(vertex_source)
+    log('Hacking fragment')
+    fragment_source = hack_source_fragment(fragment_source)
+
+    # We compile to GLES, but we really just want modern GL ...
+    vertex_source = keep_line_if(lambda line: 'precision' not in line, vertex_source)
+    fragment_source = keep_line_if(lambda line: 'precision' not in line, fragment_source)
+
+    # Kill all comments. Cg adds lots of useless comments.
+    # Remove first line. It contains the name of the cg program.
+    vertex_source = remove_comments(vertex_source[1:])
+    fragment_source = remove_comments(fragment_source[1:])
+
+    vert_hacks = []
+    vert_hacks.append('''
+#if __VERSION__ >= 130
+#define COMPAT_VARYING out
+#define COMPAT_ATTRIBUTE in
+#define COMPAT_TEXTURE texture
+#else
+#define COMPAT_VARYING varying
+#define COMPAT_ATTRIBUTE attribute
+#define COMPAT_TEXTURE texture2D
+#endif
+
+#ifdef GL_ES
+#define COMPAT_PRECISION mediump
+#else
+#define COMPAT_PRECISION
+#endif''')
+
+    out_vertex = '\n'.join(vert_hacks + vertex_source)
+
+    frag_hacks = []
+    frag_hacks.append('''
+#if __VERSION__ >= 130
+#define COMPAT_VARYING in
+#define COMPAT_TEXTURE texture
+out vec4 FragColor;
+#else
+#define COMPAT_VARYING varying
+#define FragColor gl_FragColor
+#define COMPAT_TEXTURE texture2D
+#endif
+
+#ifdef GL_ES
+#ifdef GL_FRAGMENT_PRECISION_HIGH
+precision highp float;
+#else
+precision mediump float;
+#endif
+#define COMPAT_PRECISION mediump
+#else
+#define COMPAT_PRECISION
+#endif''')
+
+    out_fragment = '\n'.join(frag_hacks + fragment_source)
+
+    if not validate_shader(out_vertex, 'glesv'):
+        log('Vertex shader does not compile ...')
+        return 1
+
+    if not validate_shader(out_fragment, 'glesf'):
+        log('Fragment shader does not compile ...')
+        return 1
+
+    with open(dest, 'w') as f:
+        f.write('// GLSL shader autogenerated by cg2glsl.py.\n')
+        f.write('#if defined(VERTEX)\n')
+        f.write(out_vertex)
+        f.write('\n')
+        f.write('#elif defined(FRAGMENT)\n')
+        f.write(out_fragment)
+        f.write('\n')
+        f.write('#endif\n')
+    return 0
+
+def convert_cgp(source, dest):
+    string = ''
+    with open(source, 'r') as f:
+        string = f.read().replace('.cg', '.glsl')
+
+    open(dest, 'w').write(string)
+
+def path_ext(path):
+    _, ext = os.path.splitext(path)
+    return ext
+
+def convert_path(source, source_dir, dest_dir, conv):
+    index = 0 if source_dir[-1] == '/' else 1
+    return os.path.join(dest_dir, source.replace(source_dir, '')[index:]).replace(conv[0], conv[1])
+
+def main():
+    if len(sys.argv) != 3:
+        print('Usage: {} prog.cg(p) prog.glsl(p)'.format(sys.argv[0]))
+        print('Batch mode usage: {} cg-dir out-xml-shader-dir'.format(sys.argv[0]))
+        print('Requires Python 3 and cgc (nvidia-cg-toolkit) 3.1.')
+        return 1
+
+    if os.path.isdir(sys.argv[1]):
+        global batch_mode
+        batch_mode = True
+        try:
+            os.makedirs(sys.argv[2])
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+
+        failed_cnt = 0
+        success_cnt = 0
+        failed_files = []
+        for dirname, _, filenames in os.walk(sys.argv[1]):
+            for source in filter(lambda path: path_ext(path) == '.cg', [os.path.join(dirname, filename) for filename in filenames]):
+                dest = convert_path(source, sys.argv[1], sys.argv[2], ('.cg', '.glsl'))
+                dirpath = os.path.split(dest)[0]
+                print('Dirpath:', dirpath)
+                if not os.path.isdir(dirpath):
+                    try:
+                        os.makedirs(dirpath)
+                    except OSError as e:
+                        if e.errno != errno.EEXIST:
+                            raise
+
+                try:
+                    ret = convert(source, dest)
+                    print(source, '->', dest, '...', 'succeeded!' if ret == 0 else 'failed!')
+
+                    if ret == 0:
+                        success_cnt += 1
+                    else:
+                        failed_cnt += 1
+                        failed_files.append(source)
+                except Exception as e:
+                    print(e)
+                    failed_files.append(source)
+                    failed_cnt += 1
+
+            for source in filter(lambda path: path_ext(path) == '.cgp', [os.path.join(dirname, filename) for filename in filenames]):
+                dest = convert_path(source, sys.argv[1], sys.argv[2], ('.cgp', '.glslp'))
+                dirpath = os.path.split(dest)[0]
+                print('Dirpath:', dirpath)
+                if not os.path.isdir(dirpath):
+                    try:
+                        os.makedirs(dirpath)
+                    except OSError as e:
+                        if e.errno != errno.EEXIST:
+                            raise
+
+                try:
+                    convert_cgp(source, dest)
+                    success_cnt += 1
+                except Exception as e:
+                    print(e)
+                    failed_files.append(source)
+                    failed_cnt += 1
+
+        print(success_cnt, 'shaders converted successfully.')
+        print(failed_cnt, 'shaders failed.')
+        if failed_cnt > 0:
+            print('Failed shaders:')
+            for path in failed_files:
+                print(path)
+
+    else:
+        source = sys.argv[1]
+        dest = sys.argv[2]
+
+        if path_ext(source) == '.cgp':
+            sys.exit(convert_cgp(source, dest))
+        else:
+            sys.exit(convert(source, dest))
+
+if __name__ == '__main__':
+    sys.exit(main())