VS_CODE/Qwixx/network-stuff/server.c

#include "server.h"

#ifdef _WIN32
/*  Msft documentation has this to say: this is required to "initiates use
 *  of the Winsock DLL by a process."  They use technical words like initiates,
 *  process, and a proper noun.  In programming parlance this makes no sense,
 *  so it boils down to: run this function or else our socket API will not work.
 *  Bonkers.  Get people to include it in their code, and now their code is
 *  proprietary windows software.
 */
int msftVendorLockInCode(WSADATA *w) {
    if(WSAStartup(MAKEWORD(2,0), w)) {
        fprintf(stderr, "WSAStartup.\n"); return 1;
    } else return 0;
}

/*  Msft does not implement a close(int socketfd) routine like literally
 *  every other operating system.  Instead they provide closesocket().  Lol.
 *  Lets ignore the Msft marketing team:
 */
int close(int fd) {
    return closesocket(fd);
} /* [?] whew, that was hard */

/*  Msft does not have a strerror routine and while they copied the Berkley
 *  Socket API they thought that they could improve on it by renaming and
 *  removing API constants.  Later they introduced a compatibility header
 *  errno.h but quickly started to erode the compatibility features as they
 *  added features to their Visual Studio product line so even errno.h is
 *  useless.  Along with redefining the errno global variable and changing
 *  it to a function (WSAGetLastError()) we still need a way to print human
 *  readable error messages.
 *
 *  [?] these messages are way too verbose
 */
char * strWinsock(int e) {
    static char buf[256];
    // apparently Msft does not clear buffers
    for(int i = sizeof(buf); i--;) buf[i] = '\0';

    FormatMessage (
        /* flags */ FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
        /* lpsource */ NULL,
        /* message id */ e,
        /* language id */ MAKELANGID (LANG_NEUTRAL, SUBLANG_DEFAULT),
        /* output buffer */ buf,
        /* size */ sizeof (buf),
        /* va_list args */ NULL
    );

    if ('\0' == buf[0]) sprintf(buf, "%d-UnknownError", e);
    return buf;
}
#endif

// nullify all the "okay" reasons for failure, otherwise signal an abort
int acceptOkay(int e) {
    switch(errno) {
        #if !defined (_WIN32) && !defined (__linux__)
        // linux assigns EAGAIN as EWOULDBLOCK while Msft does not support
        case EAGAIN:       /* same as EWOULDBLOCK                */
        #endif
        case EWOULDBLOCK: 
            // in our use case usually a multiplexer timeout
            #ifdef PRINT_MULTIPLEXER_TIMEOUT
            fprintf(stderr, "probably multiplexer timeout: %s\n", strerror(errno));
            #endif
            return DONE_FOR_NOW;
        #ifndef _WIN32
        // Msft winsock does not support these error conditions
        case EPROTO:       /* just print "client" and errno -.   */
        case ENOMEM:       /* same as ENOBUFS                |   */
        case ENFILE:       /* see EMFILE, but system wide    v   */
        #endif
        case ECONNABORTED: /* just print "client: errno     -.   */
        case EINTR:        /*   (these are all okay)         |   */
        case EMFILE:       /* per-process limit on open fd   |   */
        case ENOBUFS:      /* usually socket buffer limits   |   */
        case EOPNOTSUPP:   /*                                V   */
            fprintf(stderr, "client: %s\n", strerror(errno));
            return 1;
        #ifdef __linux__
        // linux also suppots firewall rule logging
        case EPERM:        
            fprintf(stderr, "firewall: %s\n", strerror(errno));
            return 1;
        // in addition linux can support additional logging
        case ENOSR:
        case ESOCKTNOSUPPORT:
        case EPROTONOSUPPORT:
        case ETIMEDOUT:
        //case ERESTARTSYS:
            fprintf(stderr, "additional: %s\n", strerror(errno));
            return 1;
        #endif
        // FATAL error numbers:
        case EBADF:        /* socket is a bad file descriptor    */
        case ENOTSOCK:     /* socket is not a file descriptor    */
        case EINVAL:       /* not listening, or invalid addrLen  */
        case EFAULT:       /* prevented access violation (addr)  */
        default:
            fprintf(stderr, "FATAL [%d] accept: %s\n", errno, strerror(errno));
            return 0;
    }
}

int setNonBlocking(int sockfd) {
    #ifdef _WIN32
    u_long m = 1;
    return ioctlsocket(sockfd, FIONBIO, &m);
    #else
    return fcntl(sockfd, F_SETFL, O_NONBLOCK);
    #endif
}

/* [!] allows data loss
 *     once close is called the OS will immediately delete everything
 *     in the outgoing buffer, eg: the timeout message will not be
 *     sent.  In addition the client might send a request to this server
 *     and the OS will most likely drop the packet causing the client
 *     to hang, waiting on a response
 *     
 *  request the OS to send the client tcp stack a request to reset
 *  connection (causes RST TCP flag to be set) as soon as close is
 *  called on the socket
 */
__attribute__((always_inline)) static
void setLinger(int sockfd) {
    struct linger off = { 1, 0 };
    #ifdef _WIN32
    // Msft changed the type from void pointer to character pointer
    #define optval (char *) &off
    #else
    #define optval          &off
    #endif
    if(0 > setsockopt(sockfd, SOL_SOCKET, SO_LINGER, optval, sizeof(off))) {
        fprintf(stderr, "failed to set linger: %s\n", strerror(errno));
    }
}

/*  Determine if more than allowed number of milliseconds
 *  have occurred since t
 *
 *  [!] maximum of 2147 milliseconds
 */
__attribute__((always_inline)) static
uint32_t msRemaining(struct timespec t, uint32_t allowed) {
    struct timespec now;
    int32_t seconds, nanoSeconds;
    // nanoSecond uses signed 32-bit integer (max: 2,147,483,647)
    if(2147 < allowed) {
        allowed = 2147;
        fprintf(stderr, "[!] remaining max is: %dms\n", 2147);
    }
    // convert milliseconds allowed to nanoseconds allowed
    allowed *= 1.0e6;
    // get the current time
    clock_gettime(CLOCK_REALTIME, &now);
    seconds = now.tv_sec - t.tv_sec;
    if(0 > (nanoSeconds = now.tv_nsec - t.tv_nsec)) {
        // with negative nanoseconds, there will be at least one second
        nanoSeconds += 1.0e9;
        if(seconds > 0) seconds--; else { fprintf(stderr, "ouch\n"); return 0; }
    }
    #ifdef DEBUG_REMAINING
    fprintf(stderr, "seconds %d\n", seconds);
    fprintf(stderr, "nanoseconds %d\n", nanoSeconds);
    #endif
    // compare
    if(nanoSeconds > allowed) return 0;
    /*  TODO: calculate the remaining milliseconds
     *  [!] I am failing to write this code.  What I want is to combine seconds
     *      and nanoseconds but due to the size limitations for integer
     *      datatypes and the split representation of time_t I am having a
     *      difficult time.
     *  [?] What I have so far: I know this function cannot perform calculations
     *      on time deltas greater than 2 seconds, and the nanosecond resolution
     *      can cause a fractional second to look like +/- second, so I assume
     *      anything over 4 seconds is out.  I then throw a very large integer
     *      into the ring to absorb the 1-4 seconds and let the compiler figure
     *      out the comparison.
     */
    if(seconds > 4) return 0;
    uint64_t humph = nanoSeconds;
    if(seconds > 0) while(seconds--) humph += 1.0e9;
    if(humph > allowed) return 0;
    allowed -= humph;
    return allowed / 1.0e6;
}

/*  What a hassle.  mingw32 deprecated usleep, Msft has a crazy
 *  interface that you have to spend like ten years filling out
 *  This is an inaccurate sleep routine.
 */
void spookySleep(uint32_t t) {
    #ifdef _WIN32
    Sleep(t);
    #else
    usleep(t * 1000);
    #endif
}

/*  sets up either select or poll and waits up to specified milliseconds for
 *  operating system to wake up process when an event occurs
 */
int socketReady(struct client pool[], int len, uint32_t ms) {
    int c, count = 0;
    #ifdef MULTIPLEX_USE_SELECT
    #define addToSet(x, y) FD_SET(pool[x].fd, &plexfds)
    #define isSet(x, y) FD_ISSET(pool[x].fd, &plexfds)
    #define waitfor(x) select(x, &plexfds, NULL, NULL, &tv)
    fd_set plexfds;
    struct timeval tv;
    FD_ZERO(&plexfds);
    tv.tv_sec = 0;
    tv.tv_usec = ms * 1000;
    #else
    #define addToSet(x, y) \
        do { plexfds[y].fd = pool[x].fd; plexfds[y].events = POLLIN; } while(0)
    #define isSet(x, y) (plexfds[y].revents & POLLIN)
    #define waitfor(x) poll(plexfds, x, ms)
    struct pollfd plexfds[POOL_SIZE];
    #endif
    // restrict len to POOL_SIZE, allows a fixed memory footprint for pollfd
    if(POOL_SIZE < len) {
        fprintf(stderr, "requested pool size %d limited to %d, ", len, POOL_SIZE);
        len = POOL_SIZE;
    }
    // add all sockets to the file descriptor set
    for(int i = 0; i < len; i++) {
        // skip pool entries without client socket file descriptors
        if(0 > pool[i].fd) continue;
        // ignore sockets that the client has already closed
        if(CLIENT_IS_CLOSED & pool[i].flags) continue;
        addToSet(i, count);
        #ifdef PRINT_SOCKET_ADD_TO_SET
        if(/* whatever server socket descriptor */ 3 != pool[0].fd)
        fprintf(stderr, "add socket descriptor %d to watch list\n", pool[i].fd);
        #endif
        count++;
    }
    // go to sleep, wake on OS interrupt (usually timeout or socket activity)
    struct timespec t;
    clock_gettime(CLOCK_REALTIME, &t);
    if(0 == (c = waitfor(count))) {
        #ifdef PRINT_NO_SOCKETS_READY
        /*  try to suppress the server listening socket --
         *  server socket is usually 3 when stdin, stdout, stderr are left open
         *                                    0      1       2
         */
        if(/* whatever server socket descriptor */ 3 != pool[0].fd)
        fprintf(stderr, "no socket descritprs ready: %d watched, %dms elapsed\n",
                count, 1000 - msRemaining(t, 1000));
        #endif
        return DONE_FOR_NOW;
    } else if(-1 == c) {
        // print whatever select/poll error case was and return error
        fprintf(stderr, "socketReady error: %s\n", strerror(errno));
        return /* error */ 0;
    } // otherwise...
    count = 0;
    // set flag for ready sockets
    for(int i = 0; i < len; i++) {
        // skip pool entries without client socket file descriptors
        if(0 > pool[i].fd) continue;
        // skip sockets that the client has already closed
        if(CLIENT_IS_CLOSED & pool[i].flags) continue;
        // either set or unset the READ_READY status bits
        if(isSet(i, count)) pool[i].status |= READ_READY;
        else pool[i].status &= ~READ_READY;
        count++;
    }
    // return number of clients that have data, as a negative number
    #ifdef PRINT_HOW_MANY_SOCKETS_READY
    fprintf(stderr, "there are %d socket descriptors ready\n", c);
    #endif
    return -c;
}

// convert characters in a buffer to only visible printable characters
void convertToPrintable(char * s, uint32_t len) {
    char * p = s;
    if(len > FIM_BUFFER_LEN) len = FIM_BUFFER_LEN;
    for(uint32_t i = 0; i < len; i++) {
        if((' ' < *p) && ('z' >= *p)) /* do nothing */ ;
        else if('\0' == *p) break;
        else *p = '.';
        p++;
    }
}

/*  Demo callback function:
 *
 *  This is an example definition of an example prototype that could be
 *  passed to the faseItMetal function.  Basically it is looking for a
 *  matching prototype: "char * someFunc(char * s, int len)"
 *
 *  The function returns a pointer, accepts a pointer and a length.
 *  both pointers are to a character datatype.  Users should know that the
 *  data pointed to will only be read/written up to a limited length (see: 
 *  faceItMetal FIM_BUFFER_LEN).
 *
 *  The len that is passed to the callback is how many bytes the client has
 *  currently sent to the server.
 *
 *  The limit is set by the callback to tell faceItMetal how many characters
 *  from the pointer it returns to send to the client.  faceItMetal may send
 *  fewer characters, as mentioned previously (see: faceItMetal FIM_BUFFER_LEN)
 *
 *  Setting this limit to zero means that the callback should be called again
 *  later with potentially a higher len, assuming the client has sent more
 *  data.  In this case the returned pointer is ignored.
 */
char * alwaysGood(char *s, int len, int *limit) {
    char *out = "Hola!";
    // set how many characters to send to client (hint: see CALLBACK_ERROR_CODES)
    *limit = strlen(out);
    // print what data the client sent
    s[len -1] = '\0';
    fprintf(stderr, "alwaysGood: client sent [%s]\n", s);
    return out;
}

// see notes from Demo callback function: alwaysGood
char * alwaysBad(char *s, int len, int *limit) {
    char *out = "";
    // ask to disconnect client with invalid (hint: see CALLBACK_ERROR_CODES)
    *limit = INVALID_DATA;
    // print what data the client sent
    s[len -1] = '\0';
    fprintf(stderr, "alwaysBad: client sent [%s]\n", s);
    return out;
}

// see notes from Demo callback function: alwaysGood
char * alwaysNotEnough(char *s, int len, int *limit) {
    char *out = "";
    // ask to disconnect client with invalid (hint: see CALLBACK_ERROR_CODES)
    *limit = SEND_MORE;
    // print what data the client sent
    s[len -1] = '\0';
    fprintf(stderr, "alwaysNotEnough: client sent [%s]\n", s);
    return out;
}

__attribute__((always_inline)) static
int waitForClients(uint32_t ms, struct client pool[]) {
    struct sockaddr_storage address;
    struct timespec spec;
    socklen_t addrSize;
    int c, count = 0;
    // store the time before waiting for new clients (ACCEPT_LISTEN_TIME)
    clock_gettime(CLOCK_REALTIME, &spec);
    /*  wait for OS wake-up (okay to ignore invalid events, see: acceptOkay)
     *
     *  [?] rather than check all possible fatal errors in the sleep routine
     *      we will defer to accept (called in acceptOkay).
     *      Check for the EAGAIN and EWOULDBLOCK conditions and short on
     *      either of those (see: DONE_FOR_NOW)
     */
    if(DONE_FOR_NOW == (c = socketReady(pool, POOL_SIZE, ms))) {
        #ifdef USE_CLIENT_POOL
        prepayEstimates(pool, POOL_SIZE, spec);
        #endif
        return DONE_FOR_NOW;
    }
    // how many clients can be added to the pool
    for(int i = POOL_SIZE; i--;) if(-1 == pool[i].fd) count++;
    // accept any inbound connections, make a socket - short on error
    if(READ_READY & pool[0].status) while(count) {
        // reset the address length parameter (accept may modify it)
        addrSize = sizeof(address);
        // pull the next connection request from the listening socket queue
        if(0 > (c = accept(pool[0].fd, (struct sockaddr *) &address, &addrSize))) {
            // normal case for break is when errno is EWOULDBLOCK or EAGAIN
            if(DONE_FOR_NOW == (c = acceptOkay(errno))) break;
            // abort if there is a problem with the listening socket
            if(c) {
                fprintf(stderr, "unusual accept, skipping\n");
                continue;
            } else return /* fatal */ 0;
        } //else
        // reconfigure the client for non-blocking IO mode
        if(0 > setNonBlocking(c)) {
            fprintf(stderr, "problem setting mode: %s\n", strerror(errno));
            close(c); continue;
        }
        // add client to pool (record their file descriptor and connection time
        for(int i = /* skip listen socket */ 1; i < POOL_SIZE; i++) {
            // find an unused space in the client pool
            if(-1 < pool[i].fd) continue;
            #ifdef PRINT_CLIENT_ACCEPTED
            fprintf(stderr, "new client\n");
            #endif
            pool[i].fd = c;
            pool[i].address = address;
            pool[i].addrSize = addrSize;
            pool[i].t = spec;
            pool[i].status = 0;
            pool[i].flags = 0;
            pool[i].ms = PER_CLIENT_TIMEOUT_MS;
            // reset buffer counter
            pool[i].count = 0;
            // clear out a buffer to store user data
            for(uint32_t j = FIM_BUFFER_LEN; j--;) pool[i].buf[j] = '\0';
            // decrease the amount of space in the pool
            count--;
            break;
        }
    }
    // remove the READ_READY flag
    pool[0].status &= ~READ_READY;
    #ifdef USE_CLIENT_POOL
    prepayEstimates(pool, POOL_SIZE, spec);
    #endif
    return /* okay */ 1;
}

/*  client socket has some pending data that has not been retrieved yet
 *  collect it from the OS' buffers and then test if the data is complete enough
 *  to send a response to the client.
 *
 *  returns either a negative code or how many bytes were sent to the client
 *  
 *
 */
__attribute__((always_inline)) static
int replyOnValid(struct client *c, char * (*f)(char *s, int len, int *limit)) {
    int d, e;
    char *res, response[FIM_BUFFER_LEN];
    // canned messages to send to clients
    static char * invalid = INVALID_TEXT;
    static char * error = SERVER_ERROR_TEXT;
    // skip unused spaces in pool, and clients without data to be processed
    if((0 > c->fd) || (0 == (READ_READY & c->status))) return /* skip */ -3;
    // also skip pooled clients that have closing type flags
    if(CLIENT_CLOSING & c->flags) return /* skip */ -3;
    do { // fill buffer, then flush (use response as overflow)
        if(0 < (e = FIM_BUFFER_LEN - c->count)) {
            if(0 < (d = recv(c->fd, c->buf + c->count, e, 0))) c->count += d;
        } else d = recv(c->fd, response, FIM_BUFFER_LEN, 0);
    } while(0 < d);
    // when read returns zero, client has already disconnected
    if(0 == d) {
        #ifdef PRINT_CLIENT_DISCONNECT
        fprintf(stderr, "client has disconnected\n");
        #endif
        c->flags |= CLIENT_IS_CLOSED;
        return /* it is okay */ -1;
    } else if (0 > d) {
        /*  usually on the first wake-up but sometimes a bit later, we will
         *  eventually consume everything a client wants to send and the client
         *  will not say anything until we respond, when this happens the
         *  recv function will continuously set errno to EWOULDBLOCK
         */
        //fprintf(stderr, "recv(%d): %s\n", errno, strerror(errno));
    }
    // skip callback if the server already decided to close the connection
    if(SERVER_WANTS_CLOSE & c->flags) return /* it is okay */ -1;
    // swap last character in buffer with a null terminator for safety
    c->buf[FIM_BUFFER_LEN - 1] = '\0';
    #ifdef PRINT_CLIENT_DATA
    fprintf(stderr, "client data: %s\n", c->buf);
    #endif
    {   e = SERVER_ERROR;
        res = f(c->buf, c->count > FIM_BUFFER_LEN ?FIM_BUFFER_LEN :c->count, &e);
    }
    // if the limit is negative treat like a CALLBACK_ERROR_CODE
    if(0 > e) switch(e) {
        case SEND_MORE: /* partial request */ 
            // terminate if client filled our buffer unusable stuff
            if(FIM_BUFFER_LEN == c->count) {
                fprintf(stderr, "thanks for all the fish\n");
                break;
            } else return /* partial request */ -2;
        // these error codes cause client disconnect (use break)
        case INVALID_DATA:
            fprintf(stderr, "callback: invalid data\n");
            sendResponse(c, invalid, strlen(invalid), INVALID_MSG);
            break;
        case DROP_CLIENT:
            fprintf(stderr, "callback: drop client\n");
            c->flags = CLOSE_IMMEDIATELY;
            setLinger(c->fd);
            break;
        case SERVER_ERROR:
            fprintf(stderr, "callback: server error\n");
            sendResponse(c, error, strlen(error), SERVER_ERROR_MSG);
            break;
        default:
            fprintf(stderr, "unknown CALLBACK_ERROR_CODE [%d]\n", e);
    } if(0 > e) { c->flags |= SERVER_WANTS_CLOSE; return /* error */ -1; }
    // set the limit for how much data should be copied
    d = e < FIM_BUFFER_LEN ?e :FIM_BUFFER_LEN;
    // copy data into the response buffer
    for(uint32_t i = 0; i < d; i++) response[i] = *res++;
    // client has sent a valid request, send them the response
    if(d > (e = sendResponse(c, response, d, RESPONSE_MSG))) {
        if(0 > e); else fprintf(stderr, "truncated: %d of %d\n", e, d);
    }
    // finished with this client (server wants to close their socket)
    c->flags |= SERVER_WANTS_CLOSE;
    return d;
}

/*  Note about send:
 *  when writing to a pipe that the remote end has broken, we receive a
 *  -1 from send and errno is set to EPIPE, but before that happens the
 *  OS will send the application the SIGPIPE signal.  Handle sigpipe, record
 *  the send return value (on success number of bytes sent, otherwise -1
 *  and errno is set)
 */
int sendResponse(struct client *c, char *buf, uint32_t len, sendReason reason) {
    int d;
    if(WRITE_CLOSED & c->status) {
        fprintf(stderr, "cannot send, we already closed our end\n");
        return -1;
    }
    if(-1 == (d = send(c->fd, buf, len, 0))) {
        #ifdef _WIN32
        /*  Msft winsock api does not implement anything to handle a broken
         *  pipe, instead they return either zero for normal disconnect and
         *  -1 for error.  So that is fun.
         */
        if(1) {
        #else
        // handle general send errors
        if(EPIPE != errno) {
        #endif
            fprintf(stderr, "send error: %s\n", strerror(errno));
            c->flags |= CLIENT_IS_CLOSED;
            return d;
        }
        // on broken pipe, the remote end has already closed
        c->flags |= CLIENT_IS_CLOSED;
        #ifdef PRINT_CLIENT_DISCONNECT
        switch(reason) {
            case TIME_OUT_MSG:
            case INVALID_MSG:
            case SERVER_ERROR_MSG:
                fprintf(stderr, "client has disconnected before send complete\n");
                break;
            case RESPONSE_MSG:
                fprintf(stderr, "client has disconnected before response sent\n");
                break;
            default: ;
        }
        #endif
        return d;
    }
    // prevent further transmission, sends final packet (FIN TCP flag)
    if(0 == (WRITE_CLOSED & c->status)) {
        c->status |= WRITE_CLOSED;
        shutdown(c->fd, SHUT_WR);
    }
    return d;
}

__attribute__((always_inline)) static
void sendTimeout(struct client *c) {
    // canned messages to send to clients
    static char * timeout = TIMEOUT_TEXT;
    fprintf(stderr, "client timeout\n");
    #ifndef USE_CLIENT_POOL
    /*  when the write side of the pipe is closed but client has not, maybe
     *  we can nudge the socket into closing
     */
    char buf[512]; memset(buf, 0, sizeof(buf));
    if(-1 == send(c->fd, buf, sizeof(buf), 0)) {
        if(EPIPE == errno) {
            fprintf(stderr, "closed using send\n");
            c->flags |= CLIENT_IS_CLOSED;
            return;
        }
    }
    #endif
    sendResponse(c, timeout, strlen(timeout), TIME_OUT_MSG);
    /* [!] allow data loss
     *     once close is called the OS will immediately delete everything
     *     in the outgoing buffer, eg: the timeout message will not be
     *     sent.  In addition the client might send a request to this server
     *     and the OS will most likely drop the packet causing the client
     *     to hang, waiting on a response
     *     
     *  request the OS to send the client tcp stack a request to reset
     *  connection (causes RST TCP flag to be set) as soon as close is
     *  called on the socket
     */
    setLinger(c->fd);
    c->flags |= SERVER_WANTS_CLOSE;
}

#ifdef USE_CLIENT_POOL
__attribute__((always_inline)) static
void checkTimeouts(struct client pool[], int len) {
    uint32_t ms;
    for(int i = len; i--;) {
        // do not check the server socket
        if(0 == i) continue;
        // skip unused pooled spaces, or already closing clients
        if((0 > pool[i].fd) || (CLIENT_CLOSING & pool[i].flags)) continue;
        // also skip those clients that still have remaining milliseconds
        if(0 < pool[i].ms) continue;
        // run calculations on this client to determine if really timed out
        if(0 < (ms = msRemaining(pool[i].t, PER_CLIENT_TIMEOUT_MS / 2))) {
            // restore clients actual remaining milliseconds
            pool[i].ms = ms;
            continue;
        }
        /*  at this point the client has no more time left, maybe they sent some
         *  data in the last possible instant and they may have a legitimate
         *  request, or the server is running slow and we have not sent the
         *  client a final packet, either way the timeout rule says that we are
         *  done communicating with this client, cut them off completely
         *
         *  [?] we should have sent the client a timeout and a final packet,
         *      log when we have not done so
         */
        if(SERVER_WANTS_CLOSE & pool[i].flags) {
            if(0 == (WRITE_CLOSED & pool[i].status))
                fprintf(stderr, "never sent a packet with FIN flag\n");
            setLinger(pool[i].fd);
            pool[i].flags |= CLOSE_IMMEDIATELY;
            continue;
        }
        // okay, send this client a timeout message
        sendTimeout(&pool[i]);
    }
}

/*  at various times it may be easy to make a guess about time passing, for
 *  example, after making a sleep based system call and having no interruption
 *  occur, the requested sleep time will have passed.
 *  It is not really important to get this right, it is just an easy way
 *  to sometimes save some CPU time.
 *
 *  [?] The ACCEPT_LISTEN_TIME is used to for new clients avoiding the time the
 *      sever spent waiting on old clients while listening
 */
__attribute__((always_inline)) static
void prepayEstimates(struct client p[], uint32_t len, struct timespec t) {
    // estimate remaining milliseconds elapsed to use for client timeouts
    uint32_t ms = PER_CLIENT_TIMEOUT_MS - msRemaining(t, PER_CLIENT_TIMEOUT_MS);
    for(int i = len; i--;) {
        // either remove milliseconds or do so next time
        if(ACCEPT_LISTEN_TIME & p[i].status) {
            p[i].ms = p[i].ms > ms ?(p[i].ms - ms) :0;
        } else p[i].status |= ACCEPT_LISTEN_TIME;
    }
}
#endif

/*  Three things can happen:
 *  1.  just the transmit side of the full-duplex socket can be closed, in
 *      which case the client may still send data and we will be unable to
 *      respond, the socket will not be closed although client will be sent a
 *      request to close
 *  2.  we may close a socket, after the client says it is okay to do so - this
 *      is the normal case, where a client has closed the receive side of the
 *      full-duplex socket and we are unable to send anything else to the client
 *  3.  for some reason we just want to drop a connection without notifying
 *      the client.  This will cause TIME_WAIT, but might be improved if the
 *      Operating System is notified in advance with the SO_LINGER option
 *
 *  Technically the client should be closing the connection so the nice /
 *  correct thing to do is to send our goodbye intentions by setting the FIN
 *  TCP flag in either the last or an empty packet and transmitting this.  Upon
 *  receiving the FIN flagged packet the client will send back a their close
 *  and then we can close.  But we do not really want to wait here for that,
 *  instead we are just sending the FIN and then coming back later. Because TCP
 *  guarantees packet data delivery the socket API would need to guarantee that
 *  we got whatever the client sent, even though, at this point they may not
 *  have sent it.  We have to come back later and see if the client is finished
 *  before calling close on the client socket file descriptor.
 *
 *  Unless, it is a close immediately situation, so not TCP, we are just going
 *  to drop the socket and call close on it.  If there is still a client
 *  connected it would hang / freeze.
 *
 *  returns the number of socket file descriptors released to OS (one on
 *  close immediately/client is already closed, zero otherwise)
 */
int goodbyes(struct client *c) {
    // skip unused spaces in pool
    if(0 > c->fd) return 0;
    // can only say goodbye to a closed client
    if(0 == (CLIENT_CLOSING & c->flags)) return 0;
    // print what might have been sent to client, convert non-visible characters
    if(c->count) {
        // convert the users sent buffer into only visible characters and print
        convertToPrintable(c->buf, c->count);
        c->buf[FIM_BUFFER_LEN - 1] = '\0';
        fprintf(stderr, "client sent:\n%s\n", c->buf);
    }
    /*  close without waiting for client to close first will cause TIME_WAIT
     *  sockets.  Usually the OS will reap these in several minutes, also
     *  clients that are still connected will hang.  So, probably better to
     *  not do this all the time, but we still need to do this on occasion
     *  due to the fact that we are talking to another computer that might
     *  need to perform a BSOD or some other slightly more important crash
     *  right now
     */
    char host[NI_MAXHOST], service[NI_MAXSERV];
    int d;
    // when close is performed without notifying client, print warning
    if(CLOSE_IMMEDIATELY & c->flags) fprintf(stderr, "[!] immediate ");
    if(0 == (d = getnameinfo(
            (struct sockaddr *) &c->address,
            c->addrSize, 
            host, NI_MAXHOST,
            service, NI_MAXSERV, NI_NUMERICSERV))) {
        fprintf(stderr, "close: %s\n\n", host);
    } else {
        fprintf(stderr, "getnameinfo error: %s\n", gai_strerror(d));
        fprintf(stderr, "close\n\n");
    }
    close(c->fd);
    // remove the socket file descriptor from the pool
    c->fd = -1;
    return 1;
}

/*  when handling a single client and socketReady returns zero, indicating that
 *  select/poll sleep returned -1, and errno was set.  I guess it depends on
 *  what that error is, but it seems like we can just drop the client and move
 *  on.  generate a log message at a minimum for now.
 *  
 */
__attribute__((always_inline)) static int wakeupError(int e, struct client * c) {
    if(e) return /* false */ 0;
    fprintf(stderr, "closing client socket due to socketReady fail.\n");
    c->flags |= CLOSE_IMMEDIATELY;
    return /* true */ 1;
}

/*  sleep some number of milliseconds, but wake-up when a client connects
 *  clients are given up to 10 milliseconds to send data 
 *
 *  caller will pass a function that validates a string and returns a pointer
 *  to a buffer to send back to the client as a response
 *  a buffer containing a leading null is used to determine if the client
 *  has sent a partial request.  In this case nothing will be sent to the client, 
 *  the clients request will be later resent, in full, to the validation function
 *
 *  the buffer will have a maximum length of FIM_BUFFER_LEN
 */
#ifdef USE_CLIENT_POOL
int faceItMetal(uint32_t ms, char * (*f)(char *s, int len, int *limit), 
        struct client pool[]) {
    int c;
    struct timespec spec;
    // wait for client connections using a timeout (fatal on failure)
    if(0 == waitForClients(ms, pool)) return 0;
    // store the time before waiting on client data
    clock_gettime(CLOCK_REALTIME, &spec);
    // latest wake up is half of PER_CLIENT_TIMEOUT_MS
    ms = PER_CLIENT_TIMEOUT_MS / 2;
    // on wake up retrieve any pending data for each client, if possible reply
    if(DONE_FOR_NOW == (c = socketReady(pool, POOL_SIZE, ms))) {
    } else if(c) {
        // if there was a new client, add them
        if((READ_READY & pool[0].status) && (0 == waitForClients(0, pool))) return 0;
        for(int i = POOL_SIZE; i--;) replyOnValid(&pool[i], f);
    }
    // estimate the remaining milliseconds before timeout for all clients
    prepayEstimates(pool, POOL_SIZE, spec);
    // send timeout messages to all timed out clients
    checkTimeouts(pool, POOL_SIZE);
    // send farewells to closing clients (skip the listening socket)
    for(int i = POOL_SIZE; i--;) { if(0 == i) continue; goodbyes(&pool[i]); }
    return /* okay */ 1;
}
#else
int faceItMetal(int sock, uint32_t ms, char * (*f)(char *s, int len, int *limit)) {
    // when not using pooling, POOL_SIZE is 2
    struct client pool[POOL_SIZE];
    int c;
    // [?] even though we are not using pooling, setup pool 
    pool[0].fd = sock; pool[1].fd = -1;
    // wait for client connections using a timeout (fatal on failure)
    if(0 == waitForClients(ms, pool)) return 0;
    if(-1 == pool[1].fd) return /* timeout */ 1;
    if((0 == handleClient(pool, f)) && (0 == (CLIENT_CLOSING & pool[1].flags))) {
        sendTimeout(&pool[1]);
    }
    // say goodbye to client, finished early if socket is released
    if(goodbyes(&pool[1])) return /* okay */ 1;
    // client was told to close, it has the remaining time to do so
    handleClient(pool, f);
    // set a closing flag if there is none, then initiate irrefutable farewell
    if(0 == (CLIENT_CLOSING & pool[1].flags)) pool[1].flags |= CLOSE_IMMEDIATELY;
    goodbyes(&pool[1]);
    return /* okay */ 1;
}
int handleClient(struct client p[], char * (*f)(char *s, int len, int *limit)) {
    struct timespec spec;
    uint32_t ms;
    int c;
    clock_gettime(CLOCK_REALTIME, &spec);
    while(0 < (ms = msRemaining(spec, PER_CLIENT_TIMEOUT_MS / 2))) {
        // [?] unsure - browsers may not always close their socket in time
        if(DONE_FOR_NOW == (c = socketReady(&p[1], 1, ms))) continue;
        if(wakeupError(c, &p[1])) break;
        // retrieve any pending data for the client, if possible reply
        replyOnValid(&p[1], f);
        // check for an early exit, client should close their connection
        if(CLIENT_CLOSING & p[1].flags) break;
    }
    return ms;
}
#endif
    
#ifndef _WIN32
/*  when the application receives SIGPIPE, the OS was probably was the source
 *  of the signal.  It is telling us that whatever send that is currently being
 *  called will fail with an EPIPE due to the remote end of a connected socket
 *  being closed.
 *
 *  This is fine, we cannot control the other computer or application, but not
 *  handling the signal will cause the OS to close our application if the signal
 *  is not handled.  Install this signal handler on setup to make sure that
 *  the signal is handled.
 *
 *  [?] Normally we would just set the MSG_NOSIGNAL flag but Msft Winsock API
 *      does not support this flag.  We could just set the flag on Unix based
 *      OSes.
 */
void handle_sigpipe(int s) {
    fprintf(stderr, "recieved SIGPIPE: probably remote end broke connection\n");
}
#endif

/*  The main reason for this code is to hide some Msft proprietary code
 *  but obviously if you want to run something at the "end" of the program
 *  you can do that here.  Initially non-Microsoft OSes will just return zero
 *  while Msft wanders around aimlessly.
 *
 *  [!] The name of this function is pretty important, it was picked as to not
 *      represent anything related to exiting, goodbye, or leaving.  It does not
 *      have these functionality.  For most operating systems it just returns
 *      zero.  For Msft it is used when the Winsock interface is no longer
 *      needed (see: WSACleanup)
 */
int softCuteBunnies(int c) {
    #ifdef _WIN32
    // do the shutdown dance for Msft
    WSACleanup();
    #endif
    return c;
}