echo 3 > /proc/sys/net/ipv4/tcp_fastopen   # 1 开启客户端，2 开启服务端，3 都开启

tc qdisc add dev lo root netem delay 300ms # 设置延迟才能看出效果
ifconfig lo mtu 1500

echo 3 > /proc/sys/net/ipv4/tcp_fastopen  

root@ubuntu:~/c++# sysctl -a | grep fastopen
net.ipv4.tcp_fastopen = 3
net.ipv4.tcp_fastopen_blackhole_timeout_sec = 0
net.ipv4.tcp_fastopen_key = af808897-41a286c8-804a90f4-2cae798b

root@ubuntu:~/c++# cat fast_serv.c 
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <stdio.h>

int main()
{
        int serverSock, clientSock;
        struct sockaddr_in addr, clientAddr;
        int addrLen;

        char buf[10240];
        int n, tot;

        serverSock = socket(AF_INET, SOCK_STREAM, 0);
        if (serverSock == -1) {
                printf("socket failed!\n");
                return -1;
        }

        memset(&addr, 0, sizeof(addr));
        addr.sin_family = AF_INET;
        addr.sin_port = htons(1935);
        addr.sin_addr.s_addr = inet_addr("10.10.16.82");

        if (bind(serverSock, (struct sockaddr*)&addr, sizeof(addr)) < 0) {
                printf("bind failed!\n");
                return -2;
        }

        int qlen = 5;
        setsockopt(serverSock, SOL_TCP, TCP_FASTOPEN, &qlen, sizeof(qlen));

        if (listen(serverSock, 511) < 0) {
                printf("listen failed!\n");
                return -3;
        }

        while (1) {
                addrLen = sizeof(clientAddr);
                clientSock = accept(serverSock, (struct sockaddr*)&clientAddr, &addrLen);
                if (clientSock < 0) {
                        printf("accept failed!\n");
                        return -4;
                }

                if ((n = send(clientSock, buf, 10000, 0)) < 0) {
                        printf("send error ret = %d\n", n);
                        return -5;
                }
                printf("server send len = %d\n", n);
                shutdown(clientSock, 1);

                sleep(1);

                tot = 0;
                while ((n = recv(clientSock, buf, 1024, 0)) > 0)
                        tot += n;
                printf("server recv len = %d\n", tot);
                close(clientSock);
        }

        return 0;
}

echo 3 > /proc/sys/net/ipv4/tcp_fastopen  

[root@bogon ~]# cat fast_cli.c 
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>

#include <netinet/tcp.h>

#ifndef MSG_FASTOPEN
#define MSG_FASTOPEN   0x20000000
#endif

int main(int argc, char *argv[])
{
        int sockfd, n;
        struct sockaddr_in servaddr;
        char buf[50000] = "aaabbbccc";
        int ret = 0, tot;

        if ((sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
                printf ("create socket error: %s(errno: %d)\n", strerror (errno), errno);
                return -1;
        }

        memset (&servaddr, 0, sizeof (servaddr));
        servaddr.sin_family = AF_INET;
        servaddr.sin_port = htons (1935);
        servaddr.sin_addr.s_addr = inet_addr("10.10.16.82");

#define FASTOPEN_TEST
#ifndef FASTOPEN_TEST
        if (connect(sockfd, (struct sockaddr *)&servaddr, sizeof(servaddr))) {
                printf("connect error\n");
                return -2;
        }
        ret = send(sockfd, buf, 1005, 0);
#else
        ret = sendto(sockfd, buf, 1005, MSG_FASTOPEN, (struct sockaddr *)&servaddr, sizeof(servaddr));
#endif
        if (ret < 0) {
                printf ("send msg error: %s(errno: %d)\n", strerror (errno), errno);
                // 如果是连接失败会打印：Connection refused(errno: 111)
                return -2;
        }
        printf("client fastopen sendto len=%d\n", ret);
        if ((ret = send(sockfd, buf, 20000, 0)) < 0) {
                printf("send error ret = %d\n", ret);
        }
        printf("client send len = %d\n", ret);
        shutdown(sockfd, 1);

        tot = 0;
        while ((n = recv(sockfd, buf, 1024, 0)) > 0)
                tot += n;
        printf("client recv len = %d\n", tot);
        close (sockfd);
        return 0;
}

root@ubuntu:~# sysctl -w  net.ipv4.tcp_fastopen_blackhole_timeout_sec=0
net.ipv4.tcp_fastopen_blackhole_timeout_sec = 0
root@ubuntu:~# tcpdump -i enahisic2i0 tcp port 1935 -Xnn
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes
17:27:22.590927 IP 10.10.16.81.55956 > 10.10.16.82.1935: Flags [S], seq 2910306470, win 29200, options [mss 1460,sackOK,TS val 1987645332 ecr 0,nop,wscale 7,tfo  cookiereq,nop,nop], length 0
        0x0000:  4500 0040 8418 4000 4006 81e9 0a0a 1051  E..@..@.@......Q
        0x0010:  0a0a 1root@ubuntu:~# sysctl -w  net.ipv4.tcp_fastopen_blackhole_timeout_sec=0
net.ipv4.tcp_fastopen_blackhole_timeout_sec = 0
root@ubuntu:~# tcpdump -i enahisic2i0 tcp port 1935 -Xnn
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes
17:27:22.590927 IP 10.10.16.81.55956 > 10.10.16.82.1935: Flags [S], seq 2910306470, win 29200, options [mss 1460,sackOK,TS val 1987645332 ecr 0,nop,wscale 7,tfo  cookiereq,nop,nop], length 0
        0x0000:  4500 0040 8418 4000 4006 81e9 0a0a 1051  E..@..@.@......Q
        0x0010:  0a0a 1052 da94 078f ad77 c0a6 0000 0000  ...R.....w......
        0x0020:  b002 7210 97e2 0000 0204 05b4 0402 080a  ..r.............
        0x0030:  7679 0f94 0000 0000 0103 0307 2202 0101  vy.........."...
17:27:22.590998 IP 10.10.16.82.1935 > 10.10.16.81.55956: Flags [R.], seq 0, ack 2910306471, win 0, length 0 ------竟然回复rst
        0x0000:  4500 0028 0000 4000 4006 061a 0a0a 1052  E..(..@.@......R
        0x0010:  0a0a 1051 078f da94 0000 0000 ad77 c0a7  ...Q.........w..
        0x0020:  5014 0000 2ad7 0000                      P...*...052 da94 078f ad77 c0a6 0000 0000  ...R.....w......
        0x0020:  b002 7210 97e2 0000 0204 05b4 0402 080a  ..r.............
        0x0030:  7679 0f94 0000 0000 0103 0307 2202 0101  vy.........."...
17:27:22.590998 IP 10.10.16.82.1935 > 10.10.16.81.55956: Flags [R.], seq 0, ack 2910306471, win 0, length 0
        0x0000:  4500 0028 0000 4000 4006 061a 0a0a 1052  E..(..@.@......R
        0x0010:  0a0a 1051 078f da94 0000 0000 ad77 c0a7  ...Q.........w..
        0x0020:  5014 0000 2ad7 0000                      P...*...