vlambda博客
学习文章列表

setsockopt(client_fd, SOL_TCP, TCP_NODELAY,(int[]){1}, sizeof(int));//关闭Naglesetsockopt(client_fd, SOL_TCP, TCP_NODELAY,(int[]){1}, sizeof(int));//打开Cork

sudo tcpdump -S -i any -n tcp port 8080 //8080是上图右边使用的端口

void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss, int nonagle){ /* If we are closed, the bytes will have to remain here. * In time closedown will finish, we empty the write queue and * all will be happy. */ if (unlikely(sk->sk_state == TCP_CLOSE)) return;
 if (tcp_write_xmit(sk, cur_mss, nonagle, 0, sk_gfp_mask(sk, GFP_ATOMIC)))    //什么时候会走到这里呢？在我们的测试中第一次发送的8字节包会走到    //这里，也就是说第一次发的8字节也是被延时了。 tcp_check_probe_timer(sk);}

void tcp_write_timer_handler(struct sock *sk){ struct inet_connection_sock *icsk = inet_csk(sk);  int event;
 tcp_mstamp_refresh(tcp_sk(sk)); event = icsk->icsk_pending;
  switch (event) {  //对应第二次发送剩余的5字节的小包，  //tcp_send_loss_probe会把这个小包发出去 case ICSK_TIME_LOSS_PROBE: tcp_send_loss_probe(sk);    break;    //对应第一个8字节的小包，tcp_probe_timer    //会把这个小包发出去 case ICSK_TIME_PROBE0: icsk->icsk_pending = 0; tcp_probe_timer(sk); break; }
out: sk_mem_reclaim(sk);}

#!/usr/bin/python## Copyright (c) 2022 xin.guo.# Licensed under the Apache License, Version 2.0 (the "License")## 15-May-2022 xin guo Created this.
from __future__ import print_functionfrom bcc import BPFfrom bcc.utils import printb
# define BPF programbpf_text = """#include <uapi/linux/ptrace.h>#include <net/sock.h>#include <net/tcp.h>#include <net/inet_connection_sock.h>#include <bcc/proto.h>#include <linux/socket.h>
BPF_HASH(currsock, u32, struct sock *);
int kprobe__tcp_rate_skb_sent(struct pt_regs *ctx, struct sock *sk,const struct sk_buff *skb){ u32 pid = bpf_get_current_pid_tgid();  struct tcp_skb_cb *tcb;  if (skb) { /* macro TCP_SKB_CB from net/tcp.h */ tcb = ((struct tcp_skb_cb *)&((skb)->cb[0])); u16 dport = sk->__sk_common.skc_dport;  u16 sport = sk->__sk_common.skc_num;  struct tcp_sock *tp = tcp_sk(sk);   if(sport == 8080) { bpf_trace_printk("tcp_rate_skb_sent call seq:%u dport:%u len:%d\\n", tcb->seq, ntohs(dport) , skb->len); bpf_trace_printk("tcp_rate_skb_sent call packets_out:%u skb data_len:%u\\n", tp->packets_out, skb->data_len); bpf_trace_printk("tcp_rate_skb_sent segs:%u gso_max_size:%u\\n", tp->gso_segs, sk->sk_gso_max_size); } }

 return 0;};
int kprobe____ip_queue_xmit(struct pt_regs *ctx, struct sock *sk,const struct sk_buff *skb){ u32 pid = bpf_get_current_pid_tgid();  struct tcp_skb_cb *tcb;  if (skb) { /* macro TCP_SKB_CB from net/tcp.h */ tcb = ((struct tcp_skb_cb *)&((skb)->cb[0])); u16 dport = sk->__sk_common.skc_dport; u16 sport = sk->__sk_common.skc_num;  struct tcp_sock *tp = tcp_sk(sk);  if(sport == 8080) { bpf_trace_printk("ip_queue_xmit call%u %u len:%d\\n", tcb->seq, ntohs(dport) , skb->len); bpf_trace_printk("ip_queue_xmit packets_out:%u skb data_len:%u sport:%u\\n", tp->packets_out, skb->data_len, sport); bpf_trace_printk("ip_queue_xmit segs:%u gso_max_size:%u\\n", tp->gso_segs, sk->sk_gso_max_size); } }

 return 0;};
int kprobe__tcp_write_timer_handler(struct pt_regs *ctx, struct sock *sk){ u32 pid = bpf_get_current_pid_tgid();  u16 dport = sk->__sk_common.skc_dport;  u16 sport = sk->__sk_common.skc_num;  struct tcp_sock *tp = tcp_sk(sk);  struct inet_connection_sock *icsk = inet_csk(sk);    if(sport == 8080) { bpf_trace_printk("tcp_write_timer_handler call:dst port %u sport:%u event:%u\\n",ntohs(dport) , sport, icsk->icsk_pending ); }
 return 0;};
int kprobe__tcp_send_loss_probe(struct pt_regs *ctx, struct sock *sk){ u32 pid = bpf_get_current_pid_tgid();  u16 dport = sk->__sk_common.skc_dport;  u16 sport = sk->__sk_common.skc_num;  struct tcp_sock *tp = tcp_sk(sk);  if(sport == 8080) { bpf_trace_printk("tcp_send_loss_probe call:dst port %u sport:%u\\n",ntohs(dport) , sport ); }
 return 0;};
int kprobe__tcp_push(struct pt_regs *ctx, struct sock *sk, int flags, int mss_now, int nonagle, int size_goal){ u32 pid = bpf_get_current_pid_tgid();  u16 dport = sk->__sk_common.skc_dport;  u16 sport = sk->__sk_common.skc_num;  struct tcp_sock *tp = tcp_sk(sk);  if(sport == 8080) { bpf_trace_printk("tcp_push call:dst port %u sport:%u\\n",ntohs(dport) , sport ); bpf_trace_printk("tcp_push mss_now %u nonagle:%u size_goal:%u\\n",mss_now , nonagle, size_goal ); }
 return 0;};
int kprobe__tcp_sendmsg_locked(struct pt_regs *ctx, struct sock *sk, struct msghdr *msg, size_t size){ u32 pid = bpf_get_current_pid_tgid();   struct tcp_sock *tp = tcp_sk(sk);  u16 sport = sk->__sk_common.skc_num;
 if(sport == 8080) { bpf_trace_printk("tcp_sendmsg_locked call%u %u\\n", size, tp->packets_out); }

 return 0;};
int kretprobe__tcp_sendmsg_locked(struct pt_regs *ctx){ u32 pid = bpf_get_current_pid_tgid();  int ret = PT_REGS_RC(ctx);
 bpf_trace_printk("tcp_sendmsg_locked ret %u %u\\n", pid, ret);

 return 0;}
""" 
# initialize BPFb = BPF(text=bpf_text)
# headerprint("%-6s %-12s %-16s" % ("PID", "COMM", "Msg"))

# filter and format outputwhile 1: # Read messages from kernel pipe try: (task, pid, cpu, flags, ts, msg) = b.trace_fields() #(_tag, seq, mss, nagle) = msg.split(b" ")  except ValueError: # Ignore messages from other tracers continue except KeyboardInterrupt: exit()  # Ignore messages from other tracers #if task.decode() != "packetdrill": # continue
 printb(b"%-6d %-12.12s %-16s" % (pid, task, msg))

PID COMM Msg //第一次发包8字节65098 packetdrill tcp_sendmsg_locked call8 065098 packetdrill tcp_push call:dst port 58909 sport:808065098 packetdrill tcp_push mss_now 50 nonagle:2 size_goal:1640065098 packetdrill tcp_sendmsg_locked ret 65098 8//第一次真正发包，event:3 代表ICSK_TIME_PROBE00 <idle> tcp_write_timer_handler call:dst port 58909 sport:8080 event:30 <idle> ip_queue_xmit call0 58909 len:400 <idle> ip_queue_xmit packets_out:0 skb data_len:8 sport:80800 <idle> ip_queue_xmit segs:328 gso_max_size:655360 <idle> tcp_rate_skb_sent call seq:2780301583 dport:58909 len:80 <idle> tcp_rate_skb_sent call packets_out:0 skb data_len:80 <idle> tcp_rate_skb_sent segs:328 gso_max_size:655360 <idle> tcp_write_timer_handler call:dst port 58909 sport:8080 event:0//第二次发包105字节65098 packetdrill tcp_sendmsg_locked call105 065098 packetdrill tcp_push call:dst port 58909 sport:808065098 packetdrill tcp_push mss_now 50 nonagle:2 size_goal:19200//发送100字节，最终是两个包出去，下面我们解释一些原因65098 packetdrill ip_queue_xmit call0 58909 len:13265098 packetdrill ip_queue_xmit packets_out:0 skb data_len:100 sport:808065098 packetdrill ip_queue_xmit segs:384 gso_max_size:6553665098 packetdrill tcp_rate_skb_sent call seq:2780301591 dport:58909 len:10065098 packetdrill tcp_rate_skb_sent call packets_out:0 skb data_len:10065098 packetdrill tcp_rate_skb_sent segs:384 gso_max_size:6553665098 packetdrill tcp_sendmsg_locked ret 65098 105//5字节的小包发送，event:5代表ICSK_TIME_LOSS_PROBE24618 Xorg tcp_write_timer_handler call:dst port 58909 sport:8080 event:524618 Xorg tcp_send_loss_probe call:dst port 58909 sport:808024618 Xorg ip_queue_xmit call0 58909 len:3724618 Xorg ip_queue_xmit packets_out:2 skb data_len:5 sport:808024618 Xorg ip_queue_xmit segs:384 gso_max_size:6553624618 Xorg tcp_rate_skb_sent call seq:2780301691 dport:58909 len:524618 Xorg tcp_rate_skb_sent call packets_out:2 skb data_len:524618 Xorg tcp_rate_skb_sent segs:384 gso_max_size:65536//下面是其他的一些交互，我们先不关注。0 <idle> tcp_write_timer_handler call:dst port 58909 sport:8080 event:065098 packetdrill ip_queue_xmit call0 58909 len:3265098 packetdrill ip_queue_xmit packets_out:0 skb data_len:0 sport:808065098 packetdrill ip_queue_xmit segs:384 gso_max_size:6553665098 packetdrill tcp_rate_skb_sent call seq:2780301696 dport:58909 len:065098 packetdrill tcp_rate_skb_sent call packets_out:0 skb data_len:065098 packetdrill tcp_rate_skb_sent segs:384 gso_max_size:65536

void tcp_init_sock(struct sock *sk){
sk->sk_route_forced_caps = NETIF_F_GSO;}