commit 0af00f8105bc66d792f73076d770a5b30323a956
Author: Patrick McManus <mcmanus@ducksong.com>
Date:   Fri Aug 20 20:20:24 2010 -0400

    insturment out_of_order delays

diff --git a/include/linux/tcp.h b/include/linux/tcp.h
index a778ee0..31a19cd 100644
--- a/include/linux/tcp.h
+++ b/include/linux/tcp.h
@@ -459,6 +459,13 @@ struct tcp_sock {
 	 * contains related tcp_cookie_transactions fields.
 	 */
 	struct tcp_cookie_values  *cookie_values;
+
+        struct 
+        {
+                u32 rcv_isn;
+                u32 pkts_rcv;
+                u8  fin_reported;
+        } ofo;
 };
 
 static inline struct tcp_sock *tcp_sk(const struct sock *sk)
@@ -488,6 +495,10 @@ static inline struct tcp_timewait_sock *tcp_twsk(const struct sock *sk)
 	return (struct tcp_timewait_sock *)sk;
 }
 
+
+void ofo_notify(struct sock *sk, struct sk_buff *skb, int flags);
+
+
 #endif	/* __KERNEL__ */
 
 #endif	/* _LINUX_TCP_H */
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 65afeae..2eb4a19 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -2087,6 +2087,8 @@ int tcp_disconnect(struct sock *sk, int flags)
 	tp->snd_cwnd_cnt = 0;
 	tp->bytes_acked = 0;
 	tp->window_clamp = 0;
+        tp->ofo.pkts_rcv = 0;
+        tp->ofo.fin_reported = 0;
 	tcp_set_ca_state(sk, TCP_CA_Open);
 	tcp_clear_retrans(tp);
 	inet_csk_delack_init(sk);
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index 548d575..70ea32d 100644
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -4321,6 +4321,7 @@ static void tcp_ofo_queue(struct sock *sk)
 
 		if (!after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt)) {
 			SOCK_DEBUG(sk, "ofo packet was already received\n");
+                        ofo_notify(sk, skb, 0);
 			__skb_unlink(skb, &tp->out_of_order_queue);
 			__kfree_skb(skb);
 			continue;
@@ -4329,6 +4330,7 @@ static void tcp_ofo_queue(struct sock *sk)
 			   tp->rcv_nxt, TCP_SKB_CB(skb)->seq,
 			   TCP_SKB_CB(skb)->end_seq);
 
+                ofo_notify(sk, skb, 0);
 		__skb_unlink(skb, &tp->out_of_order_queue);
 		__skb_queue_tail(&sk->sk_receive_queue, skb);
 		tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq;
@@ -4367,7 +4369,7 @@ static void tcp_data_queue(struct sock *sk, struct sk_buff *skb)
 
 	if (TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq)
 		goto drop;
-
+        
 	skb_dst_drop(skb);
 	__skb_pull(skb, th->doff * 4);
 
@@ -4487,7 +4489,7 @@ drop:
 		   tp->rcv_nxt, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq);
 
 	skb_set_owner_r(skb, sk);
-
+        
 	if (!skb_peek(&tp->out_of_order_queue)) {
 		/* Initial out of order segment, build 1 SACK. */
 		if (tcp_is_sack(tp)) {
@@ -4547,6 +4549,7 @@ drop:
 						skb1);
 			}
 		}
+                
 		if (!skb1)
 			__skb_queue_head(&tp->out_of_order_queue, skb);
 		else
@@ -5229,6 +5232,9 @@ int tcp_rcv_established(struct sock *sk, struct sk_buff *skb,
 	struct tcp_sock *tp = tcp_sk(sk);
 	int res;
 
+        if (TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq)
+                tp->ofo.pkts_rcv++;
+
 	/*
 	 *	Header prediction.
 	 *	The code loosely follows the one in the famous
@@ -5506,7 +5512,8 @@ static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb,
 		 */
 		tp->rcv_nxt = TCP_SKB_CB(skb)->seq + 1;
 		tp->rcv_wup = TCP_SKB_CB(skb)->seq + 1;
-
+                tp->ofo.rcv_isn = TCP_SKB_CB(skb)->seq;
+                
 		/* RFC1323: The window in SYN & SYN/ACK segments is
 		 * never scaled.
 		 */
@@ -5942,3 +5949,83 @@ EXPORT_SYMBOL(tcp_parse_md5sig_option);
 EXPORT_SYMBOL(tcp_rcv_established);
 EXPORT_SYMBOL(tcp_rcv_state_process);
 EXPORT_SYMBOL(tcp_initialize_rcv_mss);
+
+#include <linux/connector.h>
+
+struct uofo_notifier
+{ 
+        struct timeval tstamp1;
+        struct timeval tstamp2;
+        __u32 seq;
+        __u16 orig_len;
+        __u16 actual_len;
+        __u32 daddr;
+        __u32 saddr;
+        __u16 dport;
+        __u16 sport;
+        __u32 pkts_rcv;
+        __u16 flags;                              /* 1 - FIN */
+        
+        char data[0];
+};
+
+static atomic_t cn_ofo_nextseq;
+static struct cb_id cn_ofo_id = { CN_NETLINK_USERS + 100, 1 };
+
+// flags - fin/rst 0x01 
+void ofo_notify(struct sock *sk, struct sk_buff *skb, int flags)
+{
+	struct tcp_sock *tp = tcp_sk(sk);
+        struct uofo_notifier *notifier;
+        u16 actual = 0;
+        struct cn_msg *m;
+
+        BUG_ON (flags & ~0x01);
+        
+        if (flags & 0x01)
+        {
+                if (tp->ofo.fin_reported) return;
+                tp->ofo.fin_reported = 1;
+        }
+
+        m = kzalloc(sizeof(struct cn_msg) + sizeof(struct uofo_notifier) + actual, GFP_ATOMIC);
+        if (!m) return;
+        notifier = (struct uofo_notifier *) (m + 1);
+        
+        notifier->daddr =  inet_sk(sk)->inet_daddr;
+        notifier->saddr =  inet_sk(sk)->inet_saddr;
+        notifier->dport =  inet_sk(sk)->inet_dport;
+        notifier->sport =  inet_sk(sk)->inet_sport;
+        notifier->actual_len = actual;
+
+        if (skb)
+        {
+                notifier->orig_len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq;
+                notifier->seq = TCP_SKB_CB(skb)->seq - tp->ofo.rcv_isn;
+
+                skb_get_timestamp(skb, &notifier->tstamp1);
+                do_gettimeofday(&notifier->tstamp2);
+                memcpy (notifier->data, skb->data, actual);
+        }
+        else
+        {
+                notifier->orig_len = 0;
+                notifier->seq = 0;
+                actual = 0;
+                do_gettimeofday(&notifier->tstamp1);
+                memcpy (&notifier->tstamp2, &notifier->tstamp1, sizeof (struct timeval));
+        }
+
+        notifier->pkts_rcv = tp->ofo.pkts_rcv;
+
+        if (flags & 0x01)
+                notifier->flags |= 0x01;
+        
+        memcpy(&m->id, &cn_ofo_id, sizeof(m->id));
+        m->seq = (flags & 0x01) ? atomic_read (&cn_ofo_nextseq) : atomic_inc_return (&cn_ofo_nextseq);
+        m->len = sizeof(struct uofo_notifier) + actual;
+        cn_netlink_send(m, 1, GFP_ATOMIC);
+        kfree(m);
+        
+    return;
+}
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index fe193e5..3da94fc 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -1661,6 +1661,9 @@ process:
 	if (sk->sk_state == TCP_TIME_WAIT)
 		goto do_time_wait;
 
+        if ((sk->sk_state == TCP_ESTABLISHED) && (th->fin || th->rst))
+                ofo_notify(sk, skb, 1);
+
 	if (unlikely(iph->ttl < inet_sk(sk)->min_ttl)) {
 		NET_INC_STATS_BH(net, LINUX_MIB_TCPMINTTLDROP);
 		goto discard_and_relse;
diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c
index 794c2e1..3448f53 100644
--- a/net/ipv4/tcp_minisocks.c
+++ b/net/ipv4/tcp_minisocks.c
@@ -273,6 +273,8 @@ void tcp_time_wait(struct sock *sk, int state, int timeo)
 	const struct tcp_sock *tp = tcp_sk(sk);
 	int recycle_ok = 0;
 
+        ofo_notify(sk, NULL, 1);
+
 	if (tcp_death_row.sysctl_tw_recycle && tp->rx_opt.ts_recent_stamp)
 		recycle_ok = icsk->icsk_af_ops->remember_stamp(sk);
 
@@ -418,7 +420,8 @@ struct sock *tcp_create_openreq_child(struct sock *sk, struct request_sock *req,
 
 		newtp->rcv_wup = newtp->copied_seq =
 		newtp->rcv_nxt = treq->rcv_isn + 1;
-
+                newtp->ofo.rcv_isn = treq->rcv_isn;
+                
 		newtp->snd_sml = newtp->snd_una =
 		newtp->snd_nxt = newtp->snd_up =
 			treq->snt_isn + 1 + tcp_s_data_size(oldtp);
