modified: homa_impl.h

	modified:   homa_offload.c
	modified:   homa_offload.h
	modified:   homa_outgoing.c
	modified:   homa_plumbing.c
	modified:   homa_qdisc.c
	modified:   homa_sock.c
	modified:   homa_wire.h
	modified:   util/cp_node.cc
	modified:   util/homa_test.cc
	modified:   util/server.cc

Author: axkum10

homa_impl.h

@@ -358,6 +358,13 @@ struct homa {
 	 */
 	int hijack_tcp;
 
+	/**
+	 * @hijack_udp: Non-zero means encapsulate outgoing Homa packets
+	 * as UDP packets (i.e. use UDP as the IP protocol). Set externally
+	 * via sysctl.
+	 */
+	int hijack_udp;
+
 	/**
 	 * @max_gro_skbs: Maximum number of socket buffers that can be
 	 * aggregated by the GRO mechanism.  Set externally via sysctl.
@@ -665,7 +672,9 @@ static inline bool is_homa_pkt(struct sk_buff *skb)
 					ip_hdr(skb)->protocol;
 	return (protocol == IPPROTO_HOMA ||
 		(protocol == IPPROTO_TCP &&
-		 tcp_hdr(skb)->urg_ptr == htons(HOMA_TCP_URGENT)));
+		 tcp_hdr(skb)->urg_ptr == htons(HOMA_TCP_URGENT)) ||
+		(protocol == IPPROTO_UDP &&
+		 tcp_hdr(skb)->urg_ptr == htons(HOMA_UDP_URGENT)));
 	return protocol == IPPROTO_HOMA;
 }
 #endif /* See strip.py */

homa_offload.c

@@ -37,6 +37,19 @@ static const struct net_offload *tcp6_net_offload;
  */
 static struct net_offload hook_tcp_net_offload;
 static struct net_offload hook_tcp6_net_offload;
+
+/* Pointers to UDP's net_offload structures. NULL means homa_gro_hook_udp
+ * hasn't been called yet.
+ */
+static const struct net_offload *udp_net_offload;
+static const struct net_offload *udp6_net_offload;
+
+/*
+ * Identical to *udp_net_offload except that the gro_receive function
+ * has been replaced.
+ */
+static struct net_offload hook_udp_net_offload;
+static struct net_offload hook_udp6_net_offload;
 #endif /* See strip.py */
 
 /**
@@ -167,6 +180,82 @@ struct sk_buff *homa_tcp_gro_receive(struct list_head *held_list,
 	return homa_gro_receive(held_list, skb);
 }
 
+/**
+ * homa_gro_hook_udp() - Arranges for UDP gro_receive calls to be
+ * mediated by this file, so that Homa-over-UDP packets can be retrieved
+ * and funneled through Homa.
+ */
+void homa_gro_hook_udp(void)
+{
+	if (udp_net_offload)
+		return;
+
+	pr_notice("Homa setting up UDP hijacking\n");
+	rcu_read_lock();
+	udp_net_offload = rcu_dereference(inet_offloads[IPPROTO_UDP]);
+	hook_udp_net_offload = *udp_net_offload;
+	hook_udp_net_offload.callbacks.gro_receive = homa_udp_gro_receive;
+	inet_offloads[IPPROTO_UDP] = (struct net_offload __rcu *)
+			&hook_udp_net_offload;
+
+	udp6_net_offload = rcu_dereference(inet6_offloads[IPPROTO_UDP]);
+	hook_udp6_net_offload = *udp6_net_offload;
+	hook_udp6_net_offload.callbacks.gro_receive = homa_udp_gro_receive;
+	inet6_offloads[IPPROTO_UDP] = (struct net_offload __rcu *)
+			&hook_udp6_net_offload;
+	rcu_read_unlock();
+}
+
+/**
+ * homa_gro_unhook_udp() - Reverses the effects of a previous call to
+ * homa_gro_hook_udp, so that UDP packets are now passed directly to
+ * UDP's gro_receive function without mediation.
+ */
+void homa_gro_unhook_udp(void)
+{
+	if (!udp_net_offload)
+		return;
+	pr_notice("Homa cancelling UDP hijacking\n");
+	inet_offloads[IPPROTO_UDP] = (struct net_offload __rcu *)
+			udp_net_offload;
+	udp_net_offload = NULL;
+	inet6_offloads[IPPROTO_UDP] = (struct net_offload __rcu *)
+			udp6_net_offload;
+	udp6_net_offload = NULL;
+}
+
+/**
+ * homa_udp_gro_receive() - Invoked instead of UDP's normal gro_receive
+ * function when hooking is enabled. Identifies Homa-over-UDP packets and
+ * passes them to Homa; sends real UDP packets to UDP's gro_receive function.
+ * @held_list:  Pointer to header for list of packets that are being
+ *              held for possible GRO merging.
+ * @skb:        The newly arrived packet.
+ */
+struct sk_buff *homa_udp_gro_receive(struct list_head *held_list,
+				     struct sk_buff *skb)
+{
+	struct homa_common_hdr *h = (struct homa_common_hdr *)
+			skb_transport_header(skb);
+
+	if (h->flags != HOMA_UDP_FLAGS ||
+	    ntohs(h->urgent) != HOMA_UDP_URGENT)
+		return udp_net_offload->callbacks.gro_receive(held_list, skb);
+
+	/* Change the packet's IP protocol to Homa so that it will get
+	 * dispatched directly to Homa in the future.
+	 */
+	if (skb_is_ipv6(skb)) {
+		ipv6_hdr(skb)->nexthdr = IPPROTO_HOMA;
+	} else {
+		ip_hdr(skb)->check = ~csum16_add(csum16_sub(~ip_hdr(skb)->check,
+							    htons(ip_hdr(skb)->protocol)),
+						 htons(IPPROTO_HOMA));
+		ip_hdr(skb)->protocol = IPPROTO_HOMA;
+	}
+	return homa_gro_receive(held_list, skb);
+}
+
 /**
  * homa_set_softirq_cpu() - Arrange for SoftIRQ processing of a packet to
  * occur on a specific core (creates a socket flow table entry for the core,

homa_offload.h

@@ -78,6 +78,8 @@ void     homa_gro_gen3(struct homa *homa, struct sk_buff *skb);
 #ifndef __STRIP__ /* See strip.py */
 void     homa_gro_hook_tcp(void);
 void     homa_gro_unhook_tcp(void);
+void     homa_gro_hook_udp(void);
+void     homa_gro_unhook_udp(void);
 #endif /* See strip.py */
 struct sk_buff *homa_gro_receive(struct list_head *gro_list,
 				 struct sk_buff *skb);
@@ -90,6 +92,8 @@ void     homa_set_softirq_cpu(struct sk_buff *skb, int cpu);
 #ifndef __STRIP__ /* See strip.py */
 struct sk_buff *homa_tcp_gro_receive(struct list_head *held_list,
 				      struct sk_buff *skb);
+struct sk_buff *homa_udp_gro_receive(struct list_head *held_list,
+				      struct sk_buff *skb);
 #endif /* See strip.py */
 
 #endif /* _HOMA_OFFLOAD_H */

homa_outgoing.c

@@ -10,6 +10,7 @@
 #include "homa_wire.h"
 
 #ifndef __STRIP__ /* See strip.py */
+#include <net/udp.h>
 #include "homa_pacer.h"
 #include "homa_qdisc.h"
 #include "homa_skb.h"
@@ -49,6 +50,45 @@ static inline void homa_set_hijack(struct sk_buff *skb, struct homa_peer *peer,
 		h->checksum = ~tcp_v4_check(skb->len, peer->flow.u.ip4.saddr,
 					    peer->flow.u.ip4.daddr, 0);
 }
+
+/**
+ * homa_set_udp_hijack() - Set fields in an outgoing Homa packet that are
+ * needed for UDP hijacking to work properly. Similar to homa_set_hijack()
+ * but uses IPPROTO_UDP for checksumming and writes a UDP length/checksum
+ * overlay at bytes 4-7 of the transport header.
+ * @skb:    Packet buffer in which to set fields.
+ * @peer:   Peer that contains source and destination addresses for the packet.
+ * @ipv6:   True means the packet is going to be sent via IPv6; false means
+ *          IPv4.
+ */
+static inline void homa_set_udp_hijack(struct sk_buff *skb,
+					struct homa_peer *peer, bool ipv6)
+{
+	struct homa_common_hdr *h;
+	__be16 *udp_len_csum;
+
+	h = (struct homa_common_hdr *)skb_transport_header(skb);
+	h->flags = HOMA_UDP_FLAGS;
+	h->urgent = htons(HOMA_UDP_URGENT);
+
+	/* Write UDP length and checksum at bytes 4-7 of transport header. */
+	udp_len_csum = (__be16 *)(skb_transport_header(skb) + 4);
+	udp_len_csum[0] = htons(skb->len);    /* UDP length */
+	udp_len_csum[1] = 0;                  /* UDP checksum (0 = none) */
+
+	/* Arrange for proper UDP checksumming. */
+	skb->ip_summed = CHECKSUM_PARTIAL;
+	skb->csum_start = skb_transport_header(skb) - skb->head;
+	skb->csum_offset = offsetof(struct homa_common_hdr, checksum);
+	if (ipv6)
+		h->checksum = ~csum_ipv6_magic(&peer->flow.u.ip6.saddr,
+					       &peer->flow.u.ip6.daddr,
+					       skb->len, IPPROTO_UDP, 0);
+	else
+		h->checksum = ~udp_v4_check(skb->len,
+					    peer->flow.u.ip4.saddr,
+					    peer->flow.u.ip4.daddr, 0);
+}
 #endif /* See strip.py */
 
 /**
@@ -204,7 +244,10 @@ struct sk_buff *homa_tx_data_pkt_alloc(struct homa_rpc *rpc,
 	IF_NO_STRIP(h->cutoff_version = rpc->peer->cutoff_version);
 	h->retransmit = 0;
 #ifndef __STRIP__ /* See strip.py */
-	h->seg.offset = htonl(-1);
+	if (hsk->sock.sk_protocol == IPPROTO_UDP)
+		h->seg.offset = htonl(offset);
+	else
+		h->seg.offset = htonl(-1);
 #else /* See strip.py */
 	h->seg.offset = htonl(offset);
 #endif /* See strip.py */
@@ -219,7 +262,8 @@ struct sk_buff *homa_tx_data_pkt_alloc(struct homa_rpc *rpc,
 	homa_info->rpc = rpc;
 
 #ifndef __STRIP__ /* See strip.py */
-	if (segs > 1 && hsk->sock.sk_protocol != IPPROTO_TCP) {
+	if (segs > 1 && hsk->sock.sk_protocol != IPPROTO_TCP
+		    && hsk->sock.sk_protocol != IPPROTO_UDP) {
 #else /* See strip.py */
 	if (segs > 1) {
 #endif /* See strip.py */
@@ -322,10 +366,13 @@ int homa_message_out_fill(struct homa_rpc *rpc, struct iov_iter *iter, int xmit)
 	 * if no hijacking).
 	 */
 	if (rpc->hsk->sock.sk_protocol == IPPROTO_TCP) {
-		/* Hijacking */
+		/* TCP Hijacking */
 		segs_per_gso = gso_size - rpc->hsk->ip_header_length
 				- sizeof(struct homa_data_hdr);
 		do_div(segs_per_gso, max_seg_data);
+	} else if (rpc->hsk->sock.sk_protocol == IPPROTO_UDP) {
+		/* UDP Hijacking: one segment per GSO */
+		segs_per_gso = 1;
 	} else {
 		/* No hijacking */
 		segs_per_gso = gso_size - rpc->hsk->ip_header_length -
@@ -509,12 +556,18 @@ int __homa_xmit_control(void *contents, size_t length, struct homa_peer *peer,
 	homa_set_doff(skb, length);
 #ifndef __STRIP__ /* See strip.py */
 	if (hsk->inet.sk.sk_family == AF_INET6) {
-		homa_set_hijack(skb, peer, true);
+		if (hsk->sock.sk_protocol == IPPROTO_UDP)
+			homa_set_udp_hijack(skb, peer, true);
+		else
+			homa_set_hijack(skb, peer, true);
 		result = ip6_xmit(&hsk->inet.sk, skb, &peer->flow.u.ip6, 0,
 				  NULL, hsk->homa->priority_map[priority] << 5,
 				  0);
 	} else {
-		homa_set_hijack(skb, peer, false);
+		if (hsk->sock.sk_protocol == IPPROTO_UDP)
+			homa_set_udp_hijack(skb, peer, false);
+		else
+			homa_set_hijack(skb, peer, false);
 
 		/* This will find its way to the DSCP field in the IPv4 hdr. */
 		hsk->inet.tos = hsk->homa->priority_map[priority] << 5;
@@ -721,7 +774,10 @@ void __homa_xmit_data(struct sk_buff *skb, struct homa_rpc *rpc)
 			   tt_addr(rpc->peer->addr), rpc->id,
 			   homa_get_skb_info(skb)->offset);
 #ifndef __STRIP__ /* See strip.py */
-		homa_set_hijack(skb, rpc->peer, true);
+		if (rpc->hsk->sock.sk_protocol == IPPROTO_UDP)
+			homa_set_udp_hijack(skb, rpc->peer, true);
+		else
+			homa_set_hijack(skb, rpc->peer, true);
 		err = ip6_xmit(&rpc->hsk->inet.sk, skb, &rpc->peer->flow.u.ip6,
 			       0, NULL,
 			       rpc->hsk->homa->priority_map[priority] << 5, 0);
@@ -736,7 +792,10 @@ void __homa_xmit_data(struct sk_buff *skb, struct homa_rpc *rpc)
 			   homa_get_skb_info(skb)->offset);
 
 #ifndef __STRIP__ /* See strip.py */
-		homa_set_hijack(skb, rpc->peer, false);
+		if (rpc->hsk->sock.sk_protocol == IPPROTO_UDP)
+			homa_set_udp_hijack(skb, rpc->peer, false);
+		else
+			homa_set_hijack(skb, rpc->peer, false);
 		rpc->hsk->inet.tos =
 				rpc->hsk->homa->priority_map[priority] << 5;
 		err = ip_queue_xmit(&rpc->hsk->inet.sk, skb, &rpc->peer->flow);

homa_plumbing.c

@@ -257,6 +257,13 @@ static struct ctl_table homa_ctl_table[] = {
 		.mode		= 0644,
 		.proc_handler	= homa_dointvec
 	},
+	{
+		.procname	= "hijack_udp",
+		.data		= OFFSET(hijack_udp),
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= homa_dointvec
+	},
 	{
 		.procname	= "link_mbps",
 		.data		= OFFSET(link_mbps),
@@ -642,6 +649,7 @@ int __init homa_load(void)
 
 #ifndef __STRIP__ /* See strip.py */
 	homa_gro_hook_tcp();
+	homa_gro_hook_udp();
 #endif /* See strip.py */
 #ifndef __UPSTREAM__ /* See strip.py */
 	tt_set_temp(homa->temp);
@@ -695,6 +703,7 @@ void __exit homa_unload(void)
 
 #ifndef __STRIP__ /* See strip.py */
 	homa_gro_unhook_tcp();
+	homa_gro_unhook_udp();
 #endif /* See strip.py */
 	if (timer_kthread) {
 		timer_thread_exit = 1;

homa_qdisc.c

@@ -557,15 +557,17 @@ bool homa_qdisc_can_bypass(struct sk_buff *skb, struct homa_qdisc *q)
 	bool result;
 	int element;
 
-	/* Collect information from skb. If it isn't a TCP packet then
+	/* Collect information from skb. If it isn't a TCP or UDP packet then
 	 * reordering constraints are unknown so deny reordering.
 	 */
 	if (skb->protocol == htons(ETH_P_IP)) {
-		if (ip_hdr(skb)->protocol != IPPROTO_TCP)
+		if (ip_hdr(skb)->protocol != IPPROTO_TCP
+				&& ip_hdr(skb)->protocol != IPPROTO_UDP)
 			return false;
 		daddr = ip_hdr(skb)->daddr;
 	} else if (skb->protocol == htons(ETH_P_IPV6)) {
-		if (ipv6_hdr(skb)->nexthdr != IPPROTO_TCP)
+		if (ipv6_hdr(skb)->nexthdr != IPPROTO_TCP
+				&& ipv6_hdr(skb)->nexthdr != IPPROTO_UDP)
 			return false;
 		daddr = ipv6_hdr(skb)->daddr.in6_u.u6_addr32[0] ^
 		        ipv6_hdr(skb)->daddr.in6_u.u6_addr32[1] ^
@@ -591,11 +593,13 @@ bool homa_qdisc_can_bypass(struct sk_buff *skb, struct homa_qdisc *q)
 	skb_queue_walk(&q->deferred_tcp, skb2) {
 		element++;
 		if (skb2->protocol == htons(ETH_P_IP)) {
-			if (ip_hdr(skb2)->protocol != IPPROTO_TCP)
+			if (ip_hdr(skb2)->protocol != IPPROTO_TCP
+					&& ip_hdr(skb2)->protocol != IPPROTO_UDP)
 				continue;
 			daddr2 = ip_hdr(skb2)->daddr;
 		} else if (skb2->protocol == htons(ETH_P_IPV6)) {
-			if (ipv6_hdr(skb2)->nexthdr != IPPROTO_TCP)
+			if (ipv6_hdr(skb2)->nexthdr != IPPROTO_TCP
+					&& ipv6_hdr(skb2)->nexthdr != IPPROTO_UDP)
 				continue;
 			daddr2 = ipv6_hdr(skb2)->daddr.in6_u.u6_addr32[0] ^
 				 ipv6_hdr(skb2)->daddr.in6_u.u6_addr32[1] ^
@@ -757,8 +761,9 @@ int homa_qdisc_xmit_deferred_tcp(struct homa_qdisc_dev *qdev)
 
 	pkt_len = qdisc_pkt_len(skb);
 	homa_qdisc_update_link_idle(qdev, pkt_len, -1);
-	if (ip_hdr(skb)->protocol == IPPROTO_TCP)
-		tt_record_tcp("homa_qdisc_pacer requeued TCP packet from "
+	if (ip_hdr(skb)->protocol == IPPROTO_TCP
+			|| ip_hdr(skb)->protocol == IPPROTO_UDP)
+		tt_record_tcp("homa_qdisc_pacer requeued TCP/UDP packet from "
 			"0x%x to 0x%x, data bytes %d, seq/ack %u",
 			skb, ip_hdr(skb)->saddr, ip_hdr(skb)->daddr);
 	homa_qdisc_schedule_skb(skb, q->qdisc);

homa_sock.c

@@ -164,6 +164,8 @@ int homa_sock_init(struct homa_sock *hsk)
 #ifndef __STRIP__ /* See strip.py */
 	if (homa->hijack_tcp)
 		hsk->sock.sk_protocol = IPPROTO_TCP;
+	else if (homa->hijack_udp)
+		hsk->sock.sk_protocol = IPPROTO_UDP;
 #endif /* See strip.py */
 
 	/* Do things requiring memory allocation before locking the socket,