Rhel 9.5 - Added support for Hijack UDP

Makefile

@@ -36,6 +36,8 @@ endif
 
 LINUX_VERSION ?= $(shell uname -r)
 KDIR ?= /lib/modules/$(LINUX_VERSION)/build
+#CC = gcc-8
+CC = gcc
 
 LINUX_SRC_DIR ?= ../net-next
 
@@ -110,3 +112,6 @@ printClean-%:
 	$(MAKE) -C $(KDIR) M=$(shell pwd) $@
 
 endif
+
+# Prevents warnings related to the __init annotation for homa_load.
+CFLAGS_homa_plumbing.o += -Wno-missing-attributes

homa_devel.c

@@ -1283,6 +1283,8 @@ void homa_validate_rbtree(struct rb_node *node, int depth, char *message)
 		tt_printk();
 		BUG_ON(1);
 	}
+#else
+	return;
 #endif /* __UNIT_TEST__ */
 }
 #endif /* See strip.py */
@@ -1303,11 +1305,9 @@ int homa_tcp_checksum(struct sk_buff *skb)
     data_csum = skb_checksum(skb, skb_transport_offset(skb), tcp_len, 0);
 
     if (skb_is_ipv6(skb)) {
-        const struct ipv6hdr *ip6h = ipv6_hdr(skb);
-
         // Fold the manual sum with the IPv6 pseudo-header
-        return csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr, tcp_len,
-                               IPPROTO_TCP, data_csum);
+        return csum_ipv6_magic(&ipv6_hdr(skb)->saddr, &ipv6_hdr(skb)->daddr,
+			       tcp_len, IPPROTO_TCP, data_csum);
     } else {
         const struct iphdr *iph = ip_hdr(skb);
 

homa_grant.c

@@ -118,6 +118,7 @@ static struct ctl_table grant_ctl_table[] = {
 		.mode		= 0644,
 		.proc_handler	= homa_grant_dointvec
 	},
+	{}
 };
 #endif /* See strip.py */
 
@@ -1205,7 +1206,7 @@ void homa_grant_update_sysctl_deps(struct homa_grant *grant)
  *
  * Return: 0 for success, nonzero for error.
  */
-int homa_grant_dointvec(const struct ctl_table *table, int write,
+int homa_grant_dointvec(struct ctl_table *table, int write,
 			void *buffer, size_t *lenp, loff_t *ppos)
 {
 	struct ctl_table table_copy;

homa_grant.h

@@ -243,7 +243,7 @@ void     homa_grant_adjust_peer(struct homa_grant *grant,
 void     homa_grant_check_fifo(struct homa_grant *grant);
 void     homa_grant_check_needy(struct homa_grant *grant);
 void     homa_grant_check_rpc(struct homa_rpc *rpc);
-int      homa_grant_dointvec(const struct ctl_table *table, int write,
+int      homa_grant_dointvec(struct ctl_table *table, int write,
 			     void *buffer, size_t *lenp, loff_t *ppos);
 void     homa_grant_find_oldest(struct homa_grant *grant);
 int      homa_grant_find_victim(struct homa_grant *grant, struct homa_rpc *rpc);

homa_impl.h

@@ -43,11 +43,10 @@
 #include <linux/vmalloc.h>
 #include <net/icmp.h>
 #include <net/ip.h>
+#include <net/ip6_route.h>
 #include <net/netns/generic.h>
 #include <net/protocol.h>
 #include <net/inet_common.h>
-#include <net/gro.h>
-#include <net/rps.h>
 
 #ifndef __UPSTREAM__ /* See strip.py */
 #include "homa.h"
@@ -81,6 +80,11 @@ struct homa_peer;
 struct homa_rpc;
 struct homa_sock;
 
+/* Features not present in all kernels: */
+#ifndef __cond_acquires
+#define __cond_acquires(x)
+#endif
+
 #ifndef __STRIP__ /* See strip.py */
 #include "timetrace.h"
 #include "homa_metrics.h"
@@ -723,7 +727,7 @@ int      homa_net_start(struct net *net);
 __poll_t homa_poll(struct file *file, struct socket *sock,
 		   struct poll_table_struct *wait);
 int      homa_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
-		      int flags, int *addr_len);
+		      int flags, int noblock, int *addr_len);
 void     homa_request_retrans(struct homa_rpc *rpc);
 void     homa_resend_pkt(struct sk_buff *skb, struct homa_rpc *rpc,
 			 struct homa_sock *hsk);
@@ -755,7 +759,7 @@ void     homa_xmit_unknown(struct sk_buff *skb, struct homa_sock *hsk);
 
 #ifndef __STRIP__ /* See strip.py */
 void     homa_cutoffs_pkt(struct sk_buff *skb, struct homa_sock *hsk);
-int      homa_dointvec(const struct ctl_table *table, int write,
+int      homa_dointvec(struct ctl_table *table, int write,
 		       void *buffer, size_t *lenp, loff_t *ppos);
 void     homa_incoming_sysctl_changed(struct homa *homa);
 int      homa_ioc_abort(struct socket *sock, unsigned long arg);
@@ -764,7 +768,7 @@ int      homa_message_in_init(struct homa_rpc *rpc, int length,
 void     homa_prios_changed(struct homa *homa);
 void     homa_resend_data(struct homa_rpc *rpc, int start, int end,
 			  int priority);
-int      homa_sysctl_softirq_cores(const struct ctl_table *table,
+int      homa_sysctl_softirq_cores(struct ctl_table *table,
 				   int write, void *buffer, size_t *lenp,
 				   loff_t *ppos);
 int      homa_unsched_priority(struct homa *homa, struct homa_peer *peer,

homa_incoming.c

@@ -165,16 +165,14 @@ void homa_add_packet(struct homa_rpc *rpc, struct sk_buff *skb)
 {
 	struct homa_data_hdr *h = (struct homa_data_hdr *)skb->data;
 	struct homa_gap *gap, *dummy, *gap2;
-	u32 start = ntohl(h->seg.offset);
-	u32 length = homa_data_len(skb);
-	enum skb_drop_reason reason;
-	u32 end = start + length;
+	int start = ntohl(h->seg.offset);
+	int length = homa_data_len(skb);
+	int end = start + length;
 
 	if (start >= rpc->msgin.length ||
 	    length > (rpc->msgin.length - start)) {
 		tt_record3("Packet extended past message end; id %d, offset %d, length %d",
 			   rpc->id, start, length);
-		reason = SKB_DROP_REASON_PKT_TOO_BIG;
 		goto discard;
 	}
 
@@ -190,7 +188,6 @@ void homa_add_packet(struct homa_rpc *rpc, struct sk_buff *skb)
 				    rpc->msgin.recv_end, start)) {
 			tt_record2("Couldn't allocate gap for id %d (start %d): no memory",
 				   rpc->id, start);
-			reason = SKB_DROP_REASON_NOMEM;
 			goto discard;
 		}
 		tt_record3("Created new gap for id %d: start %d, end %d",
@@ -210,13 +207,11 @@ void homa_add_packet(struct homa_rpc *rpc, struct sk_buff *skb)
 			if (start < gap->start) {
 				tt_record4("Packet overlaps gap start: id %d, start %d, end %d, gap_start %d",
 					   rpc->id, start, end, gap->start);
-				reason = SKB_DROP_REASON_DUP_FRAG;
 				goto discard;
 			}
 			if (end > gap->end) {
 				tt_record4("Packet overlaps gap end: id %d, start %d, end %d, gap_end %d",
 					   rpc->id, start, end, gap->start);
-				reason = SKB_DROP_REASON_DUP_FRAG;
 				goto discard;
 			}
 			tt_record4("Increasing start for gap for id %d, old start %d, new %d, end %d",
@@ -238,7 +233,6 @@ void homa_add_packet(struct homa_rpc *rpc, struct sk_buff *skb)
 			if (end > gap->end) {
 				tt_record4("Packet overlaps gap end: id %d, start %d, end %d, gap_end %d",
 					   rpc->id, start, end, gap->start);
-				reason = SKB_DROP_REASON_DUP_FRAG;
 				goto discard;
 			}
 			tt_record4("Decreasing end for gap for id %d, old end %d, new %d, start %d",
@@ -252,7 +246,6 @@ void homa_add_packet(struct homa_rpc *rpc, struct sk_buff *skb)
 		if (!gap2) {
 			tt_record2("Couldn't allocate gap for split for id %d (start %d): no memory",
 				   rpc->id, end);
-			reason = SKB_DROP_REASON_NOMEM;
 			goto discard;
 		}
 		tt_record4("Splitting gap for id %d; old gap start %d, end %d, pkt_start %d",
@@ -262,7 +255,6 @@ void homa_add_packet(struct homa_rpc *rpc, struct sk_buff *skb)
 		goto keep;
 	}
 	/* Packet doesn't overlap any gap, so it is a duplicate. */
-	reason = SKB_DROP_REASON_DUP_FRAG;
 
 discard:
 #ifndef __STRIP__ /* See strip.py */
@@ -273,7 +265,7 @@ void homa_add_packet(struct homa_rpc *rpc, struct sk_buff *skb)
 #endif /* See strip.py */
 	tt_record4("homa_add_packet discarding packet for id %d, offset %d, length %d, retransmit %d",
 		   rpc->id, start, length, h->retransmit);
-	kfree_skb_reason(skb, reason);
+	kfree_skb(skb);
 	return;
 
 keep:
@@ -369,6 +361,7 @@ int homa_copy_to_user(struct homa_rpc *rpc)
 			int offset = ntohl(h->seg.offset);
 			int buf_bytes, chunk_size;
 			struct iov_iter iter;
+			struct iovec iov;
 			int copied = 0;
 			char __user *dst;
 
@@ -398,13 +391,12 @@ int homa_copy_to_user(struct homa_rpc *rpc)
 					}
 					chunk_size = buf_bytes;
 				}
-				error = import_ubuf(READ, dst, chunk_size,
-						    &iter);
-				if (error)
-					goto free_skbs;
+				iov.iov_base = dst;
+				iov.iov_len = chunk_size;
+				iov_iter_init(&iter, READ, &iov, 1, chunk_size);
 				error = skb_copy_datagram_iter(skbs[i],
 							       sizeof(*h) +
-							       copied,  &iter,
+							       copied, &iter,
 							       chunk_size);
 				if (error)
 					goto free_skbs;
@@ -466,8 +458,8 @@ void homa_dispatch_pkts(struct sk_buff *skb)
 	hsk = homa_sock_find(hnet, dport);
 	if (!hsk || (!homa_is_client(id) && !hsk->is_server)) {
 		if (skb_is_ipv6(skb))
-			icmp6_send(skb, ICMPV6_DEST_UNREACH,
-				   ICMPV6_PORT_UNREACH, 0, NULL, IP6CB(skb));
+			icmpv6_send(skb, ICMPV6_DEST_UNREACH,
+				    ICMPV6_PORT_UNREACH, 0);
 		else
 			icmp_send(skb, ICMP_DEST_UNREACH,
 				  ICMP_PORT_UNREACH, 0);

homa_metrics.c

@@ -10,10 +10,10 @@ DEFINE_PER_CPU(struct homa_metrics, homa_metrics);
 
 /* Describes file operations implemented for /proc/net/homa_metrics. */
 static const struct proc_ops homa_metrics_ops = {
-	.proc_open         = homa_metrics_open,
-	.proc_read         = homa_metrics_read,
-	.proc_lseek        = homa_metrics_lseek,
-	.proc_release      = homa_metrics_release,
+	.proc_open      = homa_metrics_open,
+	.proc_read      = homa_metrics_read,
+	.proc_lseek     = homa_metrics_lseek,
+	.proc_release   = homa_metrics_release,
 };
 
 /* Global information used to export metrics information through a file in

homa_offload.c

@@ -8,6 +8,9 @@
 #include "homa_offload.h"
 #include "homa_pacer.h"
 #include "homa_qdisc.h"
+#include "homa_wire.h"
+#include <net/rps.h>
+#include <net/gro.h>
 
 DEFINE_PER_CPU(struct homa_offload_core, homa_offload_core);
 
@@ -81,11 +84,11 @@ void homa_set_softirq_cpu(struct sk_buff *skb, int cpu)
 	int hash;
 
 	rcu_read_lock();
-	sock_flow_table = rcu_dereference(net_hotdata.rps_sock_flow_table);
+	sock_flow_table = rcu_dereference(rps_sock_flow_table);
 	if (sock_flow_table) {
-		hash = cpu + net_hotdata.rps_cpu_mask + 1;
+		hash = cpu + rps_cpu_mask + 1;
 		if (sock_flow_table->ents[hash] != hash) {
-			sock_flow_table = rcu_dereference(net_hotdata.rps_sock_flow_table);
+			sock_flow_table = rcu_dereference(rps_sock_flow_table);
 			sock_flow_table->ents[hash] = hash;
 		}
 		__skb_set_sw_hash(skb, hash, false);

homa_peer.c

@@ -64,6 +64,7 @@ static struct ctl_table peer_ctl_table[] = {
 		.mode		= 0644,
 		.proc_handler	= homa_peer_dointvec
 	},
+	{}
 };
 #endif /* See strip.py */
 
@@ -508,25 +509,24 @@ struct dst_entry *homa_get_dst(struct homa_peer *peer, struct homa_sock *hsk)
  */
 int homa_peer_reset_dst(struct homa_peer *peer, struct homa_sock *hsk)
 {
-	struct dst_entry *dst;
-	struct flowi flow;
+	struct dst_entry *dst, *old;
 	int result = 0;
 
 	homa_peer_lock(peer);
-	memset(&flow, 0, sizeof(flow));
+	memset(&peer->flow, 0, sizeof(peer->flow));
 	if (hsk->sock.sk_family == AF_INET) {
 		struct rtable *rt;
 
-		flowi4_init_output(&flow.u.ip4, hsk->sock.sk_bound_dev_if,
+		flowi4_init_output(&peer->flow.u.ip4, hsk->sock.sk_bound_dev_if,
 				   hsk->sock.sk_mark, hsk->inet.tos,
 				   RT_SCOPE_UNIVERSE, hsk->sock.sk_protocol, 0,
 				   ipv6_to_ipv4(peer->addr),
 				   hsk->inet.inet_saddr, 0, 0,
 				   hsk->sock.sk_uid);
 		security_sk_classify_flow(&hsk->sock,
-					  &flow.u.__fl_common);
+					 &peer->flow.u.__fl_common);
 		rt = ip_route_output_flow(sock_net(&hsk->sock),
-					  &flow.u.ip4, &hsk->sock);
+					  &peer->flow.u.ip4, &hsk->sock);
 		if (IS_ERR(rt)) {
 			result = PTR_ERR(rt);
 			INC_METRIC(peer_route_errors, 1);
@@ -536,27 +536,29 @@ int homa_peer_reset_dst(struct homa_peer *peer, struct homa_sock *hsk)
 		peer->dst_cookie = 0;
 	} else {
 		/* This code is derived from code in tcp_v6_connect. */
-		flow.u.ip6.flowi6_proto = hsk->sock.sk_protocol;
-		flow.u.ip6.daddr = peer->addr;
-		flow.u.ip6.saddr = hsk->inet.pinet6->saddr;
-		flow.u.ip6.flowlabel = ip6_make_flowinfo(hsk->inet.tos, 0);
-		flow.u.ip6.flowi6_oif = hsk->sock.sk_bound_dev_if;
-		flow.u.ip6.flowi6_mark = hsk->sock.sk_mark;
-		flow.u.ip6.fl6_dport = 0;
-		flow.u.ip6.fl6_sport = 0;
-		flow.u.ip6.flowi6_uid = hsk->sock.sk_uid;
+		peer->flow.u.ip6.flowi6_proto = hsk->sock.sk_protocol;
+		peer->flow.u.ip6.daddr = peer->addr;
+		peer->flow.u.ip6.saddr = hsk->inet.pinet6->saddr;
+		peer->flow.u.ip6.flowlabel = ip6_make_flowinfo(hsk->inet.tos,
+							       0);
+		peer->flow.u.ip6.flowi6_oif = hsk->sock.sk_bound_dev_if;
+		peer->flow.u.ip6.flowi6_mark = hsk->sock.sk_mark;
+		peer->flow.u.ip6.fl6_dport = 0;
+		peer->flow.u.ip6.fl6_sport = 0;
+		peer->flow.u.ip6.flowi6_uid = hsk->sock.sk_uid;
 		security_sk_classify_flow(&hsk->sock,
-					  &flow.u.__fl_common);
+					 &peer->flow.u.__fl_common);
 		dst = ip6_dst_lookup_flow(sock_net(&hsk->sock), &hsk->sock,
-					  &flow.u.ip6, NULL);
+						   &peer->flow.u.ip6,
+						   &peer->addr);
+
 		if (IS_ERR(dst)) {
 			result = PTR_ERR(dst);
 			INC_METRIC(peer_route_errors, 1);
 			goto done;
 		}
-		peer->dst_cookie = rt6_get_cookie(dst_rt6_info(dst));
+		peer->dst_cookie = rt6_get_cookie((struct rt6_info *)dst);
 	}
-	memcpy(&peer->flow, &flow, sizeof(flow));
 
 	/* From the standpoint of homa_get_dst, peer->dst is not updated
 	 * atomically with peer->dst_cookie, which means homa_get_dst could
@@ -565,7 +567,9 @@ int homa_peer_reset_dst(struct homa_peer *peer, struct homa_sock *hsk)
 	 * a lost packet) or a valid dst to be replaced (resulting in
 	 * unnecessary work).
 	 */
-	dst_release(rcu_replace_pointer(peer->dst, dst, true));
+	old = rcu_dereference_protected(peer->dst, lockdep_is_held(&peer->lock));
+	rcu_assign_pointer(peer->dst, dst);
+	dst_release(old);
 
 done:
 	homa_peer_unlock(peer);
@@ -722,7 +726,7 @@ void homa_peer_update_sysctl_deps(struct homa_peertab *peertab)
  *
  * Return: 0 for success, nonzero for error.
  */
-int homa_peer_dointvec(const struct ctl_table *table, int write,
+int homa_peer_dointvec(struct ctl_table *table, int write,
 		       void *buffer, size_t *lenp, loff_t *ppos)
 {
 	struct homa_peertab *peertab;

homa_peer.h

@@ -289,7 +289,7 @@ struct homa_peer
 	*homa_peer_alloc(struct homa_sock *hsk, const struct in6_addr *addr);
 struct homa_peertab
 	*homa_peer_alloc_peertab(void);
-int      homa_peer_dointvec(const struct ctl_table *table, int write,
+int      homa_peer_dointvec(struct ctl_table *table, int write,
 			    void *buffer, size_t *lenp, loff_t *ppos);
 void     homa_peer_free(struct rcu_head *head);
 void     homa_peer_free_net(struct homa_net *hnet);