blob: da262edc3e95e71f2aa8a52026f9951452b6ec61 [file] [log] [blame]
Dmitry Fleytmane263cd42013-03-09 11:21:05 +02001/*
Dmitry Fleytman605d52e2016-06-01 11:23:39 +03002 * QEMU TX packets abstractions
Dmitry Fleytmane263cd42013-03-09 11:21:05 +02003 *
4 * Copyright (c) 2012 Ravello Systems LTD (http://ravellosystems.com)
5 *
6 * Developed by Daynix Computing LTD (http://www.daynix.com)
7 *
8 * Authors:
9 * Dmitry Fleytman <dmitry@daynix.com>
10 * Tamir Shomer <tamirs@daynix.com>
11 * Yan Vugenfirer <yan@daynix.com>
12 *
13 * This work is licensed under the terms of the GNU GPL, version 2 or later.
14 * See the COPYING file in the top-level directory.
15 *
16 */
17
Paolo Bonzinie9abfcb2016-06-06 18:56:37 +020018#include "qemu/osdep.h"
Dmitry Fleytman605d52e2016-06-01 11:23:39 +030019#include "net_tx_pkt.h"
Dmitry Fleytmane263cd42013-03-09 11:21:05 +020020#include "net/eth.h"
Dmitry Fleytmane263cd42013-03-09 11:21:05 +020021#include "net/checksum.h"
22#include "net/tap.h"
23#include "net/net.h"
Dmitry Fleytman11171012016-06-01 11:23:42 +030024#include "hw/pci/pci.h"
Dmitry Fleytmane263cd42013-03-09 11:21:05 +020025
26enum {
Dmitry Fleytman605d52e2016-06-01 11:23:39 +030027 NET_TX_PKT_VHDR_FRAG = 0,
28 NET_TX_PKT_L2HDR_FRAG,
29 NET_TX_PKT_L3HDR_FRAG,
30 NET_TX_PKT_PL_START_FRAG
Dmitry Fleytmane263cd42013-03-09 11:21:05 +020031};
32
33/* TX packet private context */
Dmitry Fleytman605d52e2016-06-01 11:23:39 +030034struct NetTxPkt {
Dmitry Fleytman11171012016-06-01 11:23:42 +030035 PCIDevice *pci_dev;
36
Dmitry Fleytmane263cd42013-03-09 11:21:05 +020037 struct virtio_net_hdr virt_hdr;
38 bool has_virt_hdr;
39
40 struct iovec *raw;
41 uint32_t raw_frags;
42 uint32_t max_raw_frags;
43
44 struct iovec *vec;
45
46 uint8_t l2_hdr[ETH_MAX_L2_HDR_LEN];
Dmitry Fleytmaneb700022016-06-01 11:23:41 +030047 uint8_t l3_hdr[ETH_MAX_IP_DGRAM_LEN];
Dmitry Fleytmane263cd42013-03-09 11:21:05 +020048
49 uint32_t payload_len;
50
51 uint32_t payload_frags;
52 uint32_t max_payload_frags;
53
54 uint16_t hdr_len;
55 eth_pkt_types_e packet_type;
56 uint8_t l4proto;
Dmitry Fleytmaneb700022016-06-01 11:23:41 +030057
58 bool is_loopback;
Dmitry Fleytmane263cd42013-03-09 11:21:05 +020059};
60
Dmitry Fleytman11171012016-06-01 11:23:42 +030061void net_tx_pkt_init(struct NetTxPkt **pkt, PCIDevice *pci_dev,
62 uint32_t max_frags, bool has_virt_hdr)
Dmitry Fleytmane263cd42013-03-09 11:21:05 +020063{
Dmitry Fleytman605d52e2016-06-01 11:23:39 +030064 struct NetTxPkt *p = g_malloc0(sizeof *p);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +020065
Dmitry Fleytman11171012016-06-01 11:23:42 +030066 p->pci_dev = pci_dev;
67
Li Qiang47882fa2016-08-16 16:58:01 +053068 p->vec = g_new(struct iovec, max_frags + NET_TX_PKT_PL_START_FRAG);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +020069
Li Qiang47882fa2016-08-16 16:58:01 +053070 p->raw = g_new(struct iovec, max_frags);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +020071
72 p->max_payload_frags = max_frags;
73 p->max_raw_frags = max_frags;
74 p->has_virt_hdr = has_virt_hdr;
Dmitry Fleytman605d52e2016-06-01 11:23:39 +030075 p->vec[NET_TX_PKT_VHDR_FRAG].iov_base = &p->virt_hdr;
76 p->vec[NET_TX_PKT_VHDR_FRAG].iov_len =
Dmitry Fleytmane263cd42013-03-09 11:21:05 +020077 p->has_virt_hdr ? sizeof p->virt_hdr : 0;
Dmitry Fleytman605d52e2016-06-01 11:23:39 +030078 p->vec[NET_TX_PKT_L2HDR_FRAG].iov_base = &p->l2_hdr;
Dmitry Fleytmaneb700022016-06-01 11:23:41 +030079 p->vec[NET_TX_PKT_L3HDR_FRAG].iov_base = &p->l3_hdr;
Dmitry Fleytmane263cd42013-03-09 11:21:05 +020080
81 *pkt = p;
82}
83
Dmitry Fleytman605d52e2016-06-01 11:23:39 +030084void net_tx_pkt_uninit(struct NetTxPkt *pkt)
Dmitry Fleytmane263cd42013-03-09 11:21:05 +020085{
86 if (pkt) {
87 g_free(pkt->vec);
88 g_free(pkt->raw);
89 g_free(pkt);
90 }
91}
92
Dmitry Fleytmaneb700022016-06-01 11:23:41 +030093void net_tx_pkt_update_ip_hdr_checksum(struct NetTxPkt *pkt)
94{
95 uint16_t csum;
96 assert(pkt);
97 struct ip_header *ip_hdr;
98 ip_hdr = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base;
99
100 ip_hdr->ip_len = cpu_to_be16(pkt->payload_len +
101 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len);
102
103 ip_hdr->ip_sum = 0;
104 csum = net_raw_checksum((uint8_t *)ip_hdr,
105 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len);
106 ip_hdr->ip_sum = cpu_to_be16(csum);
107}
108
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300109void net_tx_pkt_update_ip_checksums(struct NetTxPkt *pkt)
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200110{
111 uint16_t csum;
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300112 uint32_t cntr, cso;
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200113 assert(pkt);
114 uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN;
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300115 void *ip_hdr = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base;
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200116
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300117 if (pkt->payload_len + pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len >
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200118 ETH_MAX_IP_DGRAM_LEN) {
119 return;
120 }
121
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300122 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 ||
123 gso_type == VIRTIO_NET_HDR_GSO_UDP) {
124 /* Calculate IP header checksum */
125 net_tx_pkt_update_ip_hdr_checksum(pkt);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200126
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300127 /* Calculate IP pseudo header checksum */
128 cntr = eth_calc_ip4_pseudo_hdr_csum(ip_hdr, pkt->payload_len, &cso);
129 csum = cpu_to_be16(~net_checksum_finish(cntr));
130 } else if (gso_type == VIRTIO_NET_HDR_GSO_TCPV6) {
131 /* Calculate IP pseudo header checksum */
132 cntr = eth_calc_ip6_pseudo_hdr_csum(ip_hdr, pkt->payload_len,
133 IP_PROTO_TCP, &cso);
134 csum = cpu_to_be16(~net_checksum_finish(cntr));
135 } else {
136 return;
137 }
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200138
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300139 iov_from_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG], pkt->payload_frags,
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200140 pkt->virt_hdr.csum_offset, &csum, sizeof(csum));
141}
142
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300143static void net_tx_pkt_calculate_hdr_len(struct NetTxPkt *pkt)
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200144{
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300145 pkt->hdr_len = pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len +
146 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len;
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200147}
148
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300149static bool net_tx_pkt_parse_headers(struct NetTxPkt *pkt)
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200150{
151 struct iovec *l2_hdr, *l3_hdr;
152 size_t bytes_read;
153 size_t full_ip6hdr_len;
154 uint16_t l3_proto;
155
156 assert(pkt);
157
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300158 l2_hdr = &pkt->vec[NET_TX_PKT_L2HDR_FRAG];
159 l3_hdr = &pkt->vec[NET_TX_PKT_L3HDR_FRAG];
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200160
161 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, 0, l2_hdr->iov_base,
162 ETH_MAX_L2_HDR_LEN);
Dana Rubina7278b32015-08-18 12:45:55 +0300163 if (bytes_read < sizeof(struct eth_header)) {
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200164 l2_hdr->iov_len = 0;
165 return false;
Dana Rubina7278b32015-08-18 12:45:55 +0300166 }
167
168 l2_hdr->iov_len = sizeof(struct eth_header);
169 switch (be16_to_cpu(PKT_GET_ETH_HDR(l2_hdr->iov_base)->h_proto)) {
170 case ETH_P_VLAN:
171 l2_hdr->iov_len += sizeof(struct vlan_header);
172 break;
173 case ETH_P_DVLAN:
174 l2_hdr->iov_len += 2 * sizeof(struct vlan_header);
175 break;
176 }
177
178 if (bytes_read < l2_hdr->iov_len) {
179 l2_hdr->iov_len = 0;
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300180 l3_hdr->iov_len = 0;
181 pkt->packet_type = ETH_PKT_UCAST;
Dana Rubina7278b32015-08-18 12:45:55 +0300182 return false;
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300183 } else {
184 l2_hdr->iov_len = ETH_MAX_L2_HDR_LEN;
185 l2_hdr->iov_len = eth_get_l2_hdr_length(l2_hdr->iov_base);
186 pkt->packet_type = get_eth_packet_type(l2_hdr->iov_base);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200187 }
188
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300189 l3_proto = eth_get_l3_proto(l2_hdr, 1, l2_hdr->iov_len);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200190
191 switch (l3_proto) {
192 case ETH_P_IP:
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200193 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len,
194 l3_hdr->iov_base, sizeof(struct ip_header));
195
196 if (bytes_read < sizeof(struct ip_header)) {
197 l3_hdr->iov_len = 0;
198 return false;
199 }
200
201 l3_hdr->iov_len = IP_HDR_GET_LEN(l3_hdr->iov_base);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200202
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300203 if (l3_hdr->iov_len < sizeof(struct ip_header)) {
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200204 l3_hdr->iov_len = 0;
205 return false;
206 }
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300207
Marc-André Lureau4f51e1d2018-02-09 20:03:40 +0100208 pkt->l4proto = IP_HDR_GET_P(l3_hdr->iov_base);
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300209
210 if (IP_HDR_GET_LEN(l3_hdr->iov_base) != sizeof(struct ip_header)) {
211 /* copy optional IPv4 header data if any*/
212 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags,
213 l2_hdr->iov_len + sizeof(struct ip_header),
214 l3_hdr->iov_base + sizeof(struct ip_header),
215 l3_hdr->iov_len - sizeof(struct ip_header));
216 if (bytes_read < l3_hdr->iov_len - sizeof(struct ip_header)) {
217 l3_hdr->iov_len = 0;
218 return false;
219 }
220 }
221
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200222 break;
223
224 case ETH_P_IPV6:
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300225 {
226 eth_ip6_hdr_info hdrinfo;
227
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200228 if (!eth_parse_ipv6_hdr(pkt->raw, pkt->raw_frags, l2_hdr->iov_len,
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300229 &hdrinfo)) {
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200230 l3_hdr->iov_len = 0;
231 return false;
232 }
233
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300234 pkt->l4proto = hdrinfo.l4proto;
235 full_ip6hdr_len = hdrinfo.full_hdr_len;
236
237 if (full_ip6hdr_len > ETH_MAX_IP_DGRAM_LEN) {
238 l3_hdr->iov_len = 0;
239 return false;
240 }
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200241
242 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len,
243 l3_hdr->iov_base, full_ip6hdr_len);
244
245 if (bytes_read < full_ip6hdr_len) {
246 l3_hdr->iov_len = 0;
247 return false;
248 } else {
249 l3_hdr->iov_len = full_ip6hdr_len;
250 }
251 break;
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300252 }
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200253 default:
254 l3_hdr->iov_len = 0;
255 break;
256 }
257
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300258 net_tx_pkt_calculate_hdr_len(pkt);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200259 return true;
260}
261
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300262static void net_tx_pkt_rebuild_payload(struct NetTxPkt *pkt)
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200263{
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300264 pkt->payload_len = iov_size(pkt->raw, pkt->raw_frags) - pkt->hdr_len;
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300265 pkt->payload_frags = iov_copy(&pkt->vec[NET_TX_PKT_PL_START_FRAG],
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200266 pkt->max_payload_frags,
267 pkt->raw, pkt->raw_frags,
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300268 pkt->hdr_len, pkt->payload_len);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200269}
270
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300271bool net_tx_pkt_parse(struct NetTxPkt *pkt)
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200272{
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300273 if (net_tx_pkt_parse_headers(pkt)) {
274 net_tx_pkt_rebuild_payload(pkt);
275 return true;
276 } else {
277 return false;
278 }
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200279}
280
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300281struct virtio_net_hdr *net_tx_pkt_get_vhdr(struct NetTxPkt *pkt)
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200282{
283 assert(pkt);
284 return &pkt->virt_hdr;
285}
286
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300287static uint8_t net_tx_pkt_get_gso_type(struct NetTxPkt *pkt,
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200288 bool tso_enable)
289{
290 uint8_t rc = VIRTIO_NET_HDR_GSO_NONE;
291 uint16_t l3_proto;
292
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300293 l3_proto = eth_get_l3_proto(&pkt->vec[NET_TX_PKT_L2HDR_FRAG], 1,
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300294 pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200295
296 if (!tso_enable) {
297 goto func_exit;
298 }
299
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300300 rc = eth_get_gso_type(l3_proto, pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base,
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200301 pkt->l4proto);
302
303func_exit:
304 return rc;
305}
306
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300307void net_tx_pkt_build_vheader(struct NetTxPkt *pkt, bool tso_enable,
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200308 bool csum_enable, uint32_t gso_size)
309{
310 struct tcp_hdr l4hdr;
311 assert(pkt);
312
313 /* csum has to be enabled if tso is. */
314 assert(csum_enable || !tso_enable);
315
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300316 pkt->virt_hdr.gso_type = net_tx_pkt_get_gso_type(pkt, tso_enable);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200317
318 switch (pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
319 case VIRTIO_NET_HDR_GSO_NONE:
320 pkt->virt_hdr.hdr_len = 0;
321 pkt->virt_hdr.gso_size = 0;
322 break;
323
324 case VIRTIO_NET_HDR_GSO_UDP:
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300325 pkt->virt_hdr.gso_size = gso_size;
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200326 pkt->virt_hdr.hdr_len = pkt->hdr_len + sizeof(struct udp_header);
327 break;
328
329 case VIRTIO_NET_HDR_GSO_TCPV4:
330 case VIRTIO_NET_HDR_GSO_TCPV6:
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300331 iov_to_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG], pkt->payload_frags,
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200332 0, &l4hdr, sizeof(l4hdr));
333 pkt->virt_hdr.hdr_len = pkt->hdr_len + l4hdr.th_off * sizeof(uint32_t);
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300334 pkt->virt_hdr.gso_size = gso_size;
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200335 break;
336
337 default:
Stefan Weildfc6f862013-07-25 18:21:28 +0200338 g_assert_not_reached();
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200339 }
340
341 if (csum_enable) {
342 switch (pkt->l4proto) {
343 case IP_PROTO_TCP:
344 pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
345 pkt->virt_hdr.csum_start = pkt->hdr_len;
346 pkt->virt_hdr.csum_offset = offsetof(struct tcp_hdr, th_sum);
347 break;
348 case IP_PROTO_UDP:
349 pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
350 pkt->virt_hdr.csum_start = pkt->hdr_len;
351 pkt->virt_hdr.csum_offset = offsetof(struct udp_hdr, uh_sum);
352 break;
353 default:
354 break;
355 }
356 }
357}
358
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300359void net_tx_pkt_setup_vlan_header_ex(struct NetTxPkt *pkt,
360 uint16_t vlan, uint16_t vlan_ethtype)
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200361{
362 bool is_new;
363 assert(pkt);
364
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300365 eth_setup_vlan_headers_ex(pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_base,
366 vlan, vlan_ethtype, &is_new);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200367
368 /* update l2hdrlen */
369 if (is_new) {
370 pkt->hdr_len += sizeof(struct vlan_header);
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300371 pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len +=
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200372 sizeof(struct vlan_header);
373 }
374}
375
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300376bool net_tx_pkt_add_raw_fragment(struct NetTxPkt *pkt, hwaddr pa,
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200377 size_t len)
378{
379 hwaddr mapped_len = 0;
380 struct iovec *ventry;
381 assert(pkt);
Mauro Matteo Cascella035e69b2020-08-01 18:42:38 +0200382
383 if (pkt->raw_frags >= pkt->max_raw_frags) {
384 return false;
385 }
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200386
387 if (!len) {
388 return true;
389 }
390
391 ventry = &pkt->raw[pkt->raw_frags];
392 mapped_len = len;
393
Dmitry Fleytman11171012016-06-01 11:23:42 +0300394 ventry->iov_base = pci_dma_map(pkt->pci_dev, pa,
395 &mapped_len, DMA_DIRECTION_TO_DEVICE);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200396
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300397 if ((ventry->iov_base != NULL) && (len == mapped_len)) {
398 ventry->iov_len = mapped_len;
399 pkt->raw_frags++;
400 return true;
401 } else {
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200402 return false;
403 }
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300404}
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200405
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300406bool net_tx_pkt_has_fragments(struct NetTxPkt *pkt)
407{
408 return pkt->raw_frags > 0;
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200409}
410
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300411eth_pkt_types_e net_tx_pkt_get_packet_type(struct NetTxPkt *pkt)
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200412{
413 assert(pkt);
414
415 return pkt->packet_type;
416}
417
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300418size_t net_tx_pkt_get_total_len(struct NetTxPkt *pkt)
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200419{
420 assert(pkt);
421
422 return pkt->hdr_len + pkt->payload_len;
423}
424
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300425void net_tx_pkt_dump(struct NetTxPkt *pkt)
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200426{
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300427#ifdef NET_TX_PKT_DEBUG
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200428 assert(pkt);
429
430 printf("TX PKT: hdr_len: %d, pkt_type: 0x%X, l2hdr_len: %lu, "
431 "l3hdr_len: %lu, payload_len: %u\n", pkt->hdr_len, pkt->packet_type,
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300432 pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len,
433 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len, pkt->payload_len);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200434#endif
435}
436
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300437void net_tx_pkt_reset(struct NetTxPkt *pkt)
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200438{
439 int i;
440
441 /* no assert, as reset can be called before tx_pkt_init */
442 if (!pkt) {
443 return;
444 }
445
446 memset(&pkt->virt_hdr, 0, sizeof(pkt->virt_hdr));
447
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200448 assert(pkt->vec);
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300449
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200450 pkt->payload_len = 0;
451 pkt->payload_frags = 0;
452
453 assert(pkt->raw);
454 for (i = 0; i < pkt->raw_frags; i++) {
455 assert(pkt->raw[i].iov_base);
Dmitry Fleytman11171012016-06-01 11:23:42 +0300456 pci_dma_unmap(pkt->pci_dev, pkt->raw[i].iov_base, pkt->raw[i].iov_len,
457 DMA_DIRECTION_TO_DEVICE, 0);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200458 }
459 pkt->raw_frags = 0;
460
461 pkt->hdr_len = 0;
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200462 pkt->l4proto = 0;
463}
464
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300465static void net_tx_pkt_do_sw_csum(struct NetTxPkt *pkt)
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200466{
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300467 struct iovec *iov = &pkt->vec[NET_TX_PKT_L2HDR_FRAG];
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200468 uint32_t csum_cntr;
469 uint16_t csum = 0;
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300470 uint32_t cso;
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200471 /* num of iovec without vhdr */
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300472 uint32_t iov_len = pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - 1;
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200473 uint16_t csl;
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200474 size_t csum_offset = pkt->virt_hdr.csum_start + pkt->virt_hdr.csum_offset;
Andrew9a8d9492020-06-29 04:17:59 +0300475 uint16_t l3_proto = eth_get_l3_proto(iov, 1, iov->iov_len);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200476
477 /* Put zero to checksum field */
478 iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum);
479
480 /* Calculate L4 TCP/UDP checksum */
481 csl = pkt->payload_len;
482
Andrew9a8d9492020-06-29 04:17:59 +0300483 csum_cntr = 0;
484 cso = 0;
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200485 /* add pseudo header to csum */
Andrew9a8d9492020-06-29 04:17:59 +0300486 if (l3_proto == ETH_P_IP) {
487 csum_cntr = eth_calc_ip4_pseudo_hdr_csum(
488 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base,
489 csl, &cso);
490 } else if (l3_proto == ETH_P_IPV6) {
491 csum_cntr = eth_calc_ip6_pseudo_hdr_csum(
492 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base,
493 csl, pkt->l4proto, &cso);
494 }
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300495
496 /* data checksum */
497 csum_cntr +=
498 net_checksum_add_iov(iov, iov_len, pkt->virt_hdr.csum_start, csl, cso);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200499
500 /* Put the checksum obtained into the packet */
Ed Swierk0dacea92017-11-16 06:06:06 -0800501 csum = cpu_to_be16(net_checksum_finish_nozero(csum_cntr));
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200502 iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum);
503}
504
505enum {
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300506 NET_TX_PKT_FRAGMENT_L2_HDR_POS = 0,
507 NET_TX_PKT_FRAGMENT_L3_HDR_POS,
508 NET_TX_PKT_FRAGMENT_HEADER_NUM
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200509};
510
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300511#define NET_MAX_FRAG_SG_LIST (64)
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200512
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300513static size_t net_tx_pkt_fetch_fragment(struct NetTxPkt *pkt,
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200514 int *src_idx, size_t *src_offset, struct iovec *dst, int *dst_idx)
515{
516 size_t fetched = 0;
517 struct iovec *src = pkt->vec;
518
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300519 *dst_idx = NET_TX_PKT_FRAGMENT_HEADER_NUM;
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200520
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300521 while (fetched < IP_FRAG_ALIGN_SIZE(pkt->virt_hdr.gso_size)) {
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200522
523 /* no more place in fragment iov */
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300524 if (*dst_idx == NET_MAX_FRAG_SG_LIST) {
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200525 break;
526 }
527
528 /* no more data in iovec */
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300529 if (*src_idx == (pkt->payload_frags + NET_TX_PKT_PL_START_FRAG)) {
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200530 break;
531 }
532
533
534 dst[*dst_idx].iov_base = src[*src_idx].iov_base + *src_offset;
535 dst[*dst_idx].iov_len = MIN(src[*src_idx].iov_len - *src_offset,
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300536 IP_FRAG_ALIGN_SIZE(pkt->virt_hdr.gso_size) - fetched);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200537
538 *src_offset += dst[*dst_idx].iov_len;
539 fetched += dst[*dst_idx].iov_len;
540
541 if (*src_offset == src[*src_idx].iov_len) {
542 *src_offset = 0;
543 (*src_idx)++;
544 }
545
546 (*dst_idx)++;
547 }
548
549 return fetched;
550}
551
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300552static inline void net_tx_pkt_sendv(struct NetTxPkt *pkt,
553 NetClientState *nc, const struct iovec *iov, int iov_cnt)
554{
555 if (pkt->is_loopback) {
556 nc->info->receive_iov(nc, iov, iov_cnt);
557 } else {
558 qemu_sendv_packet(nc, iov, iov_cnt);
559 }
560}
561
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300562static bool net_tx_pkt_do_sw_fragmentation(struct NetTxPkt *pkt,
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200563 NetClientState *nc)
564{
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300565 struct iovec fragment[NET_MAX_FRAG_SG_LIST];
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200566 size_t fragment_len = 0;
567 bool more_frags = false;
568
569 /* some pointers for shorter code */
570 void *l2_iov_base, *l3_iov_base;
571 size_t l2_iov_len, l3_iov_len;
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300572 int src_idx = NET_TX_PKT_PL_START_FRAG, dst_idx;
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200573 size_t src_offset = 0;
574 size_t fragment_offset = 0;
575
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300576 l2_iov_base = pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_base;
577 l2_iov_len = pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len;
578 l3_iov_base = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base;
579 l3_iov_len = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len;
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200580
581 /* Copy headers */
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300582 fragment[NET_TX_PKT_FRAGMENT_L2_HDR_POS].iov_base = l2_iov_base;
583 fragment[NET_TX_PKT_FRAGMENT_L2_HDR_POS].iov_len = l2_iov_len;
584 fragment[NET_TX_PKT_FRAGMENT_L3_HDR_POS].iov_base = l3_iov_base;
585 fragment[NET_TX_PKT_FRAGMENT_L3_HDR_POS].iov_len = l3_iov_len;
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200586
587
588 /* Put as much data as possible and send */
589 do {
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300590 fragment_len = net_tx_pkt_fetch_fragment(pkt, &src_idx, &src_offset,
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200591 fragment, &dst_idx);
592
593 more_frags = (fragment_offset + fragment_len < pkt->payload_len);
594
595 eth_setup_ip4_fragmentation(l2_iov_base, l2_iov_len, l3_iov_base,
596 l3_iov_len, fragment_len, fragment_offset, more_frags);
597
598 eth_fix_ip4_checksum(l3_iov_base, l3_iov_len);
599
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300600 net_tx_pkt_sendv(pkt, nc, fragment, dst_idx);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200601
602 fragment_offset += fragment_len;
603
Prasad J Panditead315e2016-08-04 13:00:14 +0530604 } while (fragment_len && more_frags);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200605
606 return true;
607}
608
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300609bool net_tx_pkt_send(struct NetTxPkt *pkt, NetClientState *nc)
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200610{
611 assert(pkt);
612
613 if (!pkt->has_virt_hdr &&
614 pkt->virt_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300615 net_tx_pkt_do_sw_csum(pkt);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200616 }
617
618 /*
619 * Since underlying infrastructure does not support IP datagrams longer
620 * than 64K we should drop such packets and don't even try to send
621 */
622 if (VIRTIO_NET_HDR_GSO_NONE != pkt->virt_hdr.gso_type) {
623 if (pkt->payload_len >
624 ETH_MAX_IP_DGRAM_LEN -
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300625 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len) {
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200626 return false;
627 }
628 }
629
630 if (pkt->has_virt_hdr ||
631 pkt->virt_hdr.gso_type == VIRTIO_NET_HDR_GSO_NONE) {
Andrewe219d302020-07-16 06:53:24 +0300632 net_tx_pkt_fix_ip6_payload_len(pkt);
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300633 net_tx_pkt_sendv(pkt, nc, pkt->vec,
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300634 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200635 return true;
636 }
637
Dmitry Fleytman605d52e2016-06-01 11:23:39 +0300638 return net_tx_pkt_do_sw_fragmentation(pkt, nc);
Dmitry Fleytmane263cd42013-03-09 11:21:05 +0200639}
Dmitry Fleytmaneb700022016-06-01 11:23:41 +0300640
641bool net_tx_pkt_send_loopback(struct NetTxPkt *pkt, NetClientState *nc)
642{
643 bool res;
644
645 pkt->is_loopback = true;
646 res = net_tx_pkt_send(pkt, nc);
647 pkt->is_loopback = false;
648
649 return res;
650}
Andrewe219d302020-07-16 06:53:24 +0300651
652void net_tx_pkt_fix_ip6_payload_len(struct NetTxPkt *pkt)
653{
654 struct iovec *l2 = &pkt->vec[NET_TX_PKT_L2HDR_FRAG];
655 if (eth_get_l3_proto(l2, 1, l2->iov_len) == ETH_P_IPV6) {
656 struct ip6_header *ip6 = (struct ip6_header *) pkt->l3_hdr;
657 /*
658 * TODO: if qemu would support >64K packets - add jumbo option check
659 * something like that:
660 * 'if (ip6->ip6_plen == 0 && !has_jumbo_option(ip6)) {'
661 */
662 if (ip6->ip6_plen == 0) {
663 if (pkt->payload_len <= ETH_MAX_IP_DGRAM_LEN) {
664 ip6->ip6_plen = htons(pkt->payload_len);
665 }
666 /*
667 * TODO: if qemu would support >64K packets
668 * add jumbo option for packets greater then 65,535 bytes
669 */
670 }
671 }
672}