Line data Source code
1 : /*
2 : * Copyright (c) 2020 Doc.ai and/or its affiliates.
3 : * Copyright (c) 2020 Cisco and/or its affiliates.
4 : * Licensed under the Apache License, Version 2.0 (the "License");
5 : * you may not use this file except in compliance with the License.
6 : * You may obtain a copy of the License at:
7 : *
8 : * http://www.apache.org/licenses/LICENSE-2.0
9 : *
10 : * Unless required by applicable law or agreed to in writing, software
11 : * distributed under the License is distributed on an "AS IS" BASIS,
12 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 : * See the License for the specific language governing permissions and
14 : * limitations under the License.
15 : */
16 :
17 : #include <vlib/vlib.h>
18 : #include <vnet/vnet.h>
19 : #include <vppinfra/error.h>
20 :
21 : #include <wireguard/wireguard.h>
22 : #include <wireguard/wireguard_send.h>
23 :
24 : #define foreach_wg_output_error \
25 : _ (NONE, "No error") \
26 : _ (PEER, "Peer error") \
27 : _ (KEYPAIR, "Keypair error") \
28 : _ (NO_BUFFERS, "No buffers") \
29 : _ (CRYPTO_ENGINE_ERROR, "crypto engine error (packet dropped)")
30 :
31 : typedef enum
32 : {
33 : #define _(sym,str) WG_OUTPUT_ERROR_##sym,
34 : foreach_wg_output_error
35 : #undef _
36 : WG_OUTPUT_N_ERROR,
37 : } wg_output_error_t;
38 :
39 : static char *wg_output_error_strings[] = {
40 : #define _(sym,string) string,
41 : foreach_wg_output_error
42 : #undef _
43 : };
44 :
45 : typedef enum
46 : {
47 : WG_OUTPUT_NEXT_ERROR,
48 : WG_OUTPUT_NEXT_HANDOFF,
49 : WG_OUTPUT_NEXT_INTERFACE_OUTPUT,
50 : WG_OUTPUT_N_NEXT,
51 : } wg_output_next_t;
52 :
53 : typedef struct
54 : {
55 : index_t peer;
56 : u8 header[sizeof (ip6_udp_header_t)];
57 : u8 is_ip4;
58 : } wg_output_tun_trace_t;
59 :
60 : typedef struct
61 : {
62 : index_t peer;
63 : u32 next_index;
64 : } wg_output_tun_post_trace_t;
65 :
66 : u8 *
67 905 : format_ip4_udp_header (u8 * s, va_list * args)
68 : {
69 905 : ip4_udp_header_t *hdr4 = va_arg (*args, ip4_udp_header_t *);
70 :
71 905 : s = format (s, "%U:$U", format_ip4_header, &hdr4->ip4, format_udp_header,
72 : &hdr4->udp);
73 905 : return (s);
74 : }
75 :
76 : u8 *
77 264 : format_ip6_udp_header (u8 *s, va_list *args)
78 : {
79 264 : ip6_udp_header_t *hdr6 = va_arg (*args, ip6_udp_header_t *);
80 :
81 264 : s = format (s, "%U:$U", format_ip6_header, &hdr6->ip6, format_udp_header,
82 : &hdr6->udp);
83 264 : return (s);
84 : }
85 :
86 : /* packet trace format function */
87 : static u8 *
88 1169 : format_wg_output_tun_trace (u8 * s, va_list * args)
89 : {
90 1169 : CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
91 1169 : CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
92 :
93 1169 : wg_output_tun_trace_t *t = va_arg (*args, wg_output_tun_trace_t *);
94 :
95 1169 : s = format (s, "peer: %d\n", t->peer);
96 1169 : s = format (s, " Encrypted packet: ");
97 :
98 1169 : s = t->is_ip4 ? format (s, "%U", format_ip4_udp_header, t->header) :
99 264 : format (s, "%U", format_ip6_udp_header, t->header);
100 1169 : return s;
101 : }
102 :
103 : /* post node - packet trace format function */
104 : static u8 *
105 0 : format_wg_output_tun_post_trace (u8 *s, va_list *args)
106 : {
107 0 : CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
108 0 : CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
109 :
110 0 : wg_output_tun_post_trace_t *t = va_arg (*args, wg_output_tun_post_trace_t *);
111 :
112 0 : s = format (s, "peer: %d\n", t->peer);
113 0 : s = format (s, " wg-post: next node index %u", t->next_index);
114 0 : return s;
115 : }
116 :
117 : static_always_inline void
118 12 : wg_output_chain_crypto (vlib_main_t *vm, wg_per_thread_data_t *ptd,
119 : vlib_buffer_t *b, vlib_buffer_t *lb, u8 *start,
120 : u32 start_len, u16 *n_ch)
121 : {
122 : vnet_crypto_op_chunk_t *ch;
123 12 : vlib_buffer_t *cb = b;
124 12 : u32 n_chunks = 1;
125 :
126 12 : vec_add2 (ptd->chunks, ch, 1);
127 12 : ch->len = start_len;
128 12 : ch->src = ch->dst = start;
129 12 : cb = vlib_get_buffer (vm, cb->next_buffer);
130 :
131 : while (1)
132 : {
133 16 : vec_add2 (ptd->chunks, ch, 1);
134 16 : n_chunks += 1;
135 16 : if (lb == cb)
136 12 : ch->len = cb->current_length - NOISE_AUTHTAG_LEN;
137 : else
138 4 : ch->len = cb->current_length;
139 :
140 16 : ch->src = ch->dst = vlib_buffer_get_current (cb);
141 :
142 16 : if (!(cb->flags & VLIB_BUFFER_NEXT_PRESENT))
143 12 : break;
144 :
145 4 : cb = vlib_get_buffer (vm, cb->next_buffer);
146 : }
147 :
148 12 : if (n_ch)
149 12 : *n_ch = n_chunks;
150 12 : }
151 :
152 : static_always_inline void
153 3355 : wg_prepare_sync_enc_op (vlib_main_t *vm, wg_per_thread_data_t *ptd,
154 : vlib_buffer_t *b, vlib_buffer_t *lb,
155 : vnet_crypto_op_t **crypto_ops, u8 *src, u32 src_len,
156 : u8 *dst, u8 *aad, u32 aad_len, u64 nonce,
157 : vnet_crypto_key_index_t key_index, u32 bi, u8 *iv)
158 : {
159 3355 : vnet_crypto_op_t _op, *op = &_op;
160 3355 : u8 src_[] = {};
161 :
162 3355 : clib_memset (iv, 0, 4);
163 3355 : clib_memcpy (iv + 4, &nonce, sizeof (nonce));
164 :
165 3355 : vec_add2_aligned (crypto_ops[0], op, 1, CLIB_CACHE_LINE_BYTES);
166 3355 : vnet_crypto_op_init (op, VNET_CRYPTO_OP_CHACHA20_POLY1305_ENC);
167 :
168 3355 : op->tag_len = NOISE_AUTHTAG_LEN;
169 3355 : op->tag = vlib_buffer_get_tail (lb) - NOISE_AUTHTAG_LEN;
170 3355 : op->key_index = key_index;
171 3355 : op->aad = aad;
172 3355 : op->aad_len = aad_len;
173 3355 : op->iv = iv;
174 3355 : op->user_data = bi;
175 :
176 3355 : if (b != lb)
177 : {
178 : /* Chained buffers */
179 12 : op->flags |= VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS;
180 12 : op->chunk_index = vec_len (ptd->chunks);
181 12 : wg_output_chain_crypto (vm, ptd, b, lb, src, src_len, &op->n_chunks);
182 : }
183 : else
184 : {
185 3343 : op->src = !src ? src_ : src;
186 3343 : op->len = src_len;
187 3343 : op->dst = dst;
188 : }
189 3355 : }
190 :
191 : static_always_inline void
192 33 : wg_output_process_chained_ops (vlib_main_t *vm, vlib_node_runtime_t *node,
193 : vnet_crypto_op_t *ops, vlib_buffer_t *b[],
194 : u16 *nexts, vnet_crypto_op_chunk_t *chunks,
195 : u16 drop_next)
196 : {
197 33 : u32 n_fail, n_ops = vec_len (ops);
198 33 : vnet_crypto_op_t *op = ops;
199 :
200 33 : if (n_ops == 0)
201 29 : return;
202 :
203 4 : n_fail = n_ops - vnet_crypto_process_chained_ops (vm, op, chunks, n_ops);
204 :
205 4 : while (n_fail)
206 : {
207 0 : ASSERT (op - ops < n_ops);
208 :
209 0 : if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED)
210 : {
211 0 : u32 bi = op->user_data;
212 0 : b[bi]->error = node->errors[WG_OUTPUT_ERROR_CRYPTO_ENGINE_ERROR];
213 0 : nexts[bi] = drop_next;
214 0 : n_fail--;
215 : }
216 0 : op++;
217 : }
218 : }
219 :
220 : static_always_inline void
221 33 : wg_output_process_ops (vlib_main_t *vm, vlib_node_runtime_t *node,
222 : vnet_crypto_op_t *ops, vlib_buffer_t *b[], u16 *nexts,
223 : u16 drop_next)
224 : {
225 33 : u32 n_fail, n_ops = vec_len (ops);
226 33 : vnet_crypto_op_t *op = ops;
227 :
228 33 : if (n_ops == 0)
229 0 : return;
230 :
231 33 : n_fail = n_ops - vnet_crypto_process_ops (vm, op, n_ops);
232 :
233 33 : while (n_fail)
234 : {
235 0 : ASSERT (op - ops < n_ops);
236 :
237 0 : if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED)
238 : {
239 0 : u32 bi = op->user_data;
240 0 : b[bi]->error = node->errors[WG_OUTPUT_ERROR_CRYPTO_ENGINE_ERROR];
241 0 : nexts[bi] = drop_next;
242 0 : n_fail--;
243 : }
244 0 : op++;
245 : }
246 : }
247 :
248 : static_always_inline void
249 20 : wg_output_tun_add_to_frame (vlib_main_t *vm, vnet_crypto_async_frame_t *f,
250 : u32 key_index, u32 crypto_len,
251 : i16 crypto_start_offset, u32 buffer_index,
252 : u16 next_node, u8 *iv, u8 *tag, u8 flags)
253 : {
254 : vnet_crypto_async_frame_elt_t *fe;
255 : u16 index;
256 :
257 20 : ASSERT (f->n_elts < VNET_CRYPTO_FRAME_SIZE);
258 :
259 20 : index = f->n_elts;
260 20 : fe = &f->elts[index];
261 20 : f->n_elts++;
262 20 : fe->key_index = key_index;
263 20 : fe->crypto_total_length = crypto_len;
264 20 : fe->crypto_start_offset = crypto_start_offset;
265 20 : fe->iv = iv;
266 20 : fe->tag = tag;
267 20 : fe->flags = flags;
268 20 : f->buffer_indices[index] = buffer_index;
269 20 : f->next_node_index[index] = next_node;
270 20 : }
271 :
272 : static_always_inline enum noise_state_crypt
273 3355 : wg_output_tun_process (vlib_main_t *vm, wg_per_thread_data_t *ptd,
274 : vlib_buffer_t *b, vlib_buffer_t *lb,
275 : vnet_crypto_op_t **crypto_ops, noise_remote_t *r,
276 : uint32_t *r_idx, uint64_t *nonce, uint8_t *src,
277 : size_t srclen, uint8_t *dst, u32 bi, u8 *iv, f64 time)
278 : {
279 : noise_keypair_t *kp;
280 3355 : enum noise_state_crypt ret = SC_FAILED;
281 :
282 3355 : if ((kp = r->r_current) == NULL)
283 0 : goto error;
284 :
285 : /* We confirm that our values are within our tolerances. We want:
286 : * - a valid keypair
287 : * - our keypair to be less than REJECT_AFTER_TIME seconds old
288 : * - our receive counter to be less than REJECT_AFTER_MESSAGES
289 : * - our send counter to be less than REJECT_AFTER_MESSAGES
290 : */
291 6710 : if (!kp->kp_valid ||
292 3355 : wg_birthdate_has_expired_opt (kp->kp_birthdate, REJECT_AFTER_TIME,
293 3355 : time) ||
294 3355 : kp->kp_ctr.c_recv >= REJECT_AFTER_MESSAGES ||
295 3355 : ((*nonce = noise_counter_send (&kp->kp_ctr)) > REJECT_AFTER_MESSAGES))
296 0 : goto error;
297 :
298 : /* We encrypt into the same buffer, so the caller must ensure that buf
299 : * has NOISE_AUTHTAG_LEN bytes to store the MAC. The nonce and index
300 : * are passed back out to the caller through the provided data pointer. */
301 3355 : *r_idx = kp->kp_remote_index;
302 :
303 3355 : wg_prepare_sync_enc_op (vm, ptd, b, lb, crypto_ops, src, srclen, dst, NULL,
304 : 0, *nonce, kp->kp_send_index, bi, iv);
305 :
306 : /* If our values are still within tolerances, but we are approaching
307 : * the tolerances, we notify the caller with ESTALE that they should
308 : * establish a new keypair. The current keypair can continue to be used
309 : * until the tolerances are hit. We notify if:
310 : * - our send counter is valid and not less than REKEY_AFTER_MESSAGES
311 : * - we're the initiator and our keypair is older than
312 : * REKEY_AFTER_TIME seconds */
313 3355 : ret = SC_KEEP_KEY_FRESH;
314 3355 : if ((kp->kp_valid && *nonce >= REKEY_AFTER_MESSAGES) ||
315 3355 : (kp->kp_is_initiator && wg_birthdate_has_expired_opt (
316 : kp->kp_birthdate, REKEY_AFTER_TIME, time)))
317 0 : goto error;
318 :
319 3355 : ret = SC_OK;
320 3355 : error:
321 3355 : return ret;
322 : }
323 :
324 : static_always_inline enum noise_state_crypt
325 20 : wg_add_to_async_frame (vlib_main_t *vm, wg_per_thread_data_t *ptd,
326 : vnet_crypto_async_frame_t **async_frame,
327 : vlib_buffer_t *b, vlib_buffer_t *lb, u8 *payload,
328 : u32 payload_len, u32 bi, u16 next, u16 async_next,
329 : noise_remote_t *r, uint32_t *r_idx, uint64_t *nonce,
330 : u8 *iv, f64 time)
331 : {
332 20 : wg_post_data_t *post = wg_post_data (b);
333 20 : u8 flag = 0;
334 : u8 *tag;
335 : noise_keypair_t *kp;
336 :
337 20 : post->next_index = next;
338 :
339 : /* crypto */
340 20 : enum noise_state_crypt ret = SC_FAILED;
341 :
342 20 : if ((kp = r->r_current) == NULL)
343 0 : goto error;
344 :
345 : /* We confirm that our values are within our tolerances. We want:
346 : * - a valid keypair
347 : * - our keypair to be less than REJECT_AFTER_TIME seconds old
348 : * - our receive counter to be less than REJECT_AFTER_MESSAGES
349 : * - our send counter to be less than REJECT_AFTER_MESSAGES
350 : */
351 40 : if (!kp->kp_valid ||
352 20 : wg_birthdate_has_expired_opt (kp->kp_birthdate, REJECT_AFTER_TIME,
353 20 : time) ||
354 20 : kp->kp_ctr.c_recv >= REJECT_AFTER_MESSAGES ||
355 20 : ((*nonce = noise_counter_send (&kp->kp_ctr)) > REJECT_AFTER_MESSAGES))
356 0 : goto error;
357 :
358 : /* We encrypt into the same buffer, so the caller must ensure that buf
359 : * has NOISE_AUTHTAG_LEN bytes to store the MAC. The nonce and index
360 : * are passed back out to the caller through the provided data pointer. */
361 20 : *r_idx = kp->kp_remote_index;
362 :
363 20 : clib_memset (iv, 0, 4);
364 20 : clib_memcpy (iv + 4, nonce, sizeof (*nonce));
365 :
366 : /* get a frame for this op if we don't yet have one or it's full */
367 20 : if (NULL == *async_frame || vnet_crypto_async_frame_is_full (*async_frame))
368 : {
369 8 : *async_frame = vnet_crypto_async_get_frame (
370 : vm, VNET_CRYPTO_OP_CHACHA20_POLY1305_TAG16_AAD0_ENC);
371 8 : if (PREDICT_FALSE (NULL == *async_frame))
372 0 : goto error;
373 : /* Save the frame to the list we'll submit at the end */
374 8 : vec_add1 (ptd->async_frames, *async_frame);
375 : }
376 :
377 20 : if (b != lb)
378 12 : flag |= VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS;
379 :
380 20 : tag = vlib_buffer_get_tail (lb) - NOISE_AUTHTAG_LEN;
381 :
382 : /* this always succeeds because we know the frame is not full */
383 20 : wg_output_tun_add_to_frame (vm, *async_frame, kp->kp_send_index, payload_len,
384 20 : payload - b->data, bi, async_next, iv, tag,
385 : flag);
386 :
387 : /* If our values are still within tolerances, but we are approaching
388 : * the tolerances, we notify the caller with ESTALE that they should
389 : * establish a new keypair. The current keypair can continue to be used
390 : * until the tolerances are hit. We notify if:
391 : * - our send counter is valid and not less than REKEY_AFTER_MESSAGES
392 : * - we're the initiator and our keypair is older than
393 : * REKEY_AFTER_TIME seconds */
394 20 : ret = SC_KEEP_KEY_FRESH;
395 20 : if ((kp->kp_valid && *nonce >= REKEY_AFTER_MESSAGES) ||
396 20 : (kp->kp_is_initiator && wg_birthdate_has_expired_opt (
397 : kp->kp_birthdate, REKEY_AFTER_TIME, time)))
398 0 : goto error;
399 :
400 20 : ret = SC_OK;
401 20 : error:
402 20 : return ret;
403 : }
404 :
405 : static_always_inline void
406 3375 : wg_calc_checksum (vlib_main_t *vm, vlib_buffer_t *b)
407 : {
408 3375 : int bogus = 0;
409 3375 : u8 ip_ver_out = (*((u8 *) vlib_buffer_get_current (b)) >> 4);
410 :
411 : /* IPv6 UDP checksum is mandatory */
412 3375 : if (ip_ver_out == 6)
413 : {
414 : ip6_header_t *ip6 =
415 1054 : (ip6_header_t *) ((u8 *) vlib_buffer_get_current (b));
416 1054 : udp_header_t *udp = ip6_next_header (ip6);
417 1054 : udp->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus);
418 : }
419 3375 : }
420 :
421 : /* is_ip4 - inner header flag */
422 : always_inline uword
423 64 : wg_output_tun_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
424 : vlib_frame_t *frame, u8 is_ip4, u16 async_next_node)
425 : {
426 64 : wg_main_t *wmp = &wg_main;
427 64 : wg_per_thread_data_t *ptd =
428 64 : vec_elt_at_index (wmp->per_thread_data, vm->thread_index);
429 64 : u32 *from = vlib_frame_vector_args (frame);
430 64 : u32 n_left_from = frame->n_vectors;
431 64 : ip4_udp_wg_header_t *hdr4_out = NULL;
432 64 : ip6_udp_wg_header_t *hdr6_out = NULL;
433 64 : message_data_t *message_data_wg = NULL;
434 64 : vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
435 : vlib_buffer_t *lb;
436 : vnet_crypto_op_t **crypto_ops;
437 64 : u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
438 : vlib_buffer_t *sync_bufs[VLIB_FRAME_SIZE];
439 64 : u32 thread_index = vm->thread_index;
440 64 : u16 n_sync = 0;
441 64 : const u16 drop_next = WG_OUTPUT_NEXT_ERROR;
442 64 : const u8 is_async = wg_op_mode_is_set_ASYNC ();
443 64 : vnet_crypto_async_frame_t *async_frame = NULL;
444 64 : u16 n_async = 0;
445 64 : u16 noop_nexts[VLIB_FRAME_SIZE], *noop_next = noop_nexts, n_noop = 0;
446 64 : u16 err = !0;
447 : u32 sync_bi[VLIB_FRAME_SIZE];
448 : u32 noop_bi[VLIB_FRAME_SIZE];
449 :
450 64 : vlib_get_buffers (vm, from, bufs, n_left_from);
451 64 : vec_reset_length (ptd->crypto_ops);
452 64 : vec_reset_length (ptd->chained_crypto_ops);
453 64 : vec_reset_length (ptd->chunks);
454 64 : vec_reset_length (ptd->async_frames);
455 :
456 64 : wg_peer_t *peer = NULL;
457 64 : u32 adj_index = 0;
458 64 : u32 last_adj_index = ~0;
459 64 : index_t peeri = INDEX_INVALID;
460 :
461 64 : f64 time = clib_time_now (&vm->clib_time) + vm->time_offset;
462 :
463 3716 : while (n_left_from > 0)
464 : {
465 3652 : u8 iph_offset = 0;
466 3652 : u8 is_ip4_out = 1;
467 : u8 *plain_data;
468 : u16 plain_data_len;
469 : u16 plain_data_len_total;
470 : u16 n_bufs;
471 : u16 b_space_left_at_beginning;
472 3652 : u32 bi = from[b - bufs];
473 :
474 3652 : if (n_left_from > 2)
475 : {
476 : u8 *p;
477 3559 : vlib_prefetch_buffer_header (b[2], LOAD);
478 3559 : p = vlib_buffer_get_current (b[1]);
479 3559 : CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD);
480 3559 : CLIB_PREFETCH (vlib_buffer_get_tail (b[1]), CLIB_CACHE_LINE_BYTES,
481 : LOAD);
482 : }
483 :
484 3652 : noop_next[0] = WG_OUTPUT_NEXT_ERROR;
485 3652 : err = WG_OUTPUT_NEXT_ERROR;
486 :
487 3652 : adj_index = vnet_buffer (b[0])->ip.adj_index[VLIB_TX];
488 :
489 3652 : if (PREDICT_FALSE (last_adj_index != adj_index))
490 : {
491 318 : peeri = wg_peer_get_by_adj_index (adj_index);
492 318 : if (peeri == INDEX_INVALID)
493 : {
494 4 : b[0]->error = node->errors[WG_OUTPUT_ERROR_PEER];
495 4 : goto out;
496 : }
497 314 : peer = wg_peer_get (peeri);
498 : }
499 :
500 3648 : if (!peer || wg_peer_is_dead (peer))
501 : {
502 0 : b[0]->error = node->errors[WG_OUTPUT_ERROR_PEER];
503 0 : goto out;
504 : }
505 3648 : if (PREDICT_FALSE (~0 == peer->output_thread_index))
506 : {
507 : /* this is the first packet to use this peer, claim the peer
508 : * for this thread.
509 : */
510 38 : clib_atomic_cmp_and_swap (&peer->output_thread_index, ~0,
511 : wg_peer_assign_thread (thread_index));
512 : }
513 :
514 3648 : if (PREDICT_FALSE (thread_index != peer->output_thread_index))
515 : {
516 255 : noop_next[0] = WG_OUTPUT_NEXT_HANDOFF;
517 255 : err = WG_OUTPUT_NEXT_HANDOFF;
518 255 : goto next;
519 : }
520 :
521 3393 : if (PREDICT_FALSE (!peer->remote.r_current))
522 : {
523 18 : wg_send_handshake_from_mt (peeri, false);
524 18 : b[0]->error = node->errors[WG_OUTPUT_ERROR_KEYPAIR];
525 18 : goto out;
526 : }
527 :
528 3375 : lb = b[0];
529 3375 : n_bufs = vlib_buffer_chain_linearize (vm, b[0]);
530 3375 : if (n_bufs == 0)
531 : {
532 0 : b[0]->error = node->errors[WG_OUTPUT_ERROR_NO_BUFFERS];
533 0 : goto out;
534 : }
535 :
536 3375 : if (n_bufs > 1)
537 : {
538 : /* Find last buffer in the chain */
539 40 : while (lb->flags & VLIB_BUFFER_NEXT_PRESENT)
540 24 : lb = vlib_get_buffer (vm, lb->next_buffer);
541 : }
542 :
543 : /* Ensure there is enough free space at the beginning of the first buffer
544 : * to write ethernet header (e.g. IPv6 VxLAN over IPv6 Wireguard will
545 : * trigger this)
546 : */
547 3375 : ASSERT ((signed) b[0]->current_data >=
548 : (signed) -VLIB_BUFFER_PRE_DATA_SIZE);
549 3375 : b_space_left_at_beginning =
550 3375 : b[0]->current_data + VLIB_BUFFER_PRE_DATA_SIZE;
551 3375 : if (PREDICT_FALSE (b_space_left_at_beginning <
552 : sizeof (ethernet_header_t)))
553 : {
554 10 : u32 size_diff =
555 10 : sizeof (ethernet_header_t) - b_space_left_at_beginning;
556 :
557 : /* Can only move buffer when it's single and has enough free space*/
558 20 : if (lb == b[0] &&
559 10 : vlib_buffer_space_left_at_end (vm, b[0]) >= size_diff)
560 : {
561 10 : vlib_buffer_move (vm, b[0],
562 10 : b[0]->current_data + (signed) size_diff);
563 : }
564 : else
565 : {
566 0 : b[0]->error = node->errors[WG_OUTPUT_ERROR_NO_BUFFERS];
567 0 : goto out;
568 : }
569 : }
570 :
571 : /*
572 : * Ensure there is enough free space at the end of the last buffer to
573 : * write auth tag */
574 3375 : if (PREDICT_FALSE (vlib_buffer_space_left_at_end (vm, lb) <
575 : NOISE_AUTHTAG_LEN))
576 : {
577 8 : u32 tmp_bi = 0;
578 8 : if (vlib_buffer_alloc (vm, &tmp_bi, 1) != 1)
579 : {
580 0 : b[0]->error = node->errors[WG_OUTPUT_ERROR_NO_BUFFERS];
581 0 : goto out;
582 : }
583 8 : lb = vlib_buffer_chain_buffer (vm, lb, tmp_bi);
584 : }
585 :
586 3375 : iph_offset = vnet_buffer (b[0])->ip.save_rewrite_length;
587 3375 : plain_data = vlib_buffer_get_current (b[0]) + iph_offset;
588 3375 : plain_data_len = b[0]->current_length - iph_offset;
589 3375 : plain_data_len_total =
590 3375 : vlib_buffer_length_in_chain (vm, b[0]) - iph_offset;
591 3375 : size_t encrypted_packet_len = message_data_len (plain_data_len_total);
592 3375 : vlib_buffer_chain_increase_length (b[0], lb, NOISE_AUTHTAG_LEN);
593 3375 : u8 *iv_data = b[0]->pre_data;
594 :
595 3375 : is_ip4_out = ip46_address_is_ip4 (&peer->src.addr);
596 3375 : if (is_ip4_out)
597 : {
598 2321 : hdr4_out = vlib_buffer_get_current (b[0]);
599 2321 : message_data_wg = &hdr4_out->wg;
600 : }
601 : else
602 : {
603 1054 : hdr6_out = vlib_buffer_get_current (b[0]);
604 1054 : message_data_wg = &hdr6_out->wg;
605 : }
606 :
607 3375 : if (PREDICT_FALSE (last_adj_index != adj_index))
608 : {
609 41 : wg_timers_any_authenticated_packet_sent_opt (peer, time);
610 41 : wg_timers_data_sent_opt (peer, time);
611 41 : wg_timers_any_authenticated_packet_traversal (peer);
612 41 : last_adj_index = adj_index;
613 : }
614 :
615 : /* Here we are sure that can send packet to next node */
616 3375 : next[0] = WG_OUTPUT_NEXT_INTERFACE_OUTPUT;
617 :
618 3375 : if (lb != b[0])
619 24 : crypto_ops = &ptd->chained_crypto_ops;
620 : else
621 3351 : crypto_ops = &ptd->crypto_ops;
622 :
623 : enum noise_state_crypt state;
624 :
625 3375 : if (is_async)
626 : {
627 20 : state = wg_add_to_async_frame (
628 : vm, ptd, &async_frame, b[0], lb, plain_data, plain_data_len_total,
629 20 : bi, next[0], async_next_node, &peer->remote,
630 20 : &message_data_wg->receiver_index, &message_data_wg->counter,
631 : iv_data, time);
632 : }
633 : else
634 : {
635 3355 : state = wg_output_tun_process (
636 : vm, ptd, b[0], lb, crypto_ops, &peer->remote,
637 3355 : &message_data_wg->receiver_index, &message_data_wg->counter,
638 : plain_data, plain_data_len, plain_data, n_sync, iv_data, time);
639 : }
640 :
641 3375 : if (PREDICT_FALSE (state == SC_KEEP_KEY_FRESH))
642 : {
643 0 : wg_send_handshake_from_mt (peeri, false);
644 : }
645 3375 : else if (PREDICT_FALSE (state == SC_FAILED))
646 : {
647 : // TODO: Maybe wrong
648 0 : wg_send_handshake_from_mt (peeri, false);
649 0 : wg_peer_update_flags (peeri, WG_PEER_ESTABLISHED, false);
650 0 : noop_next[0] = WG_OUTPUT_NEXT_ERROR;
651 0 : goto out;
652 : }
653 :
654 3375 : err = WG_OUTPUT_NEXT_INTERFACE_OUTPUT;
655 :
656 3375 : if (is_ip4_out)
657 : {
658 2321 : hdr4_out->wg.header.type = MESSAGE_DATA;
659 2321 : hdr4_out->udp.length = clib_host_to_net_u16 (encrypted_packet_len +
660 : sizeof (udp_header_t));
661 2321 : ip4_header_set_len_w_chksum (
662 2321 : &hdr4_out->ip4, clib_host_to_net_u16 (encrypted_packet_len +
663 : sizeof (ip4_udp_header_t)));
664 : }
665 : else
666 : {
667 1054 : hdr6_out->wg.header.type = MESSAGE_DATA;
668 1054 : hdr6_out->ip6.payload_length = hdr6_out->udp.length =
669 1054 : clib_host_to_net_u16 (encrypted_packet_len +
670 : sizeof (udp_header_t));
671 : }
672 :
673 3397 : out:
674 3397 : if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
675 : && (b[0]->flags & VLIB_BUFFER_IS_TRACED)))
676 : {
677 : wg_output_tun_trace_t *t =
678 3397 : vlib_add_trace (vm, node, b[0], sizeof (*t));
679 :
680 3397 : t->peer = peeri;
681 3397 : t->is_ip4 = is_ip4_out;
682 3397 : if (hdr4_out)
683 2321 : clib_memcpy (t->header, hdr4_out, sizeof (ip4_udp_header_t));
684 1076 : else if (hdr6_out)
685 1054 : clib_memcpy (t->header, hdr6_out, sizeof (ip6_udp_header_t));
686 : }
687 :
688 22 : next:
689 3652 : if (PREDICT_FALSE (err != WG_OUTPUT_NEXT_INTERFACE_OUTPUT))
690 : {
691 277 : noop_bi[n_noop] = bi;
692 277 : n_noop++;
693 277 : noop_next++;
694 277 : goto next_left;
695 : }
696 3375 : if (!is_async)
697 : {
698 3355 : sync_bi[n_sync] = bi;
699 3355 : sync_bufs[n_sync] = b[0];
700 3355 : n_sync += 1;
701 3355 : next += 1;
702 : }
703 : else
704 : {
705 20 : n_async++;
706 : }
707 3652 : next_left:
708 3652 : n_left_from -= 1;
709 3652 : b += 1;
710 : }
711 :
712 64 : if (n_sync)
713 : {
714 : /* wg-output-process-ops */
715 33 : wg_output_process_ops (vm, node, ptd->crypto_ops, sync_bufs, nexts,
716 : drop_next);
717 33 : wg_output_process_chained_ops (vm, node, ptd->chained_crypto_ops,
718 : sync_bufs, nexts, ptd->chunks, drop_next);
719 :
720 33 : int n_left_from_sync_bufs = n_sync;
721 3388 : while (n_left_from_sync_bufs > 0)
722 : {
723 3355 : n_left_from_sync_bufs--;
724 3355 : wg_calc_checksum (vm, sync_bufs[n_left_from_sync_bufs]);
725 : }
726 :
727 33 : vlib_buffer_enqueue_to_next (vm, node, sync_bi, nexts, n_sync);
728 : }
729 64 : if (n_async)
730 : {
731 : /* submit all of the open frames */
732 : vnet_crypto_async_frame_t **async_frame;
733 :
734 16 : vec_foreach (async_frame, ptd->async_frames)
735 : {
736 8 : if (PREDICT_FALSE (
737 : vnet_crypto_async_submit_open_frame (vm, *async_frame) < 0))
738 : {
739 0 : u32 n_drop = (*async_frame)->n_elts;
740 0 : u32 *bi = (*async_frame)->buffer_indices;
741 0 : u16 index = n_noop;
742 0 : while (n_drop--)
743 : {
744 0 : noop_bi[index] = bi[0];
745 0 : vlib_buffer_t *b = vlib_get_buffer (vm, bi[0]);
746 0 : noop_nexts[index] = drop_next;
747 0 : b->error = node->errors[WG_OUTPUT_ERROR_CRYPTO_ENGINE_ERROR];
748 0 : bi++;
749 0 : index++;
750 : }
751 0 : n_noop += (*async_frame)->n_elts;
752 :
753 0 : vnet_crypto_async_reset_frame (*async_frame);
754 0 : vnet_crypto_async_free_frame (vm, *async_frame);
755 : }
756 : }
757 : }
758 64 : if (n_noop)
759 : {
760 23 : vlib_buffer_enqueue_to_next (vm, node, noop_bi, noop_nexts, n_noop);
761 : }
762 :
763 64 : return frame->n_vectors;
764 : }
765 :
766 : always_inline uword
767 8 : wg_output_tun_post (vlib_main_t *vm, vlib_node_runtime_t *node,
768 : vlib_frame_t *frame)
769 : {
770 8 : vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
771 8 : u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
772 8 : u32 *from = vlib_frame_vector_args (frame);
773 8 : u32 n_left = frame->n_vectors;
774 :
775 8 : index_t peeri = ~0;
776 :
777 8 : vlib_get_buffers (vm, from, b, n_left);
778 :
779 8 : if (n_left >= 4)
780 : {
781 4 : vlib_prefetch_buffer_header (b[0], LOAD);
782 4 : vlib_prefetch_buffer_header (b[1], LOAD);
783 4 : vlib_prefetch_buffer_header (b[2], LOAD);
784 4 : vlib_prefetch_buffer_header (b[3], LOAD);
785 : }
786 :
787 8 : while (n_left > 8)
788 : {
789 0 : vlib_prefetch_buffer_header (b[4], LOAD);
790 0 : vlib_prefetch_buffer_header (b[5], LOAD);
791 0 : vlib_prefetch_buffer_header (b[6], LOAD);
792 0 : vlib_prefetch_buffer_header (b[7], LOAD);
793 :
794 0 : next[0] = (wg_post_data (b[0]))->next_index;
795 0 : next[1] = (wg_post_data (b[1]))->next_index;
796 0 : next[2] = (wg_post_data (b[2]))->next_index;
797 0 : next[3] = (wg_post_data (b[3]))->next_index;
798 :
799 0 : wg_calc_checksum (vm, b[0]);
800 0 : wg_calc_checksum (vm, b[1]);
801 0 : wg_calc_checksum (vm, b[2]);
802 0 : wg_calc_checksum (vm, b[3]);
803 :
804 0 : if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
805 : {
806 0 : if (b[0]->flags & VLIB_BUFFER_IS_TRACED)
807 : {
808 : wg_output_tun_post_trace_t *tr =
809 0 : vlib_add_trace (vm, node, b[0], sizeof (*tr));
810 0 : peeri = wg_peer_get_by_adj_index (
811 0 : vnet_buffer (b[0])->ip.adj_index[VLIB_TX]);
812 0 : tr->peer = peeri;
813 0 : tr->next_index = next[0];
814 : }
815 0 : if (b[1]->flags & VLIB_BUFFER_IS_TRACED)
816 : {
817 : wg_output_tun_post_trace_t *tr =
818 0 : vlib_add_trace (vm, node, b[1], sizeof (*tr));
819 0 : peeri = wg_peer_get_by_adj_index (
820 0 : vnet_buffer (b[1])->ip.adj_index[VLIB_TX]);
821 0 : tr->next_index = next[1];
822 : }
823 0 : if (b[2]->flags & VLIB_BUFFER_IS_TRACED)
824 : {
825 : wg_output_tun_post_trace_t *tr =
826 0 : vlib_add_trace (vm, node, b[2], sizeof (*tr));
827 0 : peeri = wg_peer_get_by_adj_index (
828 0 : vnet_buffer (b[2])->ip.adj_index[VLIB_TX]);
829 0 : tr->next_index = next[2];
830 : }
831 0 : if (b[3]->flags & VLIB_BUFFER_IS_TRACED)
832 : {
833 : wg_output_tun_post_trace_t *tr =
834 0 : vlib_add_trace (vm, node, b[3], sizeof (*tr));
835 0 : peeri = wg_peer_get_by_adj_index (
836 0 : vnet_buffer (b[3])->ip.adj_index[VLIB_TX]);
837 0 : tr->next_index = next[3];
838 : }
839 : }
840 :
841 0 : b += 4;
842 0 : next += 4;
843 0 : n_left -= 4;
844 : }
845 :
846 28 : while (n_left > 0)
847 : {
848 20 : wg_calc_checksum (vm, b[0]);
849 :
850 20 : next[0] = (wg_post_data (b[0]))->next_index;
851 20 : if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
852 : (b[0]->flags & VLIB_BUFFER_IS_TRACED)))
853 : {
854 : wg_output_tun_post_trace_t *tr =
855 0 : vlib_add_trace (vm, node, b[0], sizeof (*tr));
856 0 : peeri = wg_peer_get_by_adj_index (
857 0 : vnet_buffer (b[0])->ip.adj_index[VLIB_TX]);
858 0 : tr->next_index = next[0];
859 : }
860 :
861 20 : b += 1;
862 20 : next += 1;
863 20 : n_left -= 1;
864 : }
865 :
866 8 : vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
867 8 : return frame->n_vectors;
868 : }
869 :
870 1119 : VLIB_REGISTER_NODE (wg4_output_tun_post_node) = {
871 : .name = "wg4-output-tun-post-node",
872 : .vector_size = sizeof (u32),
873 : .format_trace = format_wg_output_tun_post_trace,
874 : .type = VLIB_NODE_TYPE_INTERNAL,
875 : .sibling_of = "wg4-output-tun",
876 : .n_errors = ARRAY_LEN (wg_output_error_strings),
877 : .error_strings = wg_output_error_strings,
878 : };
879 :
880 1119 : VLIB_REGISTER_NODE (wg6_output_tun_post_node) = {
881 : .name = "wg6-output-tun-post-node",
882 : .vector_size = sizeof (u32),
883 : .format_trace = format_wg_output_tun_post_trace,
884 : .type = VLIB_NODE_TYPE_INTERNAL,
885 : .sibling_of = "wg6-output-tun",
886 : .n_errors = ARRAY_LEN (wg_output_error_strings),
887 : .error_strings = wg_output_error_strings,
888 : };
889 :
890 563 : VLIB_NODE_FN (wg4_output_tun_post_node)
891 : (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
892 : {
893 4 : return wg_output_tun_post (vm, node, from_frame);
894 : }
895 :
896 563 : VLIB_NODE_FN (wg6_output_tun_post_node)
897 : (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
898 : {
899 4 : return wg_output_tun_post (vm, node, from_frame);
900 : }
901 :
902 599 : VLIB_NODE_FN (wg4_output_tun_node)
903 : (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
904 : {
905 80 : return wg_output_tun_inline (vm, node, frame, /* is_ip4 */ 1,
906 40 : wg_encrypt_async_next.wg4_post_next);
907 : }
908 :
909 583 : VLIB_NODE_FN (wg6_output_tun_node)
910 : (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
911 : {
912 48 : return wg_output_tun_inline (vm, node, frame, /* is_ip4 */ 0,
913 24 : wg_encrypt_async_next.wg6_post_next);
914 : }
915 :
916 : /* *INDENT-OFF* */
917 1119 : VLIB_REGISTER_NODE (wg4_output_tun_node) =
918 : {
919 : .name = "wg4-output-tun",
920 : .vector_size = sizeof (u32),
921 : .format_trace = format_wg_output_tun_trace,
922 : .type = VLIB_NODE_TYPE_INTERNAL,
923 : .n_errors = ARRAY_LEN (wg_output_error_strings),
924 : .error_strings = wg_output_error_strings,
925 : .n_next_nodes = WG_OUTPUT_N_NEXT,
926 : .next_nodes = {
927 : [WG_OUTPUT_NEXT_HANDOFF] = "wg4-output-tun-handoff",
928 : [WG_OUTPUT_NEXT_INTERFACE_OUTPUT] = "adj-midchain-tx",
929 : [WG_OUTPUT_NEXT_ERROR] = "error-drop",
930 : },
931 : };
932 :
933 1119 : VLIB_REGISTER_NODE (wg6_output_tun_node) =
934 : {
935 : .name = "wg6-output-tun",
936 : .vector_size = sizeof (u32),
937 : .format_trace = format_wg_output_tun_trace,
938 : .type = VLIB_NODE_TYPE_INTERNAL,
939 : .n_errors = ARRAY_LEN (wg_output_error_strings),
940 : .error_strings = wg_output_error_strings,
941 : .n_next_nodes = WG_OUTPUT_N_NEXT,
942 : .next_nodes = {
943 : [WG_OUTPUT_NEXT_HANDOFF] = "wg6-output-tun-handoff",
944 : [WG_OUTPUT_NEXT_INTERFACE_OUTPUT] = "adj-midchain-tx",
945 : [WG_OUTPUT_NEXT_ERROR] = "error-drop",
946 : },
947 : };
948 : /* *INDENT-ON* */
949 :
950 : /*
951 : * fd.io coding-style-patch-verification: ON
952 : *
953 : * Local Variables:
954 : * eval: (c-set-style "gnu")
955 : * End:
956 : */
|
