Line data Source code
1 : /*
2 : * Copyright (c) 2020 Doc.ai and/or its affiliates.
3 : * Copyright (c) 2020 Cisco and/or its affiliates.
4 : * Licensed under the Apache License, Version 2.0 (the "License");
5 : * you may not use this file except in compliance with the License.
6 : * You may obtain a copy of the License at:
7 : *
8 : * http://www.apache.org/licenses/LICENSE-2.0
9 : *
10 : * Unless required by applicable law or agreed to in writing, software
11 : * distributed under the License is distributed on an "AS IS" BASIS,
12 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 : * See the License for the specific language governing permissions and
14 : * limitations under the License.
15 : */
16 :
17 : #include <vlib/vlib.h>
18 : #include <vnet/vnet.h>
19 : #include <vppinfra/error.h>
20 : #include <wireguard/wireguard.h>
21 :
22 : #include <wireguard/wireguard_send.h>
23 : #include <wireguard/wireguard_if.h>
24 :
25 : #define foreach_wg_input_error \
26 : _ (NONE, "No error") \
27 : _ (HANDSHAKE_MAC, "Invalid MAC handshake") \
28 : _ (HANDSHAKE_RATELIMITED, "Handshake ratelimited") \
29 : _ (PEER, "Peer error") \
30 : _ (INTERFACE, "Interface error") \
31 : _ (DECRYPTION, "Failed during decryption") \
32 : _ (KEEPALIVE_SEND, "Failed while sending Keepalive") \
33 : _ (HANDSHAKE_SEND, "Failed while sending Handshake") \
34 : _ (HANDSHAKE_RECEIVE, "Failed while receiving Handshake") \
35 : _ (COOKIE_DECRYPTION, "Failed during Cookie decryption") \
36 : _ (COOKIE_SEND, "Failed during sending Cookie") \
37 : _ (NO_BUFFERS, "No buffers") \
38 : _ (UNDEFINED, "Undefined error") \
39 : _ (CRYPTO_ENGINE_ERROR, "crypto engine error (packet dropped)")
40 :
41 : typedef enum
42 : {
43 : #define _(sym,str) WG_INPUT_ERROR_##sym,
44 : foreach_wg_input_error
45 : #undef _
46 : WG_INPUT_N_ERROR,
47 : } wg_input_error_t;
48 :
49 : static char *wg_input_error_strings[] = {
50 : #define _(sym,string) string,
51 : foreach_wg_input_error
52 : #undef _
53 : };
54 :
55 : typedef struct
56 : {
57 : message_type_t type;
58 : u16 current_length;
59 : bool is_keepalive;
60 : index_t peer;
61 : } wg_input_trace_t;
62 :
63 : typedef struct
64 : {
65 : index_t peer;
66 : u16 next;
67 : } wg_input_post_trace_t;
68 :
69 : u8 *
70 5071 : format_wg_message_type (u8 * s, va_list * args)
71 : {
72 5071 : message_type_t type = va_arg (*args, message_type_t);
73 :
74 5071 : switch (type)
75 : {
76 : #define _(v,a) case MESSAGE_##v: return (format (s, "%s", a));
77 5071 : foreach_wg_message_type
78 : #undef _
79 : }
80 0 : return (format (s, "unknown"));
81 : }
82 :
83 : /* packet trace format function */
84 : static u8 *
85 5071 : format_wg_input_trace (u8 * s, va_list * args)
86 : {
87 5071 : CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
88 5071 : CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
89 :
90 5071 : wg_input_trace_t *t = va_arg (*args, wg_input_trace_t *);
91 :
92 5071 : s = format (s, "Wireguard input: \n");
93 5071 : s = format (s, " Type: %U\n", format_wg_message_type, t->type);
94 5071 : s = format (s, " Peer: %d\n", t->peer);
95 5071 : s = format (s, " Length: %d\n", t->current_length);
96 5071 : s = format (s, " Keepalive: %s", t->is_keepalive ? "true" : "false");
97 :
98 5071 : return s;
99 : }
100 :
101 : /* post-node packet trace format function */
102 : static u8 *
103 0 : format_wg_input_post_trace (u8 *s, va_list *args)
104 : {
105 0 : CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
106 0 : CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
107 :
108 0 : wg_input_post_trace_t *t = va_arg (*args, wg_input_post_trace_t *);
109 :
110 0 : s = format (s, "WG input post: \n");
111 0 : s = format (s, " peer: %u\n", t->peer);
112 0 : s = format (s, " next: %u\n", t->next);
113 :
114 0 : return s;
115 : }
116 :
117 : typedef enum
118 : {
119 : WG_INPUT_NEXT_HANDOFF_HANDSHAKE,
120 : WG_INPUT_NEXT_HANDOFF_DATA,
121 : WG_INPUT_NEXT_IP4_INPUT,
122 : WG_INPUT_NEXT_IP6_INPUT,
123 : WG_INPUT_NEXT_PUNT,
124 : WG_INPUT_NEXT_ERROR,
125 : WG_INPUT_N_NEXT,
126 : } wg_input_next_t;
127 :
128 : static u8
129 3624 : is_ip4_header (u8 *data)
130 : {
131 3624 : return (data[0] >> 4) == 0x4;
132 : }
133 :
134 : static wg_input_error_t
135 928 : wg_handshake_process (vlib_main_t *vm, wg_main_t *wmp, vlib_buffer_t *b,
136 : u32 node_idx, u8 is_ip4)
137 : {
138 928 : ASSERT (vm->thread_index == 0);
139 :
140 : enum cookie_mac_state mac_state;
141 : bool packet_needs_cookie;
142 : bool under_load;
143 : index_t *wg_ifs;
144 : wg_if_t *wg_if;
145 928 : wg_peer_t *peer = NULL;
146 :
147 928 : void *current_b_data = vlib_buffer_get_current (b);
148 :
149 : ip46_address_t src_ip;
150 928 : if (is_ip4)
151 : {
152 612 : ip4_header_t *iph4 =
153 : current_b_data - sizeof (udp_header_t) - sizeof (ip4_header_t);
154 612 : ip46_address_set_ip4 (&src_ip, &iph4->src_address);
155 : }
156 : else
157 : {
158 316 : ip6_header_t *iph6 =
159 : current_b_data - sizeof (udp_header_t) - sizeof (ip6_header_t);
160 316 : ip46_address_set_ip6 (&src_ip, &iph6->src_address);
161 : }
162 :
163 928 : udp_header_t *uhd = current_b_data - sizeof (udp_header_t);
164 928 : u16 udp_src_port = clib_host_to_net_u16 (uhd->src_port);
165 928 : u16 udp_dst_port = clib_host_to_net_u16 (uhd->dst_port);
166 :
167 928 : message_header_t *header = current_b_data;
168 :
169 928 : if (PREDICT_FALSE (header->type == MESSAGE_HANDSHAKE_COOKIE))
170 : {
171 16 : message_handshake_cookie_t *packet =
172 : (message_handshake_cookie_t *) current_b_data;
173 : u32 *entry =
174 16 : wg_index_table_lookup (&wmp->index_table, packet->receiver_index);
175 16 : if (entry)
176 16 : peer = wg_peer_get (*entry);
177 : else
178 0 : return WG_INPUT_ERROR_PEER;
179 :
180 16 : if (!cookie_maker_consume_payload (
181 16 : vm, &peer->cookie_maker, packet->nonce, packet->encrypted_cookie))
182 8 : return WG_INPUT_ERROR_COOKIE_DECRYPTION;
183 :
184 8 : return WG_INPUT_ERROR_NONE;
185 : }
186 :
187 912 : u32 len = (header->type == MESSAGE_HANDSHAKE_INITIATION ?
188 : sizeof (message_handshake_initiation_t) :
189 : sizeof (message_handshake_response_t));
190 :
191 912 : message_macs_t *macs = (message_macs_t *)
192 912 : ((u8 *) current_b_data + len - sizeof (*macs));
193 :
194 : index_t *ii;
195 912 : wg_ifs = wg_if_indexes_get_by_port (udp_dst_port);
196 912 : if (NULL == wg_ifs)
197 0 : return WG_INPUT_ERROR_INTERFACE;
198 :
199 926 : vec_foreach (ii, wg_ifs)
200 : {
201 918 : wg_if = wg_if_get (*ii);
202 918 : if (NULL == wg_if)
203 0 : continue;
204 :
205 918 : under_load = wg_if_is_under_load (vm, wg_if);
206 918 : mac_state = cookie_checker_validate_macs (
207 : vm, &wg_if->cookie_checker, macs, current_b_data, len, under_load,
208 : &src_ip, udp_src_port);
209 918 : if (mac_state == INVALID_MAC)
210 : {
211 14 : wg_if_dec_handshake_num (wg_if);
212 14 : wg_if = NULL;
213 14 : continue;
214 : }
215 904 : break;
216 : }
217 :
218 912 : if (NULL == wg_if)
219 8 : return WG_INPUT_ERROR_HANDSHAKE_MAC;
220 :
221 904 : if ((under_load && mac_state == VALID_MAC_WITH_COOKIE)
222 856 : || (!under_load && mac_state == VALID_MAC_BUT_NO_COOKIE))
223 822 : packet_needs_cookie = false;
224 82 : else if (under_load && mac_state == VALID_MAC_BUT_NO_COOKIE)
225 28 : packet_needs_cookie = true;
226 54 : else if (mac_state == VALID_MAC_WITH_COOKIE_BUT_RATELIMITED)
227 54 : return WG_INPUT_ERROR_HANDSHAKE_RATELIMITED;
228 : else
229 0 : return WG_INPUT_ERROR_HANDSHAKE_MAC;
230 :
231 850 : switch (header->type)
232 : {
233 659 : case MESSAGE_HANDSHAKE_INITIATION:
234 : {
235 659 : message_handshake_initiation_t *message = current_b_data;
236 :
237 659 : if (packet_needs_cookie)
238 : {
239 :
240 24 : if (!wg_send_handshake_cookie (vm, message->sender_index,
241 : &wg_if->cookie_checker, macs,
242 : &ip_addr_46 (&wg_if->src_ip),
243 24 : wg_if->port, &src_ip, udp_src_port))
244 600 : return WG_INPUT_ERROR_COOKIE_SEND;
245 :
246 24 : return WG_INPUT_ERROR_NONE;
247 : }
248 :
249 : noise_remote_t *rp;
250 635 : if (noise_consume_initiation
251 : (vm, noise_local_get (wg_if->local_idx), &rp,
252 635 : message->sender_index, message->unencrypted_ephemeral,
253 635 : message->encrypted_static, message->encrypted_timestamp))
254 : {
255 59 : peer = wg_peer_get (rp->r_peer_idx);
256 : }
257 : else
258 : {
259 576 : return WG_INPUT_ERROR_PEER;
260 : }
261 :
262 59 : wg_peer_update_endpoint (rp->r_peer_idx, &src_ip, udp_src_port);
263 :
264 59 : if (PREDICT_FALSE (!wg_send_handshake_response (vm, peer)))
265 : {
266 0 : vlib_node_increment_counter (vm, node_idx,
267 : WG_INPUT_ERROR_HANDSHAKE_SEND, 1);
268 : }
269 59 : break;
270 : }
271 191 : case MESSAGE_HANDSHAKE_RESPONSE:
272 : {
273 191 : message_handshake_response_t *resp = current_b_data;
274 :
275 191 : if (packet_needs_cookie)
276 : {
277 4 : if (!wg_send_handshake_cookie (vm, resp->sender_index,
278 : &wg_if->cookie_checker, macs,
279 : &ip_addr_46 (&wg_if->src_ip),
280 4 : wg_if->port, &src_ip, udp_src_port))
281 0 : return WG_INPUT_ERROR_COOKIE_SEND;
282 :
283 4 : return WG_INPUT_ERROR_NONE;
284 : }
285 :
286 187 : index_t peeri = INDEX_INVALID;
287 : u32 *entry =
288 187 : wg_index_table_lookup (&wmp->index_table, resp->receiver_index);
289 :
290 187 : if (PREDICT_TRUE (entry != NULL))
291 : {
292 187 : peeri = *entry;
293 187 : peer = wg_peer_get (peeri);
294 187 : if (wg_peer_is_dead (peer))
295 0 : return WG_INPUT_ERROR_PEER;
296 : }
297 : else
298 0 : return WG_INPUT_ERROR_PEER;
299 :
300 187 : if (!noise_consume_response
301 : (vm, &peer->remote, resp->sender_index,
302 187 : resp->receiver_index, resp->unencrypted_ephemeral,
303 187 : resp->encrypted_nothing))
304 : {
305 152 : return WG_INPUT_ERROR_PEER;
306 : }
307 :
308 35 : wg_peer_update_endpoint (peeri, &src_ip, udp_src_port);
309 :
310 35 : if (noise_remote_begin_session (vm, &peer->remote))
311 : {
312 :
313 35 : wg_timers_session_derived (peer);
314 35 : wg_timers_handshake_complete (peer);
315 35 : if (PREDICT_FALSE (!wg_send_keepalive (vm, peer)))
316 : {
317 0 : vlib_node_increment_counter (vm, node_idx,
318 : WG_INPUT_ERROR_KEEPALIVE_SEND, 1);
319 : }
320 : else
321 : {
322 35 : wg_peer_update_flags (peeri, WG_PEER_ESTABLISHED, true);
323 : }
324 : }
325 35 : break;
326 : }
327 0 : default:
328 0 : return WG_INPUT_ERROR_HANDSHAKE_RECEIVE;
329 : }
330 :
331 94 : wg_timers_any_authenticated_packet_received (peer);
332 94 : wg_timers_any_authenticated_packet_traversal (peer);
333 94 : return WG_INPUT_ERROR_NONE;
334 : }
335 :
336 : static_always_inline int
337 3632 : wg_input_post_process (vlib_main_t *vm, vlib_buffer_t *b, u16 *next,
338 : wg_peer_t *peer, message_data_t *data,
339 : bool *is_keepalive)
340 : {
341 3632 : next[0] = WG_INPUT_NEXT_PUNT;
342 : noise_keypair_t *kp;
343 : vlib_buffer_t *lb;
344 :
345 3632 : if ((kp = wg_get_active_keypair (&peer->remote, data->receiver_index)) ==
346 : NULL)
347 0 : return -1;
348 :
349 3632 : if (!noise_counter_recv (&kp->kp_ctr, data->counter))
350 : {
351 0 : return -1;
352 : }
353 :
354 3632 : lb = b;
355 : /* Find last buffer in the chain */
356 3664 : while (lb->flags & VLIB_BUFFER_NEXT_PRESENT)
357 32 : lb = vlib_get_buffer (vm, lb->next_buffer);
358 :
359 3632 : u16 encr_len = vlib_buffer_length_in_chain (vm, b) - sizeof (message_data_t);
360 3632 : u16 decr_len = encr_len - NOISE_AUTHTAG_LEN;
361 :
362 3632 : vlib_buffer_advance (b, sizeof (message_data_t));
363 3632 : vlib_buffer_chain_increase_length (b, lb, -NOISE_AUTHTAG_LEN);
364 3632 : vnet_buffer_offload_flags_clear (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
365 :
366 : /* Keepalive packet has zero length */
367 3632 : if (decr_len == 0)
368 : {
369 8 : *is_keepalive = true;
370 8 : return 0;
371 : }
372 :
373 3624 : wg_timers_data_received (peer);
374 :
375 : ip46_address_t src_ip;
376 3624 : u8 is_ip4_inner = is_ip4_header (vlib_buffer_get_current (b));
377 3624 : if (is_ip4_inner)
378 : {
379 2580 : ip46_address_set_ip4 (
380 2580 : &src_ip, &((ip4_header_t *) vlib_buffer_get_current (b))->src_address);
381 : }
382 : else
383 : {
384 1044 : ip46_address_set_ip6 (
385 1044 : &src_ip, &((ip6_header_t *) vlib_buffer_get_current (b))->src_address);
386 : }
387 :
388 : const fib_prefix_t *allowed_ip;
389 3624 : bool allowed = false;
390 :
391 : /*
392 : * we could make this into an ACL, but the expectation
393 : * is that there aren't many allowed IPs and thus a linear
394 : * walk is faster than an ACL
395 : */
396 3880 : vec_foreach (allowed_ip, peer->allowed_ips)
397 : {
398 3880 : if (fib_prefix_is_cover_addr_46 (allowed_ip, &src_ip))
399 : {
400 3624 : allowed = true;
401 3624 : break;
402 : }
403 : }
404 3624 : if (allowed)
405 : {
406 3624 : vnet_buffer (b)->sw_if_index[VLIB_RX] = peer->wg_sw_if_index;
407 3624 : next[0] =
408 : is_ip4_inner ? WG_INPUT_NEXT_IP4_INPUT : WG_INPUT_NEXT_IP6_INPUT;
409 : }
410 :
411 3624 : return 0;
412 : }
413 :
414 : static_always_inline void
415 260 : wg_input_process_ops (vlib_main_t *vm, vlib_node_runtime_t *node,
416 : vnet_crypto_op_t *ops, vlib_buffer_t *b[], u16 *nexts,
417 : u16 drop_next)
418 : {
419 260 : u32 n_fail, n_ops = vec_len (ops);
420 260 : vnet_crypto_op_t *op = ops;
421 :
422 260 : if (n_ops == 0)
423 214 : return;
424 :
425 46 : n_fail = n_ops - vnet_crypto_process_ops (vm, op, n_ops);
426 :
427 46 : while (n_fail)
428 : {
429 0 : ASSERT (op - ops < n_ops);
430 :
431 0 : if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED)
432 : {
433 0 : u32 bi = op->user_data;
434 0 : b[bi]->error = node->errors[WG_INPUT_ERROR_DECRYPTION];
435 0 : nexts[bi] = drop_next;
436 0 : n_fail--;
437 : }
438 0 : op++;
439 : }
440 : }
441 :
442 : static_always_inline void
443 260 : wg_input_process_chained_ops (vlib_main_t *vm, vlib_node_runtime_t *node,
444 : vnet_crypto_op_t *ops, vlib_buffer_t *b[],
445 : u16 *nexts, vnet_crypto_op_chunk_t *chunks,
446 : u16 drop_next)
447 : {
448 260 : u32 n_fail, n_ops = vec_len (ops);
449 260 : vnet_crypto_op_t *op = ops;
450 :
451 260 : if (n_ops == 0)
452 256 : return;
453 :
454 4 : n_fail = n_ops - vnet_crypto_process_chained_ops (vm, op, chunks, n_ops);
455 :
456 4 : while (n_fail)
457 : {
458 0 : ASSERT (op - ops < n_ops);
459 :
460 0 : if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED)
461 : {
462 0 : u32 bi = op->user_data;
463 0 : b[bi]->error = node->errors[WG_INPUT_ERROR_DECRYPTION];
464 0 : nexts[bi] = drop_next;
465 0 : n_fail--;
466 : }
467 0 : op++;
468 : }
469 : }
470 :
471 : static_always_inline void
472 12 : wg_input_chain_crypto (vlib_main_t *vm, wg_per_thread_data_t *ptd,
473 : vlib_buffer_t *b, vlib_buffer_t *lb, u8 *start,
474 : u32 start_len, u16 *n_ch)
475 : {
476 : vnet_crypto_op_chunk_t *ch;
477 12 : vlib_buffer_t *cb = b;
478 12 : u32 n_chunks = 1;
479 :
480 12 : vec_add2 (ptd->chunks, ch, 1);
481 12 : ch->len = start_len;
482 12 : ch->src = ch->dst = start;
483 12 : cb = vlib_get_buffer (vm, cb->next_buffer);
484 :
485 : while (1)
486 : {
487 16 : vec_add2 (ptd->chunks, ch, 1);
488 16 : n_chunks += 1;
489 16 : if (lb == cb)
490 12 : ch->len = cb->current_length - NOISE_AUTHTAG_LEN;
491 : else
492 4 : ch->len = cb->current_length;
493 :
494 16 : ch->src = ch->dst = vlib_buffer_get_current (cb);
495 :
496 16 : if (!(cb->flags & VLIB_BUFFER_NEXT_PRESENT))
497 12 : break;
498 :
499 4 : cb = vlib_get_buffer (vm, cb->next_buffer);
500 : }
501 :
502 12 : if (n_ch)
503 12 : *n_ch = n_chunks;
504 12 : }
505 :
506 : always_inline void
507 3612 : wg_prepare_sync_dec_op (vlib_main_t *vm, wg_per_thread_data_t *ptd,
508 : vlib_buffer_t *b, vlib_buffer_t *lb,
509 : vnet_crypto_op_t **crypto_ops, u8 *src, u32 src_len,
510 : u8 *dst, u8 *aad, u32 aad_len,
511 : vnet_crypto_key_index_t key_index, u32 bi, u8 *iv)
512 : {
513 3612 : vnet_crypto_op_t _op, *op = &_op;
514 3612 : u8 src_[] = {};
515 :
516 3612 : vec_add2_aligned (crypto_ops[0], op, 1, CLIB_CACHE_LINE_BYTES);
517 3612 : vnet_crypto_op_init (op, VNET_CRYPTO_OP_CHACHA20_POLY1305_DEC);
518 :
519 3612 : op->tag_len = NOISE_AUTHTAG_LEN;
520 3612 : op->tag = vlib_buffer_get_tail (lb) - NOISE_AUTHTAG_LEN;
521 3612 : op->key_index = key_index;
522 3612 : op->aad = aad;
523 3612 : op->aad_len = aad_len;
524 3612 : op->iv = iv;
525 3612 : op->user_data = bi;
526 3612 : op->flags |= VNET_CRYPTO_OP_FLAG_HMAC_CHECK;
527 :
528 3612 : if (b != lb)
529 : {
530 : /* Chained buffers */
531 12 : op->flags |= VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS;
532 12 : op->chunk_index = vec_len (ptd->chunks);
533 12 : wg_input_chain_crypto (vm, ptd, b, lb, src, src_len + NOISE_AUTHTAG_LEN,
534 12 : &op->n_chunks);
535 : }
536 : else
537 : {
538 3600 : op->src = !src ? src_ : src;
539 3600 : op->len = src_len;
540 3600 : op->dst = dst;
541 : }
542 3612 : }
543 :
544 : static_always_inline void
545 20 : wg_input_add_to_frame (vlib_main_t *vm, vnet_crypto_async_frame_t *f,
546 : u32 key_index, u32 crypto_len, i16 crypto_start_offset,
547 : u32 buffer_index, u16 next_node, u8 *iv, u8 *tag,
548 : u8 flags)
549 : {
550 : vnet_crypto_async_frame_elt_t *fe;
551 : u16 index;
552 :
553 20 : ASSERT (f->n_elts < VNET_CRYPTO_FRAME_SIZE);
554 :
555 20 : index = f->n_elts;
556 20 : fe = &f->elts[index];
557 20 : f->n_elts++;
558 20 : fe->key_index = key_index;
559 20 : fe->crypto_total_length = crypto_len;
560 20 : fe->crypto_start_offset = crypto_start_offset;
561 20 : fe->iv = iv;
562 20 : fe->tag = tag;
563 20 : fe->flags = flags;
564 20 : f->buffer_indices[index] = buffer_index;
565 20 : f->next_node_index[index] = next_node;
566 20 : }
567 :
568 : static_always_inline enum noise_state_crypt
569 3632 : wg_input_process (vlib_main_t *vm, wg_per_thread_data_t *ptd,
570 : vnet_crypto_op_t **crypto_ops,
571 : vnet_crypto_async_frame_t **async_frame, vlib_buffer_t *b,
572 : vlib_buffer_t *lb, u32 buf_idx, noise_remote_t *r,
573 : uint32_t r_idx, uint64_t nonce, uint8_t *src, size_t srclen,
574 : size_t srclen_total, uint8_t *dst, u32 from_idx, u8 *iv,
575 : f64 time, u8 is_async, u16 async_next_node)
576 : {
577 : noise_keypair_t *kp;
578 3632 : enum noise_state_crypt ret = SC_FAILED;
579 :
580 3632 : if ((kp = wg_get_active_keypair (r, r_idx)) == NULL)
581 : {
582 0 : goto error;
583 : }
584 :
585 : /* We confirm that our values are within our tolerances. These values
586 : * are the same as the encrypt routine.
587 : *
588 : * kp_ctr isn't locked here, we're happy to accept a racy read. */
589 3632 : if (wg_birthdate_has_expired_opt (kp->kp_birthdate, REJECT_AFTER_TIME,
590 3632 : time) ||
591 3632 : kp->kp_ctr.c_recv >= REJECT_AFTER_MESSAGES)
592 0 : goto error;
593 :
594 : /* Decrypt, then validate the counter. We don't want to validate the
595 : * counter before decrypting as we do not know the message is authentic
596 : * prior to decryption. */
597 :
598 3632 : clib_memset (iv, 0, 4);
599 3632 : clib_memcpy (iv + 4, &nonce, sizeof (nonce));
600 :
601 3632 : if (is_async)
602 : {
603 20 : u8 flags = VNET_CRYPTO_OP_FLAG_HMAC_CHECK;
604 20 : u8 *tag = vlib_buffer_get_tail (lb) - NOISE_AUTHTAG_LEN;
605 :
606 20 : if (b != lb)
607 12 : flags |= VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS;
608 :
609 32 : if (NULL == *async_frame ||
610 12 : vnet_crypto_async_frame_is_full (*async_frame))
611 : {
612 8 : *async_frame = vnet_crypto_async_get_frame (
613 : vm, VNET_CRYPTO_OP_CHACHA20_POLY1305_TAG16_AAD0_DEC);
614 8 : if (PREDICT_FALSE (NULL == *async_frame))
615 0 : goto error;
616 : /* Save the frame to the list we'll submit at the end */
617 8 : vec_add1 (ptd->async_frames, *async_frame);
618 : }
619 :
620 20 : wg_input_add_to_frame (vm, *async_frame, kp->kp_recv_index, srclen_total,
621 20 : src - b->data, buf_idx, async_next_node, iv, tag,
622 : flags);
623 : }
624 : else
625 : {
626 3612 : wg_prepare_sync_dec_op (vm, ptd, b, lb, crypto_ops, src, srclen, dst,
627 : NULL, 0, kp->kp_recv_index, from_idx, iv);
628 : }
629 :
630 : /* If we've received the handshake confirming data packet then move the
631 : * next keypair into current. If we do slide the next keypair in, then
632 : * we skip the REKEY_AFTER_TIME_RECV check. This is safe to do as a
633 : * data packet can't confirm a session that we are an INITIATOR of. */
634 3632 : if (kp == r->r_next)
635 : {
636 21 : clib_rwlock_writer_lock (&r->r_keypair_lock);
637 21 : if (kp == r->r_next && kp->kp_local_index == r_idx)
638 : {
639 21 : noise_remote_keypair_free (vm, r, &r->r_previous);
640 21 : r->r_previous = r->r_current;
641 21 : r->r_current = r->r_next;
642 21 : r->r_next = NULL;
643 :
644 21 : ret = SC_CONN_RESET;
645 21 : clib_rwlock_writer_unlock (&r->r_keypair_lock);
646 21 : goto error;
647 : }
648 0 : clib_rwlock_writer_unlock (&r->r_keypair_lock);
649 : }
650 :
651 : /* Similar to when we encrypt, we want to notify the caller when we
652 : * are approaching our tolerances. We notify if:
653 : * - we're the initiator and the current keypair is older than
654 : * REKEY_AFTER_TIME_RECV seconds. */
655 3611 : ret = SC_KEEP_KEY_FRESH;
656 3611 : kp = r->r_current;
657 4671 : if (kp != NULL && kp->kp_valid && kp->kp_is_initiator &&
658 1060 : wg_birthdate_has_expired_opt (kp->kp_birthdate, REKEY_AFTER_TIME_RECV,
659 : time))
660 0 : goto error;
661 :
662 3611 : ret = SC_OK;
663 3632 : error:
664 3632 : return ret;
665 : }
666 :
667 : static_always_inline void
668 3632 : wg_find_outer_addr_port (vlib_buffer_t *b, ip46_address_t *addr, u16 *port,
669 : u8 is_ip4)
670 : {
671 3632 : if (is_ip4)
672 : {
673 2078 : ip4_udp_header_t *ip4_udp_hdr =
674 2078 : vlib_buffer_get_current (b) - sizeof (ip4_udp_header_t);
675 2078 : ip46_address_set_ip4 (addr, &ip4_udp_hdr->ip4.src_address);
676 2078 : *port = clib_net_to_host_u16 (ip4_udp_hdr->udp.src_port);
677 : }
678 : else
679 : {
680 1554 : ip6_udp_header_t *ip6_udp_hdr =
681 1554 : vlib_buffer_get_current (b) - sizeof (ip6_udp_header_t);
682 1554 : ip46_address_set_ip6 (addr, &ip6_udp_hdr->ip6.src_address);
683 1554 : *port = clib_net_to_host_u16 (ip6_udp_hdr->udp.src_port);
684 : }
685 3632 : }
686 :
687 : always_inline uword
688 260 : wg_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
689 : vlib_frame_t *frame, u8 is_ip4, u16 async_next_node)
690 : {
691 260 : vnet_main_t *vnm = vnet_get_main ();
692 260 : vnet_interface_main_t *im = &vnm->interface_main;
693 260 : wg_main_t *wmp = &wg_main;
694 260 : wg_per_thread_data_t *ptd =
695 260 : vec_elt_at_index (wmp->per_thread_data, vm->thread_index);
696 260 : u32 *from = vlib_frame_vector_args (frame);
697 260 : u32 n_left_from = frame->n_vectors;
698 :
699 260 : vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
700 : vlib_buffer_t *lb;
701 260 : u32 thread_index = vm->thread_index;
702 : vnet_crypto_op_t **crypto_ops;
703 260 : const u16 drop_next = WG_INPUT_NEXT_PUNT;
704 : message_type_t header_type;
705 : vlib_buffer_t *data_bufs[VLIB_FRAME_SIZE];
706 : u32 data_bi[VLIB_FRAME_SIZE]; /* buffer index for data */
707 : u32 other_bi[VLIB_FRAME_SIZE]; /* buffer index for drop or handoff */
708 260 : u16 other_nexts[VLIB_FRAME_SIZE], *other_next = other_nexts, n_other = 0;
709 260 : u16 data_nexts[VLIB_FRAME_SIZE], *data_next = data_nexts, n_data = 0;
710 260 : u16 n_async = 0;
711 260 : const u8 is_async = wg_op_mode_is_set_ASYNC ();
712 260 : vnet_crypto_async_frame_t *async_frame = NULL;
713 :
714 260 : vlib_get_buffers (vm, from, bufs, n_left_from);
715 260 : vec_reset_length (ptd->crypto_ops);
716 260 : vec_reset_length (ptd->chained_crypto_ops);
717 260 : vec_reset_length (ptd->chunks);
718 260 : vec_reset_length (ptd->async_frames);
719 :
720 260 : f64 time = clib_time_now (&vm->clib_time) + vm->time_offset;
721 :
722 260 : wg_peer_t *peer = NULL;
723 260 : u32 *last_peer_time_idx = NULL;
724 260 : u32 last_rec_idx = ~0;
725 :
726 260 : bool is_keepalive = false;
727 260 : u32 *peer_idx = NULL;
728 260 : index_t peeri = INDEX_INVALID;
729 :
730 5537 : while (n_left_from > 0)
731 : {
732 5277 : if (n_left_from > 2)
733 : {
734 : u8 *p;
735 4955 : vlib_prefetch_buffer_header (b[2], LOAD);
736 4955 : p = vlib_buffer_get_current (b[1]);
737 4955 : CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD);
738 4955 : CLIB_PREFETCH (vlib_buffer_get_tail (b[1]), CLIB_CACHE_LINE_BYTES,
739 : LOAD);
740 : }
741 :
742 5277 : other_next[n_other] = WG_INPUT_NEXT_PUNT;
743 5277 : data_nexts[n_data] = WG_INPUT_N_NEXT;
744 :
745 5277 : header_type =
746 5277 : ((message_header_t *) vlib_buffer_get_current (b[0]))->type;
747 :
748 5277 : if (PREDICT_TRUE (header_type == MESSAGE_DATA))
749 : {
750 3887 : message_data_t *data = vlib_buffer_get_current (b[0]);
751 3887 : u8 *iv_data = b[0]->pre_data;
752 3887 : u32 buf_idx = from[b - bufs];
753 : u32 n_bufs;
754 3887 : peer_idx = wg_index_table_lookup (&wmp->index_table,
755 : data->receiver_index);
756 :
757 3887 : if (data->receiver_index != last_rec_idx)
758 : {
759 55 : peer_idx = wg_index_table_lookup (&wmp->index_table,
760 : data->receiver_index);
761 55 : if (PREDICT_TRUE (peer_idx != NULL))
762 : {
763 55 : peeri = *peer_idx;
764 55 : peer = wg_peer_get (peeri);
765 55 : last_rec_idx = data->receiver_index;
766 : }
767 : else
768 : {
769 0 : peer = NULL;
770 0 : last_rec_idx = ~0;
771 : }
772 : }
773 :
774 3887 : if (PREDICT_FALSE (!peer_idx))
775 : {
776 0 : other_next[n_other] = WG_INPUT_NEXT_ERROR;
777 0 : b[0]->error = node->errors[WG_INPUT_ERROR_PEER];
778 0 : other_bi[n_other] = buf_idx;
779 0 : n_other += 1;
780 0 : goto out;
781 : }
782 :
783 3887 : if (PREDICT_FALSE (~0 == peer->input_thread_index))
784 : {
785 : /* this is the first packet to use this peer, claim the peer
786 : * for this thread.
787 : */
788 41 : clib_atomic_cmp_and_swap (&peer->input_thread_index, ~0,
789 : wg_peer_assign_thread (thread_index));
790 : }
791 :
792 3887 : if (PREDICT_TRUE (thread_index != peer->input_thread_index))
793 : {
794 255 : other_next[n_other] = WG_INPUT_NEXT_HANDOFF_DATA;
795 255 : other_bi[n_other] = buf_idx;
796 255 : n_other += 1;
797 255 : goto next;
798 : }
799 :
800 3632 : lb = b[0];
801 3632 : n_bufs = vlib_buffer_chain_linearize (vm, b[0]);
802 3632 : if (n_bufs == 0)
803 : {
804 0 : other_next[n_other] = WG_INPUT_NEXT_ERROR;
805 0 : b[0]->error = node->errors[WG_INPUT_ERROR_NO_BUFFERS];
806 0 : other_bi[n_other] = buf_idx;
807 0 : n_other += 1;
808 0 : goto out;
809 : }
810 :
811 3632 : if (n_bufs > 1)
812 : {
813 24 : vlib_buffer_t *before_last = b[0];
814 :
815 : /* Find last and before last buffer in the chain */
816 56 : while (lb->flags & VLIB_BUFFER_NEXT_PRESENT)
817 : {
818 32 : before_last = lb;
819 32 : lb = vlib_get_buffer (vm, lb->next_buffer);
820 : }
821 :
822 : /* Ensure auth tag is contiguous and not splitted into two last
823 : * buffers */
824 24 : if (PREDICT_FALSE (lb->current_length < NOISE_AUTHTAG_LEN))
825 : {
826 8 : u32 len_diff = NOISE_AUTHTAG_LEN - lb->current_length;
827 :
828 8 : before_last->current_length -= len_diff;
829 8 : if (before_last == b[0])
830 8 : before_last->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
831 :
832 8 : vlib_buffer_advance (lb, (signed) -len_diff);
833 :
834 8 : clib_memcpy_fast (vlib_buffer_get_current (lb),
835 8 : vlib_buffer_get_tail (before_last),
836 : len_diff);
837 : }
838 : }
839 :
840 3632 : u16 encr_len = b[0]->current_length - sizeof (message_data_t);
841 3632 : u16 decr_len = encr_len - NOISE_AUTHTAG_LEN;
842 3632 : u16 encr_len_total =
843 3632 : vlib_buffer_length_in_chain (vm, b[0]) - sizeof (message_data_t);
844 3632 : u16 decr_len_total = encr_len_total - NOISE_AUTHTAG_LEN;
845 :
846 3632 : if (lb != b[0])
847 24 : crypto_ops = &ptd->chained_crypto_ops;
848 : else
849 3608 : crypto_ops = &ptd->crypto_ops;
850 :
851 : enum noise_state_crypt state_cr =
852 3632 : wg_input_process (vm, ptd, crypto_ops, &async_frame, b[0], lb,
853 : buf_idx, &peer->remote, data->receiver_index,
854 3632 : data->counter, data->encrypted_data, decr_len,
855 3632 : decr_len_total, data->encrypted_data, n_data,
856 : iv_data, time, is_async, async_next_node);
857 :
858 3632 : if (PREDICT_FALSE (state_cr == SC_FAILED))
859 : {
860 0 : wg_peer_update_flags (*peer_idx, WG_PEER_ESTABLISHED, false);
861 0 : other_next[n_other] = WG_INPUT_NEXT_ERROR;
862 0 : b[0]->error = node->errors[WG_INPUT_ERROR_DECRYPTION];
863 0 : other_bi[n_other] = buf_idx;
864 0 : n_other += 1;
865 0 : goto out;
866 : }
867 3632 : if (!is_async)
868 : {
869 3612 : data_bufs[n_data] = b[0];
870 3612 : data_bi[n_data] = buf_idx;
871 3612 : n_data += 1;
872 : }
873 : else
874 : {
875 20 : n_async += 1;
876 : }
877 :
878 3632 : if (PREDICT_FALSE (state_cr == SC_CONN_RESET))
879 : {
880 21 : wg_timers_handshake_complete (peer);
881 21 : goto next;
882 : }
883 3611 : else if (PREDICT_FALSE (state_cr == SC_KEEP_KEY_FRESH))
884 : {
885 0 : wg_send_handshake_from_mt (peeri, false);
886 0 : goto next;
887 : }
888 3611 : else if (PREDICT_TRUE (state_cr == SC_OK))
889 3611 : goto next;
890 : }
891 : else
892 : {
893 : /* Handshake packets should be processed in main thread */
894 1390 : if (thread_index != 0)
895 : {
896 462 : other_next[n_other] = WG_INPUT_NEXT_HANDOFF_HANDSHAKE;
897 462 : other_bi[n_other] = from[b - bufs];
898 462 : n_other += 1;
899 462 : goto next;
900 : }
901 :
902 : wg_input_error_t ret =
903 928 : wg_handshake_process (vm, wmp, b[0], node->node_index, is_ip4);
904 928 : if (ret != WG_INPUT_ERROR_NONE)
905 : {
906 798 : other_next[n_other] = WG_INPUT_NEXT_ERROR;
907 798 : b[0]->error = node->errors[ret];
908 798 : other_bi[n_other] = from[b - bufs];
909 798 : n_other += 1;
910 : }
911 : else
912 : {
913 130 : other_bi[n_other] = from[b - bufs];
914 130 : n_other += 1;
915 : }
916 : }
917 :
918 928 : out:
919 928 : if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
920 : (b[0]->flags & VLIB_BUFFER_IS_TRACED)))
921 : {
922 928 : wg_input_trace_t *t = vlib_add_trace (vm, node, b[0], sizeof (*t));
923 928 : t->type = header_type;
924 928 : t->current_length = b[0]->current_length;
925 928 : t->is_keepalive = is_keepalive;
926 928 : t->peer = peer_idx ? peeri : INDEX_INVALID;
927 : }
928 :
929 0 : next:
930 5277 : n_left_from -= 1;
931 5277 : b += 1;
932 : }
933 :
934 : /* decrypt packets */
935 260 : wg_input_process_ops (vm, node, ptd->crypto_ops, data_bufs, data_nexts,
936 : drop_next);
937 260 : wg_input_process_chained_ops (vm, node, ptd->chained_crypto_ops, data_bufs,
938 : data_nexts, ptd->chunks, drop_next);
939 :
940 : /* process after decryption */
941 260 : b = data_bufs;
942 260 : n_left_from = n_data;
943 260 : last_rec_idx = ~0;
944 260 : last_peer_time_idx = NULL;
945 :
946 3872 : while (n_left_from > 0)
947 : {
948 3612 : bool is_keepalive = false;
949 3612 : u32 *peer_idx = NULL;
950 :
951 3612 : if (PREDICT_FALSE (data_next[0] == WG_INPUT_NEXT_PUNT))
952 : {
953 0 : goto trace;
954 : }
955 3612 : if (n_left_from > 2)
956 : {
957 : u8 *p;
958 3545 : vlib_prefetch_buffer_header (b[2], LOAD);
959 3545 : p = vlib_buffer_get_current (b[1]);
960 3545 : CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD);
961 3545 : CLIB_PREFETCH (vlib_buffer_get_tail (b[1]), CLIB_CACHE_LINE_BYTES,
962 : LOAD);
963 : }
964 :
965 3612 : message_data_t *data = vlib_buffer_get_current (b[0]);
966 : ip46_address_t out_src_ip;
967 : u16 out_udp_src_port;
968 :
969 3612 : wg_find_outer_addr_port (b[0], &out_src_ip, &out_udp_src_port, is_ip4);
970 :
971 3612 : if (data->receiver_index != last_rec_idx)
972 : {
973 : peer_idx =
974 46 : wg_index_table_lookup (&wmp->index_table, data->receiver_index);
975 46 : if (PREDICT_TRUE (peer_idx != NULL))
976 : {
977 46 : peeri = *peer_idx;
978 46 : peer = wg_peer_get (peeri);
979 46 : last_rec_idx = data->receiver_index;
980 : }
981 : else
982 : {
983 0 : peer = NULL;
984 0 : last_rec_idx = ~0;
985 : }
986 : }
987 :
988 3612 : if (PREDICT_TRUE (peer != NULL))
989 : {
990 3612 : if (PREDICT_FALSE (wg_input_post_process (vm, b[0], data_next, peer,
991 : data, &is_keepalive) < 0))
992 0 : goto trace;
993 : }
994 : else
995 : {
996 0 : data_next[0] = WG_INPUT_NEXT_PUNT;
997 0 : goto trace;
998 : }
999 :
1000 3612 : if (PREDICT_FALSE (peer_idx && (last_peer_time_idx != peer_idx)))
1001 : {
1002 46 : if (PREDICT_FALSE (
1003 : !ip46_address_is_equal (&peer->dst.addr, &out_src_ip) ||
1004 : peer->dst.port != out_udp_src_port))
1005 4 : wg_peer_update_endpoint_from_mt (peeri, &out_src_ip,
1006 : out_udp_src_port);
1007 46 : wg_timers_any_authenticated_packet_received_opt (peer, time);
1008 46 : wg_timers_any_authenticated_packet_traversal (peer);
1009 46 : wg_peer_update_flags (*peer_idx, WG_PEER_ESTABLISHED, true);
1010 46 : last_peer_time_idx = peer_idx;
1011 : }
1012 :
1013 3612 : vlib_increment_combined_counter (im->combined_sw_if_counters +
1014 : VNET_INTERFACE_COUNTER_RX,
1015 : vm->thread_index, peer->wg_sw_if_index,
1016 3612 : 1 /* packets */, b[0]->current_length);
1017 :
1018 3612 : trace:
1019 3612 : if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
1020 : (b[0]->flags & VLIB_BUFFER_IS_TRACED)))
1021 : {
1022 3612 : wg_input_trace_t *t = vlib_add_trace (vm, node, b[0], sizeof (*t));
1023 3612 : t->type = header_type;
1024 3612 : t->current_length = b[0]->current_length;
1025 3612 : t->is_keepalive = is_keepalive;
1026 3612 : t->peer = peer_idx ? peeri : INDEX_INVALID;
1027 : }
1028 :
1029 3612 : b += 1;
1030 3612 : n_left_from -= 1;
1031 3612 : data_next += 1;
1032 : }
1033 :
1034 260 : if (n_async)
1035 : {
1036 : /* submit all of the open frames */
1037 : vnet_crypto_async_frame_t **async_frame;
1038 16 : vec_foreach (async_frame, ptd->async_frames)
1039 : {
1040 8 : if (PREDICT_FALSE (
1041 : vnet_crypto_async_submit_open_frame (vm, *async_frame) < 0))
1042 : {
1043 0 : u32 n_drop = (*async_frame)->n_elts;
1044 0 : u32 *bi = (*async_frame)->buffer_indices;
1045 0 : u16 index = n_other;
1046 0 : while (n_drop--)
1047 : {
1048 0 : other_bi[index] = bi[0];
1049 0 : vlib_buffer_t *b = vlib_get_buffer (vm, bi[0]);
1050 0 : other_nexts[index] = drop_next;
1051 0 : b->error = node->errors[WG_INPUT_ERROR_CRYPTO_ENGINE_ERROR];
1052 0 : bi++;
1053 0 : index++;
1054 : }
1055 0 : n_other += (*async_frame)->n_elts;
1056 :
1057 0 : vnet_crypto_async_reset_frame (*async_frame);
1058 0 : vnet_crypto_async_free_frame (vm, *async_frame);
1059 : }
1060 : }
1061 : }
1062 :
1063 : /* enqueue other bufs */
1064 260 : if (n_other)
1065 206 : vlib_buffer_enqueue_to_next (vm, node, other_bi, other_next, n_other);
1066 :
1067 : /* enqueue data bufs */
1068 260 : if (n_data)
1069 46 : vlib_buffer_enqueue_to_next (vm, node, data_bi, data_nexts, n_data);
1070 :
1071 260 : return frame->n_vectors;
1072 : }
1073 :
1074 : always_inline uword
1075 8 : wg_input_post (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame,
1076 : u8 is_ip4)
1077 : {
1078 8 : vnet_main_t *vnm = vnet_get_main ();
1079 8 : vnet_interface_main_t *im = &vnm->interface_main;
1080 8 : wg_main_t *wmp = &wg_main;
1081 8 : vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
1082 8 : u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
1083 8 : u32 *from = vlib_frame_vector_args (frame);
1084 8 : u32 n_left = frame->n_vectors;
1085 8 : wg_peer_t *peer = NULL;
1086 8 : u32 *peer_idx = NULL;
1087 8 : u32 *last_peer_time_idx = NULL;
1088 8 : index_t peeri = INDEX_INVALID;
1089 8 : u32 last_rec_idx = ~0;
1090 8 : f64 time = clib_time_now (&vm->clib_time) + vm->time_offset;
1091 :
1092 8 : vlib_get_buffers (vm, from, b, n_left);
1093 :
1094 8 : if (n_left >= 2)
1095 : {
1096 4 : vlib_prefetch_buffer_header (b[0], LOAD);
1097 4 : vlib_prefetch_buffer_header (b[1], LOAD);
1098 : }
1099 :
1100 28 : while (n_left > 0)
1101 : {
1102 20 : if (n_left > 2)
1103 : {
1104 : u8 *p;
1105 8 : vlib_prefetch_buffer_header (b[2], LOAD);
1106 8 : p = vlib_buffer_get_current (b[1]);
1107 8 : CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD);
1108 : }
1109 :
1110 20 : bool is_keepalive = false;
1111 20 : message_data_t *data = vlib_buffer_get_current (b[0]);
1112 : ip46_address_t out_src_ip;
1113 : u16 out_udp_src_port;
1114 :
1115 20 : wg_find_outer_addr_port (b[0], &out_src_ip, &out_udp_src_port, is_ip4);
1116 :
1117 20 : if (data->receiver_index != last_rec_idx)
1118 : {
1119 : peer_idx =
1120 8 : wg_index_table_lookup (&wmp->index_table, data->receiver_index);
1121 :
1122 8 : if (PREDICT_TRUE (peer_idx != NULL))
1123 : {
1124 8 : peeri = *peer_idx;
1125 8 : peer = wg_peer_get (peeri);
1126 8 : last_rec_idx = data->receiver_index;
1127 : }
1128 : else
1129 : {
1130 0 : peer = NULL;
1131 0 : last_rec_idx = ~0;
1132 : }
1133 : }
1134 :
1135 20 : if (PREDICT_TRUE (peer != NULL))
1136 : {
1137 20 : if (PREDICT_FALSE (wg_input_post_process (vm, b[0], next, peer, data,
1138 : &is_keepalive) < 0))
1139 0 : goto trace;
1140 : }
1141 : else
1142 : {
1143 0 : next[0] = WG_INPUT_NEXT_PUNT;
1144 0 : goto trace;
1145 : }
1146 :
1147 20 : if (PREDICT_FALSE (peer_idx && (last_peer_time_idx != peer_idx)))
1148 : {
1149 8 : if (PREDICT_FALSE (
1150 : !ip46_address_is_equal (&peer->dst.addr, &out_src_ip) ||
1151 : peer->dst.port != out_udp_src_port))
1152 4 : wg_peer_update_endpoint_from_mt (peeri, &out_src_ip,
1153 : out_udp_src_port);
1154 8 : wg_timers_any_authenticated_packet_received_opt (peer, time);
1155 8 : wg_timers_any_authenticated_packet_traversal (peer);
1156 8 : wg_peer_update_flags (*peer_idx, WG_PEER_ESTABLISHED, true);
1157 8 : last_peer_time_idx = peer_idx;
1158 : }
1159 :
1160 20 : vlib_increment_combined_counter (im->combined_sw_if_counters +
1161 : VNET_INTERFACE_COUNTER_RX,
1162 : vm->thread_index, peer->wg_sw_if_index,
1163 20 : 1 /* packets */, b[0]->current_length);
1164 :
1165 20 : trace:
1166 20 : if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
1167 : (b[0]->flags & VLIB_BUFFER_IS_TRACED)))
1168 : {
1169 : wg_input_post_trace_t *t =
1170 0 : vlib_add_trace (vm, node, b[0], sizeof (*t));
1171 0 : t->next = next[0];
1172 0 : t->peer = peer_idx ? peeri : INDEX_INVALID;
1173 : }
1174 :
1175 20 : b += 1;
1176 20 : next += 1;
1177 20 : n_left -= 1;
1178 : }
1179 :
1180 8 : vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
1181 8 : return frame->n_vectors;
1182 : }
1183 :
1184 736 : VLIB_NODE_FN (wg4_input_node)
1185 : (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1186 : {
1187 322 : return wg_input_inline (vm, node, frame, /* is_ip4 */ 1,
1188 161 : wg_decrypt_async_next.wg4_post_next);
1189 : }
1190 :
1191 674 : VLIB_NODE_FN (wg6_input_node)
1192 : (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1193 : {
1194 198 : return wg_input_inline (vm, node, frame, /* is_ip4 */ 0,
1195 99 : wg_decrypt_async_next.wg6_post_next);
1196 : }
1197 :
1198 579 : VLIB_NODE_FN (wg4_input_post_node)
1199 : (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
1200 : {
1201 4 : return wg_input_post (vm, node, from_frame, /* is_ip4 */ 1);
1202 : }
1203 :
1204 579 : VLIB_NODE_FN (wg6_input_post_node)
1205 : (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
1206 : {
1207 4 : return wg_input_post (vm, node, from_frame, /* is_ip4 */ 0);
1208 : }
1209 :
1210 : /* *INDENT-OFF* */
1211 1151 : VLIB_REGISTER_NODE (wg4_input_node) =
1212 : {
1213 : .name = "wg4-input",
1214 : .vector_size = sizeof (u32),
1215 : .format_trace = format_wg_input_trace,
1216 : .type = VLIB_NODE_TYPE_INTERNAL,
1217 : .n_errors = ARRAY_LEN (wg_input_error_strings),
1218 : .error_strings = wg_input_error_strings,
1219 : .n_next_nodes = WG_INPUT_N_NEXT,
1220 : /* edit / add dispositions here */
1221 : .next_nodes = {
1222 : [WG_INPUT_NEXT_HANDOFF_HANDSHAKE] = "wg4-handshake-handoff",
1223 : [WG_INPUT_NEXT_HANDOFF_DATA] = "wg4-input-data-handoff",
1224 : [WG_INPUT_NEXT_IP4_INPUT] = "ip4-input-no-checksum",
1225 : [WG_INPUT_NEXT_IP6_INPUT] = "ip6-input",
1226 : [WG_INPUT_NEXT_PUNT] = "error-punt",
1227 : [WG_INPUT_NEXT_ERROR] = "error-drop",
1228 : },
1229 : };
1230 :
1231 1151 : VLIB_REGISTER_NODE (wg6_input_node) =
1232 : {
1233 : .name = "wg6-input",
1234 : .vector_size = sizeof (u32),
1235 : .format_trace = format_wg_input_trace,
1236 : .type = VLIB_NODE_TYPE_INTERNAL,
1237 : .n_errors = ARRAY_LEN (wg_input_error_strings),
1238 : .error_strings = wg_input_error_strings,
1239 : .n_next_nodes = WG_INPUT_N_NEXT,
1240 : /* edit / add dispositions here */
1241 : .next_nodes = {
1242 : [WG_INPUT_NEXT_HANDOFF_HANDSHAKE] = "wg6-handshake-handoff",
1243 : [WG_INPUT_NEXT_HANDOFF_DATA] = "wg6-input-data-handoff",
1244 : [WG_INPUT_NEXT_IP4_INPUT] = "ip4-input-no-checksum",
1245 : [WG_INPUT_NEXT_IP6_INPUT] = "ip6-input",
1246 : [WG_INPUT_NEXT_PUNT] = "error-punt",
1247 : [WG_INPUT_NEXT_ERROR] = "error-drop",
1248 : },
1249 : };
1250 :
1251 1151 : VLIB_REGISTER_NODE (wg4_input_post_node) = {
1252 : .name = "wg4-input-post-node",
1253 : .vector_size = sizeof (u32),
1254 : .format_trace = format_wg_input_post_trace,
1255 : .type = VLIB_NODE_TYPE_INTERNAL,
1256 : .sibling_of = "wg4-input",
1257 :
1258 : .n_errors = ARRAY_LEN (wg_input_error_strings),
1259 : .error_strings = wg_input_error_strings,
1260 : };
1261 :
1262 1151 : VLIB_REGISTER_NODE (wg6_input_post_node) = {
1263 : .name = "wg6-input-post-node",
1264 : .vector_size = sizeof (u32),
1265 : .format_trace = format_wg_input_post_trace,
1266 : .type = VLIB_NODE_TYPE_INTERNAL,
1267 : .sibling_of = "wg6-input",
1268 :
1269 : .n_errors = ARRAY_LEN (wg_input_error_strings),
1270 : .error_strings = wg_input_error_strings,
1271 : };
1272 :
1273 : /* *INDENT-ON* */
1274 :
1275 : /*
1276 : * fd.io coding-style-patch-verification: ON
1277 : *
1278 : * Local Variables:
1279 : * eval: (c-set-style "gnu")
1280 : * End:
1281 : */
|
