Line data Source code
1 : /*
2 : *------------------------------------------------------------------
3 : * Copyright (c) 2019 Cisco and/or its affiliates.
4 : * Licensed under the Apache License, Version 2.0 (the "License");
5 : * you may not use this file except in compliance with the License.
6 : * You may obtain a copy of the License at:
7 : *
8 : * http://www.apache.org/licenses/LICENSE-2.0
9 : *
10 : * Unless required by applicable law or agreed to in writing, software
11 : * distributed under the License is distributed on an "AS IS" BASIS,
12 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 : * See the License for the specific language governing permissions and
14 : * limitations under the License.
15 : *------------------------------------------------------------------
16 : */
17 :
18 : #include <vlib/vlib.h>
19 : #include <vnet/plugin/plugin.h>
20 : #include <vnet/crypto/crypto.h>
21 : #include <crypto_native/crypto_native.h>
22 : #include <vppinfra/crypto/aes_gcm.h>
23 :
24 : #if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
25 : #pragma GCC optimize("O3")
26 : #endif
27 :
28 : static_always_inline u32
29 51294 : aes_ops_enc_aes_gcm (vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops,
30 : aes_key_size_t ks)
31 : {
32 51294 : crypto_native_main_t *cm = &crypto_native_main;
33 51294 : vnet_crypto_op_t *op = ops[0];
34 : aes_gcm_key_data_t *kd;
35 51294 : u32 n_left = n_ops;
36 :
37 71246 : next:
38 71246 : kd = (aes_gcm_key_data_t *) cm->key_data[op->key_index];
39 71246 : aes_gcm (op->src, op->dst, op->aad, (u8 *) op->iv, op->tag, op->len,
40 71246 : op->aad_len, op->tag_len, kd, AES_KEY_ROUNDS (ks),
41 : AES_GCM_OP_ENCRYPT);
42 71246 : op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
43 :
44 71246 : if (--n_left)
45 : {
46 19952 : op += 1;
47 19952 : goto next;
48 : }
49 :
50 51294 : return n_ops;
51 : }
52 :
53 : static_always_inline u32
54 51112 : aes_ops_dec_aes_gcm (vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops,
55 : aes_key_size_t ks)
56 : {
57 51112 : crypto_native_main_t *cm = &crypto_native_main;
58 51112 : vnet_crypto_op_t *op = ops[0];
59 : aes_gcm_key_data_t *kd;
60 51112 : u32 n_left = n_ops;
61 : int rv;
62 :
63 69795 : next:
64 69795 : kd = (aes_gcm_key_data_t *) cm->key_data[op->key_index];
65 69795 : rv = aes_gcm (op->src, op->dst, op->aad, (u8 *) op->iv, op->tag, op->len,
66 69795 : op->aad_len, op->tag_len, kd, AES_KEY_ROUNDS (ks),
67 : AES_GCM_OP_DECRYPT);
68 :
69 69795 : if (rv)
70 : {
71 69447 : op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
72 : }
73 : else
74 : {
75 348 : op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC;
76 348 : n_ops--;
77 : }
78 :
79 69795 : if (--n_left)
80 : {
81 18683 : op += 1;
82 18683 : goto next;
83 : }
84 :
85 51112 : return n_ops;
86 : }
87 :
88 : static_always_inline void *
89 103154 : aes_gcm_key_exp (vnet_crypto_key_t *key, aes_key_size_t ks)
90 : {
91 : aes_gcm_key_data_t *kd;
92 :
93 103154 : kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES);
94 :
95 103154 : clib_aes_gcm_key_expand (kd, key->data, ks);
96 :
97 103154 : return kd;
98 : }
99 :
100 : #define foreach_aes_gcm_handler_type _ (128) _ (192) _ (256)
101 :
102 : #define _(x) \
103 : static u32 aes_ops_dec_aes_gcm_##x (vlib_main_t *vm, \
104 : vnet_crypto_op_t *ops[], u32 n_ops) \
105 : { \
106 : return aes_ops_dec_aes_gcm (vm, ops, n_ops, AES_KEY_##x); \
107 : } \
108 : static u32 aes_ops_enc_aes_gcm_##x (vlib_main_t *vm, \
109 : vnet_crypto_op_t *ops[], u32 n_ops) \
110 : { \
111 : return aes_ops_enc_aes_gcm (vm, ops, n_ops, AES_KEY_##x); \
112 : } \
113 : static void *aes_gcm_key_exp_##x (vnet_crypto_key_t *key) \
114 : { \
115 : return aes_gcm_key_exp (key, AES_KEY_##x); \
116 : }
117 :
118 205560 : foreach_aes_gcm_handler_type;
119 : #undef _
120 :
121 : clib_error_t *
122 : #if defined(__VAES__) && defined(__AVX512F__)
123 0 : crypto_native_aes_gcm_init_icl (vlib_main_t *vm)
124 : #elif defined(__VAES__)
125 : crypto_native_aes_gcm_init_adl (vlib_main_t *vm)
126 : #elif __AVX512F__
127 575 : crypto_native_aes_gcm_init_skx (vlib_main_t *vm)
128 : #elif __AVX2__
129 0 : crypto_native_aes_gcm_init_hsw (vlib_main_t *vm)
130 : #elif __aarch64__
131 : crypto_native_aes_gcm_init_neon (vlib_main_t *vm)
132 : #else
133 0 : crypto_native_aes_gcm_init_slm (vlib_main_t *vm)
134 : #endif
135 : {
136 575 : crypto_native_main_t *cm = &crypto_native_main;
137 :
138 : #define _(x) \
139 : vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \
140 : VNET_CRYPTO_OP_AES_##x##_GCM_ENC, \
141 : aes_ops_enc_aes_gcm_##x); \
142 : vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \
143 : VNET_CRYPTO_OP_AES_##x##_GCM_DEC, \
144 : aes_ops_dec_aes_gcm_##x); \
145 : cm->key_fn[VNET_CRYPTO_ALG_AES_##x##_GCM] = aes_gcm_key_exp_##x;
146 575 : foreach_aes_gcm_handler_type;
147 : #undef _
148 575 : return 0;
149 : }
150 :
151 : /*
152 : * fd.io coding-style-patch-verification: ON
153 : *
154 : * Local Variables:
155 : * eval: (c-set-style "gnu")
156 : * End:
157 : */
|
