dummy/linux-stable-mirror
1
0
Fork 1
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/ synced 2025-04-19 20:58:31 +09:00
Code Issues Packages Projects Releases Wiki Activity

bpf-next-6.15

Browse Source 
-----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEE+soXsSLHKoYyzcli6rmadz2vbToFAmfi6ZAACgkQ6rmadz2v
 bTpLOg/+J7xUddPMhlpFAUlifQEadE5hmw6v1tXpM3zyKHzUWJiv/qsx3j8/ckgD
 D+d4P8bqIbI9SSuIS4oZ0+D9pr/g7GYztnoYZmPiYJ7v2AijPuof5dsagFQE8E2y
 rhfbt9KHTMzzkdkTvaAZaITS/HWAoJ2YVRB6gfLex2ghcXYHcgmtKRZniQrbBiFZ
 MIXBN8Rg6HP+pUdIVllSXFcQCb3XIgjPONRAos4hr5tIm+3Ku7Jvkgk2H/9vUcoF
 bdXAcg8xygyH7eY+1l3e7nEPQlG0jUZEsL+tq+vpdoLRLqlIpAUYmwUvqcmq4dPS
 QGFjiUcpDbXlxsUFpzjXHIFto7fXCfND7HEICQPwAncdflIIfYaATSQUfkEexn0a
 wBCFlAChrEzAmg2vFl4EeEr0fdSe/3jswrgKx0m6ctKieMjgloBUeeH4fXOpfkhS
 9tvhuduVFuronlebM8ew4w9T/mBgbyxkE5KkvP4hNeB3ni3N0K6Mary5/u2HyN1e
 lqTlnZxRA4p6lrvxce/mDrR4VSwlKLcSeQVjxAL1afD5KRkuZJnUv7bUhS361vkG
 IjNrQX30EisDAz+X7tMn3ndBf9vVatwFT4+c3yaxlQRor1WofhDfT88HPiyB4QqQ
 Kdx2EHgbQxJp4vkzhp4/OXlTfkihsMEn8egzZuphdPEQ9Y+Jdwg=
 =aN/V
 -----END PGP SIGNATURE-----

Merge tag 'bpf-next-6.15' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Pull bpf updates from Alexei Starovoitov:
 "For this merge window we're splitting BPF pull request into three for
  higher visibility: main changes, res_spin_lock, try_alloc_pages.

  These are the main BPF changes:

   - Add DFA-based live registers analysis to improve verification of
     programs with loops (Eduard Zingerman)

   - Introduce load_acquire and store_release BPF instructions and add
     x86, arm64 JIT support (Peilin Ye)

   - Fix loop detection logic in the verifier (Eduard Zingerman)

   - Drop unnecesary lock in bpf_map_inc_not_zero() (Eric Dumazet)

   - Add kfunc for populating cpumask bits (Emil Tsalapatis)

   - Convert various shell based tests to selftests/bpf/test_progs
     format (Bastien Curutchet)

   - Allow passing referenced kptrs into struct_ops callbacks (Amery
     Hung)

   - Add a flag to LSM bpf hook to facilitate bpf program signing
     (Blaise Boscaccy)

   - Track arena arguments in kfuncs (Ihor Solodrai)

   - Add copy_remote_vm_str() helper for reading strings from remote VM
     and bpf_copy_from_user_task_str() kfunc (Jordan Rome)

   - Add support for timed may_goto instruction (Kumar Kartikeya
     Dwivedi)

   - Allow bpf_get_netns_cookie() int cgroup_skb programs (Mahe Tardy)

   - Reduce bpf_cgrp_storage_busy false positives when accessing cgroup
     local storage (Martin KaFai Lau)

   - Introduce bpf_dynptr_copy() kfunc (Mykyta Yatsenko)

   - Allow retrieving BTF data with BTF token (Mykyta Yatsenko)

   - Add BPF kfuncs to set and get xattrs with 'security.bpf.' prefix
     (Song Liu)

   - Reject attaching programs to noreturn functions (Yafang Shao)

   - Introduce pre-order traversal of cgroup bpf programs (Yonghong
     Song)"

* tag 'bpf-next-6.15' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next: (186 commits)
  selftests/bpf: Add selftests for load-acquire/store-release when register number is invalid
  bpf: Fix out-of-bounds read in check_atomic_load/store()
  libbpf: Add namespace for errstr making it libbpf_errstr
  bpf: Add struct_ops context information to struct bpf_prog_aux
  selftests/bpf: Sanitize pointer prior fclose()
  selftests/bpf: Migrate test_xdp_vlan.sh into test_progs
  selftests/bpf: test_xdp_vlan: Rename BPF sections
  bpf: clarify a misleading verifier error message
  selftests/bpf: Add selftest for attaching fexit to __noreturn functions
  bpf: Reject attaching fexit/fmod_ret to __noreturn functions
  bpf: Only fails the busy counter check in bpf_cgrp_storage_get if it creates storage
  bpf: Make perf_event_read_output accessible in all program types.
  bpftool: Using the right format specifiers
  bpftool: Add -Wformat-signedness flag to detect format errors
  selftests/bpf: Test freplace from user namespace
  libbpf: Pass BPF token from find_prog_btf_id to BPF_BTF_GET_FD_BY_ID
  bpf: Return prog btf_id without capable check
  bpf: BPF token support for BPF_BTF_GET_FD_BY_ID
  bpf, x86: Fix objtool warning for timed may_goto
  bpf: Check map->record at the beginning of check_and_free_fields()
  ...
This commit is contained in:
Linus Torvalds 2025-03-30 12:43:03 -07:00
commit fa593d0f96
213 changed files with 10654 additions and 3508 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Documentation/bpf/bpf_iterators.rst
View File

@ -86,7 +86,7 @@ following steps:
The following are a few examples of selftest BPF iterator programs:
* `bpf_iter_tcp4.c <https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c>`_
* `bpf_iter_task_vma.c <https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/tools/testing/selftests/bpf/progs/bpf_iter_task_vma.c>`_
* `bpf_iter_task_vmas.c <https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/tools/testing/selftests/bpf/progs/bpf_iter_task_vmas.c>`_
* `bpf_iter_task_file.c <https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/tools/testing/selftests/bpf/progs/bpf_iter_task_file.c>`_
Let us look at ``bpf_iter_task_file.c``, which runs in kernel space:

25
Documentation/bpf/btf.rst
View File

@ -102,7 +102,8 @@ Each type contains the following common data::
* bits 24-28: kind (e.g. int, ptr, array...etc)
* bits 29-30: unused
* bit 31: kind_flag, currently used by
* struct, union, fwd, enum and enum64.
* struct, union, enum, fwd, enum64,
* decl_tag and type_tag
*/
__u32 info;
/* "size" is used by INT, ENUM, STRUCT, UNION and ENUM64.
@ -478,7 +479,7 @@ No additional type data follow ``btf_type``.
``struct btf_type`` encoding requirement:
* ``name_off``: offset to a non-empty string
* ``info.kind_flag``: 0
* ``info.kind_flag``: 0 or 1
* ``info.kind``: BTF_KIND_DECL_TAG
* ``info.vlen``: 0
* ``type``: ``struct``, ``union``, ``func``, ``var`` or ``typedef``
@ -489,7 +490,6 @@ No additional type data follow ``btf_type``.
__u32 component_idx;
};
The ``name_off`` encodes btf_decl_tag attribute string.
The ``type`` should be ``struct``, ``union``, ``func``, ``var`` or ``typedef``.
For ``var`` or ``typedef`` type, ``btf_decl_tag.component_idx`` must be ``-1``.
For the other three types, if the btf_decl_tag attribute is
@ -499,12 +499,21 @@ the attribute is applied to a ``struct``/``union`` member or
a ``func`` argument, and ``btf_decl_tag.component_idx`` should be a
valid index (starting from 0) pointing to a member or an argument.
If ``info.kind_flag`` is 0, then this is a normal decl tag, and the
``name_off`` encodes btf_decl_tag attribute string.
If ``info.kind_flag`` is 1, then the decl tag represents an arbitrary
__attribute__. In this case, ``name_off`` encodes a string
representing the attribute-list of the attribute specifier. For
example, for an ``__attribute__((aligned(4)))`` the string's contents
is ``aligned(4)``.
2.2.18 BTF_KIND_TYPE_TAG
~~~~~~~~~~~~~~~~~~~~~~~~
``struct btf_type`` encoding requirement:
* ``name_off``: offset to a non-empty string
* ``info.kind_flag``: 0
* ``info.kind_flag``: 0 or 1
* ``info.kind``: BTF_KIND_TYPE_TAG
* ``info.vlen``: 0
* ``type``: the type with ``btf_type_tag`` attribute
@ -522,6 +531,14 @@ type_tag, then zero or more const/volatile/restrict/typedef
and finally the base type. The base type is one of
int, ptr, array, struct, union, enum, func_proto and float types.
Similarly to decl tags, if the ``info.kind_flag`` is 0, then this is a
normal type tag, and the ``name_off`` encodes btf_type_tag attribute
string.
If ``info.kind_flag`` is 1, then the type tag represents an arbitrary
__attribute__, and the ``name_off`` encodes a string representing the
attribute-list of the attribute specifier.
2.2.19 BTF_KIND_ENUM64
~~~~~~~~~~~~~~~~~~~~~~

20
Documentation/bpf/standardization/instruction-set.rst
View File

@ -324,34 +324,42 @@ register.
.. table:: Arithmetic instructions
===== ===== ======= ==========================================================
===== ===== ======= ===================================================================================
name code offset description
===== ===== ======= ==========================================================
===== ===== ======= ===================================================================================
ADD 0x0 0 dst += src
SUB 0x1 0 dst -= src
MUL 0x2 0 dst \*= src
DIV 0x3 0 dst = (src != 0) ? (dst / src) : 0
SDIV 0x3 1 dst = (src != 0) ? (dst s/ src) : 0
SDIV 0x3 1 dst = (src == 0) ? 0 : ((src == -1 && dst == LLONG_MIN) ? LLONG_MIN : (dst s/ src))
OR 0x4 0 dst \|= src
AND 0x5 0 dst &= src
LSH 0x6 0 dst <<= (src & mask)
RSH 0x7 0 dst >>= (src & mask)
NEG 0x8 0 dst = -dst
MOD 0x9 0 dst = (src != 0) ? (dst % src) : dst
SMOD 0x9 1 dst = (src != 0) ? (dst s% src) : dst
SMOD 0x9 1 dst = (src == 0) ? dst : ((src == -1 && dst == LLONG_MIN) ? 0: (dst s% src))
XOR 0xa 0 dst ^= src
MOV 0xb 0 dst = src
MOVSX 0xb 8/16/32 dst = (s8,s16,s32)src
ARSH 0xc 0 :term:`sign extending<Sign Extend>` dst >>= (src & mask)
END 0xd 0 byte swap operations (see `Byte swap instructions`_ below)
===== ===== ======= ==========================================================
===== ===== ======= ===================================================================================
Underflow and overflow are allowed during arithmetic operations, meaning
the 64-bit or 32-bit value will wrap. If BPF program execution would
result in division by zero, the destination register is instead set to zero.
Otherwise, for ``ALU64``, if execution would result in ``LLONG_MIN``
dividing -1, the desination register is instead set to ``LLONG_MIN``. For
``ALU``, if execution would result in ``INT_MIN`` dividing -1, the
desination register is instead set to ``INT_MIN``.
If execution would result in modulo by zero, for ``ALU64`` the value of
the destination register is unchanged whereas for ``ALU`` the upper
32 bits of the destination register are zeroed.
32 bits of the destination register are zeroed. Otherwise, for ``ALU64``,
if execution would resuslt in ``LLONG_MIN`` modulo -1, the destination
register is instead set to 0. For ``ALU``, if execution would result in
``INT_MIN`` modulo -1, the destination register is instead set to 0.
``{ADD, X, ALU}``, where 'code' = ``ADD``, 'source' = ``X``, and 'class' = ``ALU``, means::

12
arch/arm64/include/asm/insn.h
View File

@ -188,8 +188,10 @@ enum aarch64_insn_ldst_type {
AARCH64_INSN_LDST_STORE_PAIR_PRE_INDEX,
AARCH64_INSN_LDST_LOAD_PAIR_POST_INDEX,
AARCH64_INSN_LDST_STORE_PAIR_POST_INDEX,
AARCH64_INSN_LDST_LOAD_ACQ,
AARCH64_INSN_LDST_LOAD_EX,
AARCH64_INSN_LDST_LOAD_ACQ_EX,
AARCH64_INSN_LDST_STORE_REL,
AARCH64_INSN_LDST_STORE_EX,
AARCH64_INSN_LDST_STORE_REL_EX,
AARCH64_INSN_LDST_SIGNED_LOAD_IMM_OFFSET,
@ -351,8 +353,10 @@ __AARCH64_INSN_FUNCS(ldr_imm, 0x3FC00000, 0x39400000)
__AARCH64_INSN_FUNCS(ldr_lit, 0xBF000000, 0x18000000)
__AARCH64_INSN_FUNCS(ldrsw_lit, 0xFF000000, 0x98000000)
__AARCH64_INSN_FUNCS(exclusive, 0x3F800000, 0x08000000)
__AARCH64_INSN_FUNCS(load_ex, 0x3F400000, 0x08400000)
__AARCH64_INSN_FUNCS(store_ex, 0x3F400000, 0x08000000)
__AARCH64_INSN_FUNCS(load_acq, 0x3FDFFC00, 0x08DFFC00)
__AARCH64_INSN_FUNCS(store_rel, 0x3FDFFC00, 0x089FFC00)
__AARCH64_INSN_FUNCS(load_ex, 0x3FC00000, 0x08400000)
__AARCH64_INSN_FUNCS(store_ex, 0x3FC00000, 0x08000000)
__AARCH64_INSN_FUNCS(mops, 0x3B200C00, 0x19000400)
__AARCH64_INSN_FUNCS(stp, 0x7FC00000, 0x29000000)
__AARCH64_INSN_FUNCS(ldp, 0x7FC00000, 0x29400000)
@ -602,6 +606,10 @@ u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1,
int offset,
enum aarch64_insn_variant variant,
enum aarch64_insn_ldst_type type);
u32 aarch64_insn_gen_load_acq_store_rel(enum aarch64_insn_register reg,
enum aarch64_insn_register base,
enum aarch64_insn_size_type size,
enum aarch64_insn_ldst_type type);
u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,
enum aarch64_insn_register base,
enum aarch64_insn_register state,

29
arch/arm64/lib/insn.c
View File

@ -540,6 +540,35 @@ u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1,
offset >> shift);
}
u32 aarch64_insn_gen_load_acq_store_rel(enum aarch64_insn_register reg,
enum aarch64_insn_register base,
enum aarch64_insn_size_type size,
enum aarch64_insn_ldst_type type)
{
u32 insn;
switch (type) {
case AARCH64_INSN_LDST_LOAD_ACQ:
insn = aarch64_insn_get_load_acq_value();
break;
case AARCH64_INSN_LDST_STORE_REL:
insn = aarch64_insn_get_store_rel_value();
break;
default:
pr_err("%s: unknown load-acquire/store-release encoding %d\n",
__func__, type);
return AARCH64_BREAK_FAULT;
}
insn = aarch64_insn_encode_ldst_size(size, insn);
insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
reg);
return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
base);
}
u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,
enum aarch64_insn_register base,
enum aarch64_insn_register state,

20
arch/arm64/net/bpf_jit.h
View File

@ -119,6 +119,26 @@
aarch64_insn_gen_load_store_ex(Rt, Rn, Rs, A64_SIZE(sf), \
AARCH64_INSN_LDST_STORE_REL_EX)
/* Load-acquire & store-release */
#define A64_LDAR(Rt, Rn, size) \
aarch64_insn_gen_load_acq_store_rel(Rt, Rn, AARCH64_INSN_SIZE_##size, \
AARCH64_INSN_LDST_LOAD_ACQ)
#define A64_STLR(Rt, Rn, size) \
aarch64_insn_gen_load_acq_store_rel(Rt, Rn, AARCH64_INSN_SIZE_##size, \
AARCH64_INSN_LDST_STORE_REL)
/* Rt = [Rn] (load acquire) */
#define A64_LDARB(Wt, Xn) A64_LDAR(Wt, Xn, 8)
#define A64_LDARH(Wt, Xn) A64_LDAR(Wt, Xn, 16)
#define A64_LDAR32(Wt, Xn) A64_LDAR(Wt, Xn, 32)
#define A64_LDAR64(Xt, Xn) A64_LDAR(Xt, Xn, 64)
/* [Rn] = Rt (store release) */
#define A64_STLRB(Wt, Xn) A64_STLR(Wt, Xn, 8)
#define A64_STLRH(Wt, Xn) A64_STLR(Wt, Xn, 16)
#define A64_STLR32(Wt, Xn) A64_STLR(Wt, Xn, 32)
#define A64_STLR64(Xt, Xn) A64_STLR(Xt, Xn, 64)
/*
* LSE atomics
*

92
arch/arm64/net/bpf_jit_comp.c
View File

@ -272,7 +272,7 @@ static inline void emit_a64_add_i(const bool is64, const int dst, const int src,
{
if (is_addsub_imm(imm)) {
emit(A64_ADD_I(is64, dst, src, imm), ctx);
} else if (is_addsub_imm(-imm)) {
} else if (is_addsub_imm(-(u32)imm)) {
emit(A64_SUB_I(is64, dst, src, -imm), ctx);
} else {
emit_a64_mov_i(is64, tmp, imm, ctx);
@ -647,6 +647,81 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
return 0;
}
static int emit_atomic_ld_st(const struct bpf_insn *insn, struct jit_ctx *ctx)
{
const s32 imm = insn->imm;
const s16 off = insn->off;
const u8 code = insn->code;
const bool arena = BPF_MODE(code) == BPF_PROBE_ATOMIC;
const u8 arena_vm_base = bpf2a64[ARENA_VM_START];
const u8 dst = bpf2a64[insn->dst_reg];
const u8 src = bpf2a64[insn->src_reg];
const u8 tmp = bpf2a64[TMP_REG_1];
u8 reg;
switch (imm) {
case BPF_LOAD_ACQ:
reg = src;
break;
case BPF_STORE_REL:
reg = dst;
break;
default:
pr_err_once("unknown atomic load/store op code %02x\n", imm);
return -EINVAL;
}
if (off) {
emit_a64_add_i(1, tmp, reg, tmp, off, ctx);
reg = tmp;
}
if (arena) {
emit(A64_ADD(1, tmp, reg, arena_vm_base), ctx);
reg = tmp;
}
switch (imm) {
case BPF_LOAD_ACQ:
switch (BPF_SIZE(code)) {
case BPF_B:
emit(A64_LDARB(dst, reg), ctx);
break;
case BPF_H:
emit(A64_LDARH(dst, reg), ctx);
break;
case BPF_W:
emit(A64_LDAR32(dst, reg), ctx);
break;
case BPF_DW:
emit(A64_LDAR64(dst, reg), ctx);
break;
}
break;
case BPF_STORE_REL:
switch (BPF_SIZE(code)) {
case BPF_B:
emit(A64_STLRB(src, reg), ctx);
break;
case BPF_H:
emit(A64_STLRH(src, reg), ctx);
break;
case BPF_W:
emit(A64_STLR32(src, reg), ctx);
break;
case BPF_DW:
emit(A64_STLR64(src, reg), ctx);
break;
}
break;
default:
pr_err_once("unexpected atomic load/store op code %02x\n",
imm);
return -EINVAL;
}
return 0;
}
#ifdef CONFIG_ARM64_LSE_ATOMICS
static int emit_lse_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx)
{
@ -1159,7 +1234,7 @@ emit_bswap_uxt:
case BPF_ALU64 | BPF_SUB | BPF_K:
if (is_addsub_imm(imm)) {
emit(A64_SUB_I(is64, dst, dst, imm), ctx);
} else if (is_addsub_imm(-imm)) {
} else if (is_addsub_imm(-(u32)imm)) {
emit(A64_ADD_I(is64, dst, dst, -imm), ctx);
} else {
emit_a64_mov_i(is64, tmp, imm, ctx);
@ -1330,7 +1405,7 @@ emit_cond_jmp:
case BPF_JMP32 | BPF_JSLE | BPF_K:
if (is_addsub_imm(imm)) {
emit(A64_CMP_I(is64, dst, imm), ctx);
} else if (is_addsub_imm(-imm)) {
} else if (is_addsub_imm(-(u32)imm)) {
emit(A64_CMN_I(is64, dst, -imm), ctx);
} else {
emit_a64_mov_i(is64, tmp, imm, ctx);
@ -1641,11 +1716,17 @@ emit_cond_jmp:
return ret;
break;
case BPF_STX | BPF_ATOMIC | BPF_B:
case BPF_STX | BPF_ATOMIC | BPF_H:
case BPF_STX | BPF_ATOMIC | BPF_W:
case BPF_STX | BPF_ATOMIC | BPF_DW:
case BPF_STX | BPF_PROBE_ATOMIC | BPF_B:
case BPF_STX | BPF_PROBE_ATOMIC | BPF_H:
case BPF_STX | BPF_PROBE_ATOMIC | BPF_W:
case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW:
if (cpus_have_cap(ARM64_HAS_LSE_ATOMICS))
if (bpf_atomic_is_load_store(insn))
ret = emit_atomic_ld_st(insn, ctx);
else if (cpus_have_cap(ARM64_HAS_LSE_ATOMICS))
ret = emit_lse_atomic(insn, ctx);
else
ret = emit_ll_sc_atomic(insn, ctx);
@ -2669,7 +2750,8 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
switch (insn->code) {
case BPF_STX | BPF_ATOMIC | BPF_W:
case BPF_STX | BPF_ATOMIC | BPF_DW:
if (!cpus_have_cap(ARM64_HAS_LSE_ATOMICS))
if (!bpf_atomic_is_load_store(insn) &&
!cpus_have_cap(ARM64_HAS_LSE_ATOMICS))
return false;
}
return true;

14
arch/s390/net/bpf_jit_comp.c
View File

@ -2919,10 +2919,16 @@ bool bpf_jit_supports_arena(void)
bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
{
/*
* Currently the verifier uses this function only to check which
* atomic stores to arena are supported, and they all are.
*/
if (!in_arena)
return true;
switch (insn->code) {
case BPF_STX | BPF_ATOMIC | BPF_B:
case BPF_STX | BPF_ATOMIC | BPF_H:
case BPF_STX | BPF_ATOMIC | BPF_W:
case BPF_STX | BPF_ATOMIC | BPF_DW:
if (bpf_atomic_is_load_store(insn))
return false;
}
return true;
}

2
arch/x86/net/Makefile
View File

@ -6,5 +6,5 @@
ifeq ($(CONFIG_X86_32),y)
obj-$(CONFIG_BPF_JIT) += bpf_jit_comp32.o
else
obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o
obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o bpf_timed_may_goto.o
endif

100
arch/x86/net/bpf_jit_comp.c
View File

@ -1250,8 +1250,8 @@ static void emit_st_r12(u8 **pprog, u32 size, u32 dst_reg, int off, int imm)
emit_st_index(pprog, size, dst_reg, X86_REG_R12, off, imm);
}
static int emit_atomic(u8 **pprog, u8 atomic_op,
u32 dst_reg, u32 src_reg, s16 off, u8 bpf_size)
static int emit_atomic_rmw(u8 **pprog, u32 atomic_op,
u32 dst_reg, u32 src_reg, s16 off, u8 bpf_size)
{
u8 *prog = *pprog;
@ -1291,8 +1291,9 @@ static int emit_atomic(u8 **pprog, u8 atomic_op,
return 0;
}
static int emit_atomic_index(u8 **pprog, u8 atomic_op, u32 size,
u32 dst_reg, u32 src_reg, u32 index_reg, int off)
static int emit_atomic_rmw_index(u8 **pprog, u32 atomic_op, u32 size,
u32 dst_reg, u32 src_reg, u32 index_reg,
int off)
{
u8 *prog = *pprog;
@ -1305,7 +1306,7 @@ static int emit_atomic_index(u8 **pprog, u8 atomic_op, u32 size,
EMIT1(add_3mod(0x48, dst_reg, src_reg, index_reg));
break;
default:
pr_err("bpf_jit: 1 and 2 byte atomics are not supported\n");
pr_err("bpf_jit: 1- and 2-byte RMW atomics are not supported\n");
return -EFAULT;
}
@ -1339,6 +1340,49 @@ static int emit_atomic_index(u8 **pprog, u8 atomic_op, u32 size,
return 0;
}
static int emit_atomic_ld_st(u8 **pprog, u32 atomic_op, u32 dst_reg,
u32 src_reg, s16 off, u8 bpf_size)
{
switch (atomic_op) {
case BPF_LOAD_ACQ:
/* dst_reg = smp_load_acquire(src_reg + off16) */
emit_ldx(pprog, bpf_size, dst_reg, src_reg, off);
break;
case BPF_STORE_REL:
/* smp_store_release(dst_reg + off16, src_reg) */
emit_stx(pprog, bpf_size, dst_reg, src_reg, off);
break;
default:
pr_err("bpf_jit: unknown atomic load/store opcode %02x\n",
atomic_op);
return -EFAULT;
}
return 0;
}
static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size,
u32 dst_reg, u32 src_reg, u32 index_reg,
int off)
{
switch (atomic_op) {
case BPF_LOAD_ACQ:
/* dst_reg = smp_load_acquire(src_reg + idx_reg + off16) */
emit_ldx_index(pprog, size, dst_reg, src_reg, index_reg, off);
break;
case BPF_STORE_REL:
/* smp_store_release(dst_reg + idx_reg + off16, src_reg) */
emit_stx_index(pprog, size, dst_reg, src_reg, index_reg, off);
break;
default:
pr_err("bpf_jit: unknown atomic load/store opcode %02x\n",
atomic_op);
return -EFAULT;
}
return 0;
}
#define DONT_CLEAR 1
bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs)
@ -2121,6 +2165,13 @@ populate_extable:
}
break;
case BPF_STX | BPF_ATOMIC | BPF_B:
case BPF_STX | BPF_ATOMIC | BPF_H:
if (!bpf_atomic_is_load_store(insn)) {
pr_err("bpf_jit: 1- and 2-byte RMW atomics are not supported\n");
return -EFAULT;
}
fallthrough;
case BPF_STX | BPF_ATOMIC | BPF_W:
case BPF_STX | BPF_ATOMIC | BPF_DW:
if (insn->imm == (BPF_AND | BPF_FETCH) ||
@ -2156,10 +2207,10 @@ populate_extable:
EMIT2(simple_alu_opcodes[BPF_OP(insn->imm)],
add_2reg(0xC0, AUX_REG, real_src_reg));
/* Attempt to swap in new value */
err = emit_atomic(&prog, BPF_CMPXCHG,
real_dst_reg, AUX_REG,
insn->off,
BPF_SIZE(insn->code));
err = emit_atomic_rmw(&prog, BPF_CMPXCHG,
real_dst_reg, AUX_REG,
insn->off,
BPF_SIZE(insn->code));
if (WARN_ON(err))
return err;
/*
@ -2174,17 +2225,35 @@ populate_extable:
break;
}
err = emit_atomic(&prog, insn->imm, dst_reg, src_reg,
insn->off, BPF_SIZE(insn->code));
if (bpf_atomic_is_load_store(insn))
err = emit_atomic_ld_st(&prog, insn->imm, dst_reg, src_reg,
insn->off, BPF_SIZE(insn->code));
else
err = emit_atomic_rmw(&prog, insn->imm, dst_reg, src_reg,
insn->off, BPF_SIZE(insn->code));
if (err)
return err;
break;
case BPF_STX | BPF_PROBE_ATOMIC | BPF_B:
case BPF_STX | BPF_PROBE_ATOMIC | BPF_H:
if (!bpf_atomic_is_load_store(insn)) {
pr_err("bpf_jit: 1- and 2-byte RMW atomics are not supported\n");
return -EFAULT;
}
fallthrough;
case BPF_STX | BPF_PROBE_ATOMIC | BPF_W:
case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW:
start_of_ldx = prog;
err = emit_atomic_index(&prog, insn->imm, BPF_SIZE(insn->code),
dst_reg, src_reg, X86_REG_R12, insn->off);
if (bpf_atomic_is_load_store(insn))
err = emit_atomic_ld_st_index(&prog, insn->imm,
BPF_SIZE(insn->code), dst_reg,
src_reg, X86_REG_R12, insn->off);
else
err = emit_atomic_rmw_index(&prog, insn->imm, BPF_SIZE(insn->code),
dst_reg, src_reg, X86_REG_R12,
insn->off);
if (err)
return err;
goto populate_extable;
@ -3801,3 +3870,8 @@ u64 bpf_arch_uaddress_limit(void)
{
return 0;
}
bool bpf_jit_supports_timed_may_goto(void)
{
return true;
}

55
arch/x86/net/bpf_timed_may_goto.S Normal file
View File

@ -0,0 +1,55 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
#include <linux/export.h>
#include <linux/linkage.h>
#include <asm/nospec-branch.h>
.code64
.section .text, "ax"
SYM_FUNC_START(arch_bpf_timed_may_goto)
ANNOTATE_NOENDBR
/*
* r10 passes us stack depth, load the pointer to count and timestamp
* into r10 by adding it to BPF frame pointer.
*/
leaq (%rbp, %r10, 1), %r10
/* Setup frame. */
pushq %rbp
movq %rsp, %rbp
/* Save r0-r5. */
pushq %rax
pushq %rdi
pushq %rsi
pushq %rdx
pushq %rcx
pushq %r8
/*
* r10 has the pointer to count and timestamp, pass it as first
* argument.
*/
movq %r10, %rdi
/* Emit call depth accounting for call below. */
CALL_DEPTH_ACCOUNT
call bpf_check_timed_may_goto
/* BPF_REG_AX=r10 will be stored into count, so move return value to it. */
movq %rax, %r10
/* Restore r5-r0. */
popq %r8
popq %rcx
popq %rdx
popq %rsi
popq %rdi
popq %rax
leave
RET
SYM_FUNC_END(arch_bpf_timed_may_goto)

225
fs/bpf_fs_kfuncs.c
View File

@ -2,10 +2,12 @@
/* Copyright (c) 2024 Google LLC. */
#include <linux/bpf.h>
#include <linux/bpf_lsm.h>
#include <linux/btf.h>
#include <linux/btf_ids.h>
#include <linux/dcache.h>
#include <linux/fs.h>
#include <linux/fsnotify.h>
#include <linux/file.h>
#include <linux/mm.h>
#include <linux/xattr.h>
@ -93,6 +95,24 @@ __bpf_kfunc int bpf_path_d_path(struct path *path, char *buf, size_t buf__sz)
return len;
}
static bool match_security_bpf_prefix(const char *name__str)
{
return !strncmp(name__str, XATTR_NAME_BPF_LSM, XATTR_NAME_BPF_LSM_LEN);
}
static int bpf_xattr_read_permission(const char *name, struct inode *inode)
{
if (WARN_ON(!inode))
return -EINVAL;
/* Allow reading xattr with user. and security.bpf. prefix */
if (strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN) &&
!match_security_bpf_prefix(name))
return -EPERM;
return inode_permission(&nop_mnt_idmap, inode, MAY_READ);
}
/**
* bpf_get_dentry_xattr - get xattr of a dentry
* @dentry: dentry to get xattr from
@ -101,9 +121,10 @@ __bpf_kfunc int bpf_path_d_path(struct path *path, char *buf, size_t buf__sz)
*
* Get xattr *name__str* of *dentry* and store the output in *value_ptr*.
*
* For security reasons, only *name__str* with prefix "user." is allowed.
* For security reasons, only *name__str* with prefixes "user." or
* "security.bpf." are allowed.
*
* Return: 0 on success, a negative value on error.
* Return: length of the xattr value on success, a negative value on error.
*/
__bpf_kfunc int bpf_get_dentry_xattr(struct dentry *dentry, const char *name__str,
struct bpf_dynptr *value_p)
@ -114,18 +135,12 @@ __bpf_kfunc int bpf_get_dentry_xattr(struct dentry *dentry, const char *name__st
void *value;
int ret;
if (WARN_ON(!inode))
return -EINVAL;
if (strncmp(name__str, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN))
return -EPERM;
value_len = __bpf_dynptr_size(value_ptr);
value = __bpf_dynptr_data_rw(value_ptr, value_len);
if (!value)
return -EINVAL;
ret = inode_permission(&nop_mnt_idmap, inode, MAY_READ);
ret = bpf_xattr_read_permission(name__str, inode);
if (ret)
return ret;
return __vfs_getxattr(dentry, inode, name__str, value, value_len);
@ -139,9 +154,10 @@ __bpf_kfunc int bpf_get_dentry_xattr(struct dentry *dentry, const char *name__st
*
* Get xattr *name__str* of *file* and store the output in *value_ptr*.
*
* For security reasons, only *name__str* with prefix "user." is allowed.
* For security reasons, only *name__str* with prefixes "user." or
* "security.bpf." are allowed.
*
* Return: 0 on success, a negative value on error.
* Return: length of the xattr value on success, a negative value on error.
*/
__bpf_kfunc int bpf_get_file_xattr(struct file *file, const char *name__str,
struct bpf_dynptr *value_p)
@ -154,6 +170,160 @@ __bpf_kfunc int bpf_get_file_xattr(struct file *file, const char *name__str,
__bpf_kfunc_end_defs();
static int bpf_xattr_write_permission(const char *name, struct inode *inode)
{
if (WARN_ON(!inode))
return -EINVAL;
/* Only allow setting and removing security.bpf. xattrs */
if (!match_security_bpf_prefix(name))
return -EPERM;
return inode_permission(&nop_mnt_idmap, inode, MAY_WRITE);
}
/**
* bpf_set_dentry_xattr_locked - set a xattr of a dentry
* @dentry: dentry to get xattr from
* @name__str: name of the xattr
* @value_p: xattr value
* @flags: flags to pass into filesystem operations
*
* Set xattr *name__str* of *dentry* to the value in *value_ptr*.
*
* For security reasons, only *name__str* with prefix "security.bpf."
* is allowed.
*
* The caller already locked dentry->d_inode.
*
* Return: 0 on success, a negative value on error.
*/
int bpf_set_dentry_xattr_locked(struct dentry *dentry, const char *name__str,
const struct bpf_dynptr *value_p, int flags)
{
struct bpf_dynptr_kern *value_ptr = (struct bpf_dynptr_kern *)value_p;
struct inode *inode = d_inode(dentry);
const void *value;
u32 value_len;
int ret;
value_len = __bpf_dynptr_size(value_ptr);
value = __bpf_dynptr_data(value_ptr, value_len);
if (!value)
return -EINVAL;
ret = bpf_xattr_write_permission(name__str, inode);
if (ret)
return ret;
ret = __vfs_setxattr(&nop_mnt_idmap, dentry, inode, name__str,
value, value_len, flags);
if (!ret) {
fsnotify_xattr(dentry);
/* This xattr is set by BPF LSM, so we do not call
* security_inode_post_setxattr. Otherwise, we would
* risk deadlocks by calling back to the same kfunc.
*
* This is the same as security_inode_setsecurity().
*/
}
return ret;
}
/**
* bpf_remove_dentry_xattr_locked - remove a xattr of a dentry
* @dentry: dentry to get xattr from
* @name__str: name of the xattr
*
* Rmove xattr *name__str* of *dentry*.
*
* For security reasons, only *name__str* with prefix "security.bpf."
* is allowed.
*
* The caller already locked dentry->d_inode.
*
* Return: 0 on success, a negative value on error.
*/
int bpf_remove_dentry_xattr_locked(struct dentry *dentry, const char *name__str)
{
struct inode *inode = d_inode(dentry);
int ret;
ret = bpf_xattr_write_permission(name__str, inode);
if (ret)
return ret;
ret = __vfs_removexattr(&nop_mnt_idmap, dentry, name__str);
if (!ret) {
fsnotify_xattr(dentry);
/* This xattr is removed by BPF LSM, so we do not call
* security_inode_post_removexattr. Otherwise, we would
* risk deadlocks by calling back to the same kfunc.
*/
}
return ret;
}
__bpf_kfunc_start_defs();
/**
* bpf_set_dentry_xattr - set a xattr of a dentry
* @dentry: dentry to get xattr from
* @name__str: name of the xattr
* @value_p: xattr value
* @flags: flags to pass into filesystem operations
*
* Set xattr *name__str* of *dentry* to the value in *value_ptr*.
*
* For security reasons, only *name__str* with prefix "security.bpf."
* is allowed.
*
* The caller has not locked dentry->d_inode.
*
* Return: 0 on success, a negative value on error.
*/
__bpf_kfunc int bpf_set_dentry_xattr(struct dentry *dentry, const char *name__str,
const struct bpf_dynptr *value_p, int flags)
{
struct inode *inode = d_inode(dentry);
int ret;
inode_lock(inode);
ret = bpf_set_dentry_xattr_locked(dentry, name__str, value_p, flags);
inode_unlock(inode);
return ret;
}
/**
* bpf_remove_dentry_xattr - remove a xattr of a dentry
* @dentry: dentry to get xattr from
* @name__str: name of the xattr
*
* Rmove xattr *name__str* of *dentry*.
*
* For security reasons, only *name__str* with prefix "security.bpf."
* is allowed.
*
* The caller has not locked dentry->d_inode.
*
* Return: 0 on success, a negative value on error.
*/
__bpf_kfunc int bpf_remove_dentry_xattr(struct dentry *dentry, const char *name__str)
{
struct inode *inode = d_inode(dentry);
int ret;
inode_lock(inode);
ret = bpf_remove_dentry_xattr_locked(dentry, name__str);
inode_unlock(inode);
return ret;
}
__bpf_kfunc_end_defs();
BTF_KFUNCS_START(bpf_fs_kfunc_set_ids)
BTF_ID_FLAGS(func, bpf_get_task_exe_file,
KF_ACQUIRE | KF_TRUSTED_ARGS | KF_RET_NULL)
@ -161,6 +331,8 @@ BTF_ID_FLAGS(func, bpf_put_file, KF_RELEASE)
BTF_ID_FLAGS(func, bpf_path_d_path, KF_TRUSTED_ARGS)
BTF_ID_FLAGS(func, bpf_get_dentry_xattr, KF_SLEEPABLE | KF_TRUSTED_ARGS)
BTF_ID_FLAGS(func, bpf_get_file_xattr, KF_SLEEPABLE | KF_TRUSTED_ARGS)
BTF_ID_FLAGS(func, bpf_set_dentry_xattr, KF_SLEEPABLE | KF_TRUSTED_ARGS)
BTF_ID_FLAGS(func, bpf_remove_dentry_xattr, KF_SLEEPABLE | KF_TRUSTED_ARGS)
BTF_KFUNCS_END(bpf_fs_kfunc_set_ids)
static int bpf_fs_kfuncs_filter(const struct bpf_prog *prog, u32 kfunc_id)
@ -171,6 +343,37 @@ static int bpf_fs_kfuncs_filter(const struct bpf_prog *prog, u32 kfunc_id)
return -EACCES;
}
/* bpf_[set|remove]_dentry_xattr.* hooks have KF_TRUSTED_ARGS and
* KF_SLEEPABLE, so they are only available to sleepable hooks with
* dentry arguments.
*
* Setting and removing xattr requires exclusive lock on dentry->d_inode.
* Some hooks already locked d_inode, while some hooks have not locked
* d_inode. Therefore, we need different kfuncs for different hooks.
* Specifically, hooks in the following list (d_inode_locked_hooks)
* should call bpf_[set|remove]_dentry_xattr_locked; while other hooks
* should call bpf_[set|remove]_dentry_xattr.
*/
BTF_SET_START(d_inode_locked_hooks)
BTF_ID(func, bpf_lsm_inode_post_removexattr)
BTF_ID(func, bpf_lsm_inode_post_setattr)
BTF_ID(func, bpf_lsm_inode_post_setxattr)
BTF_ID(func, bpf_lsm_inode_removexattr)
BTF_ID(func, bpf_lsm_inode_rmdir)
BTF_ID(func, bpf_lsm_inode_setattr)
BTF_ID(func, bpf_lsm_inode_setxattr)
BTF_ID(func, bpf_lsm_inode_unlink)
#ifdef CONFIG_SECURITY_PATH
BTF_ID(func, bpf_lsm_path_unlink)
BTF_ID(func, bpf_lsm_path_rmdir)
#endif /* CONFIG_SECURITY_PATH */
BTF_SET_END(d_inode_locked_hooks)
bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog)
{
return btf_id_set_contains(&d_inode_locked_hooks, prog->aux->attach_btf_id);
}
static const struct btf_kfunc_id_set bpf_fs_kfunc_set = {
.owner = THIS_MODULE,
.set = &bpf_fs_kfunc_set_ids,

1
include/linux/bpf-cgroup.h
View File

@ -111,6 +111,7 @@ struct bpf_prog_list {
struct bpf_prog *prog;
struct bpf_cgroup_link *link;
struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];
u32 flags;
};
int cgroup_bpf_inherit(struct cgroup *cgrp);

31
include/linux/bpf.h
View File

@ -968,6 +968,7 @@ struct bpf_insn_access_aux {
struct {
struct btf *btf;
u32 btf_id;
u32 ref_obj_id;
};
};
struct bpf_verifier_log *log; /* for verbose logs */
@ -990,6 +991,21 @@ static inline bool bpf_pseudo_func(const struct bpf_insn *insn)
return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
}
/* Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an
* atomic load or store, and false if it is a read-modify-write instruction.
*/
static inline bool
bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn)
{
switch (atomic_insn->imm) {
case BPF_LOAD_ACQ:
case BPF_STORE_REL:
return true;
default:
return false;
}
}
struct bpf_prog_ops {
int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,
union bpf_attr __user *uattr);
@ -1481,6 +1497,8 @@ struct bpf_ctx_arg_aux {
enum bpf_reg_type reg_type;
struct btf *btf;
u32 btf_id;
u32 ref_obj_id;
bool refcounted;
};
struct btf_mod_pair {
@ -1503,11 +1521,12 @@ struct bpf_prog_aux {
u32 real_func_cnt; /* includes hidden progs, only used for JIT and freeing progs */
u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */
u32 attach_btf_id; /* in-kernel BTF type id to attach to */
u32 attach_st_ops_member_off;
u32 ctx_arg_info_size;
u32 max_rdonly_access;
u32 max_rdwr_access;
struct btf *attach_btf;
const struct bpf_ctx_arg_aux *ctx_arg_info;
struct bpf_ctx_arg_aux *ctx_arg_info;
void __percpu *priv_stack_ptr;
struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */
struct bpf_prog *dst_prog;
@ -1528,6 +1547,7 @@ struct bpf_prog_aux {
bool jits_use_priv_stack;
bool priv_stack_requested;
bool changes_pkt_data;
bool might_sleep;
u64 prog_array_member_cnt; /* counts how many times as member of prog_array */
struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */
struct bpf_arena *arena;
@ -1547,6 +1567,7 @@ struct bpf_prog_aux {
#endif
struct bpf_ksym ksym;
const struct bpf_prog_ops *ops;
const struct bpf_struct_ops *st_ops;
struct bpf_map **used_maps;
struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */
struct btf_mod_pair *used_btfs;
@ -1945,6 +1966,9 @@ static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_op
#endif
int bpf_prog_ctx_arg_info_init(struct bpf_prog *prog,
const struct bpf_ctx_arg_aux *info, u32 cnt);
#if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)
int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
int cgroup_atype);
@ -1980,6 +2004,7 @@ struct bpf_array {
*/
enum {
BPF_MAX_LOOPS = 8 * 1024 * 1024,
BPF_MAX_TIMED_LOOPS = 0xffff,
};
#define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \
@ -2036,6 +2061,8 @@ int bpf_prog_calc_tag(struct bpf_prog *fp);
const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void);
const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void);
typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src,
unsigned long off, unsigned long len);
typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type,
@ -2546,7 +2573,7 @@ struct bpf_iter__bpf_map_elem {
int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info);
void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info);
bool bpf_iter_prog_supported(struct bpf_prog *prog);
int bpf_iter_prog_supported(struct bpf_prog *prog);
const struct bpf_func_proto *
bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_prog *prog);

18
include/linux/bpf_lsm.h
View File

@ -48,6 +48,11 @@ void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, bpf_func_t *bpf_func)
int bpf_lsm_get_retval_range(const struct bpf_prog *prog,
struct bpf_retval_range *range);
int bpf_set_dentry_xattr_locked(struct dentry *dentry, const char *name__str,
const struct bpf_dynptr *value_p, int flags);
int bpf_remove_dentry_xattr_locked(struct dentry *dentry, const char *name__str);
bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog);
#else /* !CONFIG_BPF_LSM */
static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id)
@ -86,6 +91,19 @@ static inline int bpf_lsm_get_retval_range(const struct bpf_prog *prog,
{
return -EOPNOTSUPP;
}
static inline int bpf_set_dentry_xattr_locked(struct dentry *dentry, const char *name__str,
const struct bpf_dynptr *value_p, int flags)
{
return -EOPNOTSUPP;
}
static inline int bpf_remove_dentry_xattr_locked(struct dentry *dentry, const char *name__str)
{
return -EOPNOTSUPP;
}
static inline bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog)
{
return false;
}
#endif /* CONFIG_BPF_LSM */
#endif /* _LINUX_BPF_LSM_H */

32
include/linux/bpf_verifier.h
View File

@ -427,11 +427,6 @@ struct bpf_verifier_state {
bool active_rcu_lock;
bool speculative;
/* If this state was ever pointed-to by other state's loop_entry field
* this flag would be set to true. Used to avoid freeing such states
* while they are still in use.
*/
bool used_as_loop_entry;
bool in_sleepable;
/* first and last insn idx of this verifier state */
@ -458,6 +453,11 @@ struct bpf_verifier_state {
u32 dfs_depth;
u32 callback_unroll_depth;
u32 may_goto_depth;
/* If this state was ever pointed-to by other state's loop_entry field
* this flag would be set to true. Used to avoid freeing such states
* while they are still in use.
*/
u32 used_as_loop_entry;
};
#define bpf_get_spilled_reg(slot, frame, mask) \
@ -498,8 +498,10 @@ struct bpf_verifier_state {
/* linked list of verifier states used to prune search */
struct bpf_verifier_state_list {
struct bpf_verifier_state state;
struct bpf_verifier_state_list *next;
int miss_cnt, hit_cnt;
struct list_head node;
u32 miss_cnt;
u32 hit_cnt:31;
u32 in_free_list:1;
};
struct bpf_loop_inline_state {
@ -589,6 +591,8 @@ struct bpf_insn_aux_data {
* accepts callback function as a parameter.
*/
bool calls_callback;
/* registers alive before this instruction. */
u16 live_regs_before;
};
#define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */
@ -665,6 +669,7 @@ struct bpf_subprog_info {
/* true if bpf_fastcall stack region is used by functions that can't be inlined */
bool keep_fastcall_stack: 1;
bool changes_pkt_data: 1;
bool might_sleep: 1;
enum priv_stack_mode priv_stack_mode;
u8 arg_cnt;
@ -710,8 +715,11 @@ struct bpf_verifier_env {
bool test_state_freq; /* test verifier with different pruning frequency */
bool test_reg_invariants; /* fail verification on register invariants violations */
struct bpf_verifier_state *cur_state; /* current verifier state */
struct bpf_verifier_state_list **explored_states; /* search pruning optimization */
struct bpf_verifier_state_list *free_list;
/* Search pruning optimization, array of list_heads for
* lists of struct bpf_verifier_state_list.
*/
struct list_head *explored_states;
struct list_head free_list; /* list of struct bpf_verifier_state_list */
struct bpf_map *used_maps[MAX_USED_MAPS]; /* array of map's used by eBPF program */
struct btf_mod_pair used_btfs[MAX_USED_BTFS]; /* array of BTF's used by BPF program */
u32 used_map_cnt; /* number of used maps */
@ -742,7 +750,11 @@ struct bpf_verifier_env {
struct {
int *insn_state;
int *insn_stack;
/* vector of instruction indexes sorted in post-order */
int *insn_postorder;
int cur_stack;
/* current position in the insn_postorder vector */
int cur_postorder;
} cfg;
struct backtrack_state bt;
struct bpf_insn_hist_entry *insn_hist;
@ -767,6 +779,8 @@ struct bpf_verifier_env {
u32 peak_states;
/* longest register parentage chain walked for liveness marking */
u32 longest_mark_read_walk;
u32 free_list_size;
u32 explored_states_size;
bpfptr_t fd_array;
/* bit mask to keep track of whether a register has been accessed

3
include/linux/btf.h
View File

@ -76,6 +76,9 @@
#define KF_ITER_DESTROY (1 << 10) /* kfunc implements BPF iter destructor */
#define KF_RCU_PROTECTED (1 << 11) /* kfunc should be protected by rcu cs when they are invoked */
#define KF_FASTCALL (1 << 12) /* kfunc supports bpf_fastcall protocol */
#define KF_ARENA_RET (1 << 13) /* kfunc returns an arena pointer */
#define KF_ARENA_ARG1 (1 << 14) /* kfunc takes an arena pointer as its first argument */
#define KF_ARENA_ARG2 (1 << 15) /* kfunc takes an arena pointer as its second argument */
/*
* Tag marking a kernel function as a kfunc. This is meant to minimize the

20
include/linux/filter.h
View File

@ -364,6 +364,8 @@ static inline bool insn_is_cast_user(const struct bpf_insn *insn)
* BPF_XOR | BPF_FETCH src_reg = atomic_fetch_xor(dst_reg + off16, src_reg);
* BPF_XCHG src_reg = atomic_xchg(dst_reg + off16, src_reg)
* BPF_CMPXCHG r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg)
* BPF_LOAD_ACQ dst_reg = smp_load_acquire(src_reg + off16)
* BPF_STORE_REL smp_store_release(dst_reg + off16, src_reg)
*/
#define BPF_ATOMIC_OP(SIZE, OP, DST, SRC, OFF) \
@ -469,6 +471,16 @@ static inline bool insn_is_cast_user(const struct bpf_insn *insn)
.off = 0, \
.imm = BPF_CALL_IMM(FUNC) })
/* Kfunc call */
#define BPF_CALL_KFUNC(OFF, IMM) \
((struct bpf_insn) { \
.code = BPF_JMP | BPF_CALL, \
.dst_reg = 0, \
.src_reg = BPF_PSEUDO_KFUNC_CALL, \
.off = OFF, \
.imm = IMM })
/* Raw code statement block */
#define BPF_RAW_INSN(CODE, DST, SRC, OFF, IMM) \
@ -659,6 +671,11 @@ struct bpf_prog_stats {
struct u64_stats_sync syncp;
} __aligned(2 * sizeof(u64));
struct bpf_timed_may_goto {
u64 count;
u64 timestamp;
};
struct sk_filter {
refcount_t refcnt;
struct rcu_head rcu;
@ -1120,8 +1137,11 @@ bool bpf_jit_supports_ptr_xchg(void);
bool bpf_jit_supports_arena(void);
bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena);
bool bpf_jit_supports_private_stack(void);
bool bpf_jit_supports_timed_may_goto(void);
u64 bpf_arch_uaddress_limit(void);
void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);
u64 arch_bpf_timed_may_goto(void);
u64 bpf_check_timed_may_goto(struct bpf_timed_may_goto *);
bool bpf_helper_changes_pkt_data(enum bpf_func_id func_id);
static inline bool bpf_dump_raw_ok(const struct cred *cred)

6
include/linux/lsm_hook_defs.h
View File

@ -426,14 +426,14 @@ LSM_HOOK(void, LSM_RET_VOID, audit_rule_free, void *lsmrule)
#endif /* CONFIG_AUDIT */
#ifdef CONFIG_BPF_SYSCALL
LSM_HOOK(int, 0, bpf, int cmd, union bpf_attr *attr, unsigned int size)
LSM_HOOK(int, 0, bpf, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
LSM_HOOK(int, 0, bpf_map, struct bpf_map *map, fmode_t fmode)
LSM_HOOK(int, 0, bpf_prog, struct bpf_prog *prog)
LSM_HOOK(int, 0, bpf_map_create, struct bpf_map *map, union bpf_attr *attr,
struct bpf_token *token)
struct bpf_token *token, bool kernel)
LSM_HOOK(void, LSM_RET_VOID, bpf_map_free, struct bpf_map *map)
LSM_HOOK(int, 0, bpf_prog_load, struct bpf_prog *prog, union bpf_attr *attr,
struct bpf_token *token)
struct bpf_token *token, bool kernel)
LSM_HOOK(void, LSM_RET_VOID, bpf_prog_free, struct bpf_prog *prog)
LSM_HOOK(int, 0, bpf_token_create, struct bpf_token *token, union bpf_attr *attr,
const struct path *path)

5
include/linux/mm.h
View File

@ -2477,6 +2477,11 @@ extern int access_process_vm(struct task_struct *tsk, unsigned long addr,
extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
void *buf, int len, unsigned int gup_flags);
#ifdef CONFIG_BPF_SYSCALL
extern int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr,
void *buf, int len, unsigned int gup_flags);
#endif
long get_user_pages_remote(struct mm_struct *mm,
unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages,

12
include/linux/security.h
View File

@ -2249,14 +2249,14 @@ struct bpf_map;
struct bpf_prog;
struct bpf_token;
#ifdef CONFIG_SECURITY
extern int security_bpf(int cmd, union bpf_attr *attr, unsigned int size);
extern int security_bpf(int cmd, union bpf_attr *attr, unsigned int size, bool kernel);
extern int security_bpf_map(struct bpf_map *map, fmode_t fmode);
extern int security_bpf_prog(struct bpf_prog *prog);
extern int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr,
struct bpf_token *token);
struct bpf_token *token, bool kernel);
extern void security_bpf_map_free(struct bpf_map *map);
extern int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr,
struct bpf_token *token);
struct bpf_token *token, bool kernel);
extern void security_bpf_prog_free(struct bpf_prog *prog);
extern int security_bpf_token_create(struct bpf_token *token, union bpf_attr *attr,
const struct path *path);
@ -2265,7 +2265,7 @@ extern int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cm
extern int security_bpf_token_capable(const struct bpf_token *token, int cap);
#else
static inline int security_bpf(int cmd, union bpf_attr *attr,
unsigned int size)
unsigned int size, bool kernel)
{
return 0;
}
@ -2281,7 +2281,7 @@ static inline int security_bpf_prog(struct bpf_prog *prog)
}
static inline int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr,
struct bpf_token *token)
struct bpf_token *token, bool kernel)
{
return 0;
}
@ -2290,7 +2290,7 @@ static inline void security_bpf_map_free(struct bpf_map *map)
{ }
static inline int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr,
struct bpf_token *token)
struct bpf_token *token, bool kernel)
{
return 0;
}

10
include/uapi/linux/bpf.h
View File

@ -51,6 +51,9 @@
#define BPF_XCHG (0xe0 | BPF_FETCH) /* atomic exchange */
#define BPF_CMPXCHG (0xf0 | BPF_FETCH) /* atomic compare-and-write */
#define BPF_LOAD_ACQ 0x100 /* load-acquire */
#define BPF_STORE_REL 0x110 /* store-release */
enum bpf_cond_pseudo_jmp {
BPF_MAY_GOTO = 0,
};
@ -1207,6 +1210,7 @@ enum bpf_perf_event_type {
#define BPF_F_BEFORE (1U << 3)
#define BPF_F_AFTER (1U << 4)
#define BPF_F_ID (1U << 5)
#define BPF_F_PREORDER (1U << 6)
#define BPF_F_LINK BPF_F_LINK /* 1 << 13 */
/* If BPF_F_STRICT_ALIGNMENT is used in BPF_PROG_LOAD command, the
@ -1648,6 +1652,7 @@ union bpf_attr {
};
__u32 next_id;
__u32 open_flags;
__s32 fd_by_id_token_fd;
};
struct { /* anonymous struct used by BPF_OBJ_GET_INFO_BY_FD */
@ -6019,7 +6024,10 @@ union bpf_attr {
FN(user_ringbuf_drain, 209, ##ctx) \
FN(cgrp_storage_get, 210, ##ctx) \
FN(cgrp_storage_delete, 211, ##ctx) \
/* */
/* This helper list is effectively frozen. If you are trying to \
* add a new helper, you should add a kfunc instead which has \
* less stability guarantees. See Documentation/bpf/kfuncs.rst \
*/
/* backwards-compatibility macros for users of __BPF_FUNC_MAPPER that don't
* know or care about integer value that is now passed as second argument

3
include/uapi/linux/btf.h
View File

@ -36,7 +36,8 @@ struct btf_type {
* bits 24-28: kind (e.g. int, ptr, array...etc)
* bits 29-30: unused
* bit 31: kind_flag, currently used by
* struct, union, enum, fwd and enum64
* struct, union, enum, fwd, enum64,
* decl_tag and type_tag
*/
__u32 info;
/* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC and ENUM64.

4
include/uapi/linux/xattr.h
View File

@ -83,6 +83,10 @@ struct xattr_args {
#define XATTR_CAPS_SUFFIX "capability"
#define XATTR_NAME_CAPS XATTR_SECURITY_PREFIX XATTR_CAPS_SUFFIX
#define XATTR_BPF_LSM_SUFFIX "bpf."
#define XATTR_NAME_BPF_LSM (XATTR_SECURITY_PREFIX XATTR_BPF_LSM_SUFFIX)
#define XATTR_NAME_BPF_LSM_LEN (sizeof(XATTR_NAME_BPF_LSM) - 1)
#define XATTR_POSIX_ACL_ACCESS "posix_acl_access"
#define XATTR_NAME_POSIX_ACL_ACCESS XATTR_SYSTEM_PREFIX XATTR_POSIX_ACL_ACCESS
#define XATTR_POSIX_ACL_DEFAULT "posix_acl_default"

4
kernel/bpf/arena.c
View File

@ -577,8 +577,8 @@ __bpf_kfunc void bpf_arena_free_pages(void *p__map, void *ptr__ign, u32 page_cnt
__bpf_kfunc_end_defs();
BTF_KFUNCS_START(arena_kfuncs)
BTF_ID_FLAGS(func, bpf_arena_alloc_pages, KF_TRUSTED_ARGS | KF_SLEEPABLE)
BTF_ID_FLAGS(func, bpf_arena_free_pages, KF_TRUSTED_ARGS | KF_SLEEPABLE)
BTF_ID_FLAGS(func, bpf_arena_alloc_pages, KF_TRUSTED_ARGS | KF_SLEEPABLE | KF_ARENA_RET | KF_ARENA_ARG2)
BTF_ID_FLAGS(func, bpf_arena_free_pages, KF_TRUSTED_ARGS | KF_SLEEPABLE | KF_ARENA_ARG2)
BTF_KFUNCS_END(arena_kfuncs)
static const struct btf_kfunc_id_set common_kfunc_set = {

11
kernel/bpf/bpf_cgrp_storage.c
View File

@ -161,6 +161,7 @@ BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup,
void *, value, u64, flags, gfp_t, gfp_flags)
{
struct bpf_local_storage_data *sdata;
bool nobusy;
WARN_ON_ONCE(!bpf_rcu_lock_held());
if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
@ -169,21 +170,21 @@ BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup,
if (!cgroup)
return (unsigned long)NULL;
if (!bpf_cgrp_storage_trylock())
return (unsigned long)NULL;
nobusy = bpf_cgrp_storage_trylock();
sdata = cgroup_storage_lookup(cgroup, map, true);
sdata = cgroup_storage_lookup(cgroup, map, nobusy);
if (sdata)
goto unlock;
/* only allocate new storage, when the cgroup is refcounted */
if (!percpu_ref_is_dying(&cgroup->self.refcnt) &&
(flags & BPF_LOCAL_STORAGE_GET_F_CREATE))
(flags & BPF_LOCAL_STORAGE_GET_F_CREATE) && nobusy)
sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
value, BPF_NOEXIST, false, gfp_flags);
unlock:
bpf_cgrp_storage_unlock();
if (nobusy)
bpf_cgrp_storage_unlock();
return IS_ERR_OR_NULL(sdata) ? (unsigned long)NULL : (unsigned long)sdata->data;
}

13
kernel/bpf/bpf_iter.c
View File

@ -335,7 +335,7 @@ static void cache_btf_id(struct bpf_iter_target_info *tinfo,
tinfo->btf_id = prog->aux->attach_btf_id;
}
bool bpf_iter_prog_supported(struct bpf_prog *prog)
int bpf_iter_prog_supported(struct bpf_prog *prog)
{
const char *attach_fname = prog->aux->attach_func_name;
struct bpf_iter_target_info *tinfo = NULL, *iter;
@ -344,7 +344,7 @@ bool bpf_iter_prog_supported(struct bpf_prog *prog)
int prefix_len = strlen(prefix);
if (strncmp(attach_fname, prefix, prefix_len))
return false;
return -EINVAL;
mutex_lock(&targets_mutex);
list_for_each_entry(iter, &targets, list) {
@ -360,12 +360,11 @@ bool bpf_iter_prog_supported(struct bpf_prog *prog)
}
mutex_unlock(&targets_mutex);
if (tinfo) {
prog->aux->ctx_arg_info_size = tinfo->reg_info->ctx_arg_info_size;
prog->aux->ctx_arg_info = tinfo->reg_info->ctx_arg_info;
}
if (!tinfo)
return -EINVAL;
return tinfo != NULL;
return bpf_prog_ctx_arg_info_init(prog, tinfo->reg_info->ctx_arg_info,
tinfo->reg_info->ctx_arg_info_size);
}
const struct bpf_func_proto *

2
kernel/bpf/bpf_lsm.c
View File

@ -316,7 +316,9 @@ BTF_ID(func, bpf_lsm_inode_getxattr)
BTF_ID(func, bpf_lsm_inode_mknod)
BTF_ID(func, bpf_lsm_inode_need_killpriv)
BTF_ID(func, bpf_lsm_inode_post_setxattr)
BTF_ID(func, bpf_lsm_inode_post_removexattr)
BTF_ID(func, bpf_lsm_inode_readlink)
BTF_ID(func, bpf_lsm_inode_removexattr)
BTF_ID(func, bpf_lsm_inode_rename)
BTF_ID(func, bpf_lsm_inode_rmdir)
BTF_ID(func, bpf_lsm_inode_setattr)

135
kernel/bpf/bpf_struct_ops.c
View File

@ -146,39 +146,7 @@ void bpf_struct_ops_image_free(void *image)
}
#define MAYBE_NULL_SUFFIX "__nullable"
#define MAX_STUB_NAME 128
/* Return the type info of a stub function, if it exists.
*
* The name of a stub function is made up of the name of the struct_ops and
* the name of the function pointer member, separated by "__". For example,
* if the struct_ops type is named "foo_ops" and the function pointer
* member is named "bar", the stub function name would be "foo_ops__bar".
*/
static const struct btf_type *
find_stub_func_proto(const struct btf *btf, const char *st_op_name,
const char *member_name)
{
char stub_func_name[MAX_STUB_NAME];
const struct btf_type *func_type;
s32 btf_id;
int cp;
cp = snprintf(stub_func_name, MAX_STUB_NAME, "%s__%s",
st_op_name, member_name);
if (cp >= MAX_STUB_NAME) {
pr_warn("Stub function name too long\n");
return NULL;
}
btf_id = btf_find_by_name_kind(btf, stub_func_name, BTF_KIND_FUNC);
if (btf_id < 0)
return NULL;
func_type = btf_type_by_id(btf, btf_id);
if (!func_type)
return NULL;
return btf_type_by_id(btf, func_type->type); /* FUNC_PROTO */
}
#define REFCOUNTED_SUFFIX "__ref"
/* Prepare argument info for every nullable argument of a member of a
* struct_ops type.
@ -203,27 +171,44 @@ find_stub_func_proto(const struct btf *btf, const char *st_op_name,
static int prepare_arg_info(struct btf *btf,
const char *st_ops_name,
const char *member_name,
const struct btf_type *func_proto,
const struct btf_type *func_proto, void *stub_func_addr,
struct bpf_struct_ops_arg_info *arg_info)
{
const struct btf_type *stub_func_proto, *pointed_type;
bool is_nullable = false, is_refcounted = false;
const struct btf_param *stub_args, *args;
struct bpf_ctx_arg_aux *info, *info_buf;
u32 nargs, arg_no, info_cnt = 0;
char ksym[KSYM_SYMBOL_LEN];
const char *stub_fname;
const char *suffix;
s32 stub_func_id;
u32 arg_btf_id;
int offset;
stub_func_proto = find_stub_func_proto(btf, st_ops_name, member_name);
if (!stub_func_proto)
return 0;
stub_fname = kallsyms_lookup((unsigned long)stub_func_addr, NULL, NULL, NULL, ksym);
if (!stub_fname) {
pr_warn("Cannot find the stub function name for the %s in struct %s\n",
member_name, st_ops_name);
return -ENOENT;
}
stub_func_id = btf_find_by_name_kind(btf, stub_fname, BTF_KIND_FUNC);
if (stub_func_id < 0) {
pr_warn("Cannot find the stub function %s in btf\n", stub_fname);
return -ENOENT;
}
stub_func_proto = btf_type_by_id(btf, stub_func_id);
stub_func_proto = btf_type_by_id(btf, stub_func_proto->type);
/* Check if the number of arguments of the stub function is the same
* as the number of arguments of the function pointer.
*/
nargs = btf_type_vlen(func_proto);
if (nargs != btf_type_vlen(stub_func_proto)) {
pr_warn("the number of arguments of the stub function %s__%s does not match the number of arguments of the member %s of struct %s\n",
st_ops_name, member_name, member_name, st_ops_name);
pr_warn("the number of arguments of the stub function %s does not match the number of arguments of the member %s of struct %s\n",
stub_fname, member_name, st_ops_name);
return -EINVAL;
}
@ -241,10 +226,18 @@ static int prepare_arg_info(struct btf *btf,
info = info_buf;
for (arg_no = 0; arg_no < nargs; arg_no++) {
/* Skip arguments that is not suffixed with
* "__nullable".
* "__nullable or __ref".
*/
if (!btf_param_match_suffix(btf, &stub_args[arg_no],
MAYBE_NULL_SUFFIX))
is_nullable = btf_param_match_suffix(btf, &stub_args[arg_no],
MAYBE_NULL_SUFFIX);
is_refcounted = btf_param_match_suffix(btf, &stub_args[arg_no],
REFCOUNTED_SUFFIX);
if (is_nullable)
suffix = MAYBE_NULL_SUFFIX;
else if (is_refcounted)
suffix = REFCOUNTED_SUFFIX;
else
continue;
/* Should be a pointer to struct */
@ -253,30 +246,34 @@ static int prepare_arg_info(struct btf *btf,
&arg_btf_id);
if (!pointed_type ||
!btf_type_is_struct(pointed_type)) {
pr_warn("stub function %s__%s has %s tagging to an unsupported type\n",
st_ops_name, member_name, MAYBE_NULL_SUFFIX);
pr_warn("stub function %s has %s tagging to an unsupported type\n",
stub_fname, suffix);
goto err_out;
}
offset = btf_ctx_arg_offset(btf, func_proto, arg_no);
if (offset < 0) {
pr_warn("stub function %s__%s has an invalid trampoline ctx offset for arg#%u\n",
st_ops_name, member_name, arg_no);
pr_warn("stub function %s has an invalid trampoline ctx offset for arg#%u\n",
stub_fname, arg_no);
goto err_out;
}
if (args[arg_no].type != stub_args[arg_no].type) {
pr_warn("arg#%u type in stub function %s__%s does not match with its original func_proto\n",
arg_no, st_ops_name, member_name);
pr_warn("arg#%u type in stub function %s does not match with its original func_proto\n",
arg_no, stub_fname);
goto err_out;
}
/* Fill the information of the new argument */
info->reg_type =
PTR_TRUSTED | PTR_TO_BTF_ID | PTR_MAYBE_NULL;
info->btf_id = arg_btf_id;
info->btf = btf;
info->offset = offset;
if (is_nullable) {
info->reg_type = PTR_TRUSTED | PTR_TO_BTF_ID | PTR_MAYBE_NULL;
} else if (is_refcounted) {
info->reg_type = PTR_TRUSTED | PTR_TO_BTF_ID;
info->refcounted = true;
}
info++;
info_cnt++;
@ -324,6 +321,13 @@ static bool is_module_member(const struct btf *btf, u32 id)
return !strcmp(btf_name_by_offset(btf, t->name_off), "module");
}
int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff)
{
void *func_ptr = *(void **)(st_ops->cfi_stubs + moff);
return func_ptr ? 0 : -ENOTSUPP;
}
int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,
struct btf *btf,
struct bpf_verifier_log *log)
@ -386,8 +390,11 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,
st_ops_desc->value_type = btf_type_by_id(btf, value_id);
for_each_member(i, t, member) {
const struct btf_type *func_proto;
const struct btf_type *func_proto, *ret_type;
void **stub_func_addr;
u32 moff;
moff = __btf_member_bit_offset(t, member) / 8;
mname = btf_name_by_offset(btf, member->name_off);
if (!*mname) {
pr_warn("anon member in struct %s is not supported\n",
@ -413,9 +420,23 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,
func_proto = btf_type_resolve_func_ptr(btf,
member->type,
NULL);
if (!func_proto)
/* The member is not a function pointer or
* the function pointer is not supported.
*/
if (!func_proto || bpf_struct_ops_supported(st_ops, moff))
continue;
if (func_proto->type) {
ret_type = btf_type_resolve_ptr(btf, func_proto->type, NULL);
if (ret_type && !__btf_type_is_struct(ret_type)) {
pr_warn("func ptr %s in struct %s returns non-struct pointer, which is not supported\n",
mname, st_ops->name);
err = -EOPNOTSUPP;
goto errout;
}
}
if (btf_distill_func_proto(log, btf,
func_proto, mname,
&st_ops->func_models[i])) {
@ -425,8 +446,9 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,
goto errout;
}
stub_func_addr = *(void **)(st_ops->cfi_stubs + moff);
err = prepare_arg_info(btf, st_ops->name, mname,
func_proto,
func_proto, stub_func_addr,
arg_info + i);
if (err)
goto errout;
@ -1152,13 +1174,6 @@ void bpf_struct_ops_put(const void *kdata)
bpf_map_put(&st_map->map);
}
int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff)
{
void *func_ptr = *(void **)(st_ops->cfi_stubs + moff);
return func_ptr ? 0 : -ENOTSUPP;
}
static bool bpf_struct_ops_valid_to_reg(struct bpf_map *map)
{
struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;

127
kernel/bpf/btf.c
View File

@ -606,6 +606,7 @@ s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p)
spin_unlock_bh(&btf_idr_lock);
return ret;
}
EXPORT_SYMBOL_GPL(bpf_find_btf_id);
const struct btf_type *btf_type_skip_modifiers(const struct btf *btf,
u32 id, u32 *res_id)
@ -2575,7 +2576,7 @@ static int btf_ref_type_check_meta(struct btf_verifier_env *env,
return -EINVAL;
}
if (btf_type_kflag(t)) {
if (btf_type_kflag(t) && !btf_type_is_type_tag(t)) {
btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
return -EINVAL;
}
@ -3332,6 +3333,8 @@ static int btf_find_kptr(const struct btf *btf, const struct btf_type *t,
u32 off, int sz, struct btf_field_info *info, u32 field_mask)
{
enum btf_field_type type;
const char *tag_value;
bool is_type_tag;
u32 res_id;
/* Permit modifiers on the pointer itself */
@ -3341,19 +3344,20 @@ static int btf_find_kptr(const struct btf *btf, const struct btf_type *t,
if (!btf_type_is_ptr(t))
return BTF_FIELD_IGNORE;
t = btf_type_by_id(btf, t->type);
if (!btf_type_is_type_tag(t))
is_type_tag = btf_type_is_type_tag(t) && !btf_type_kflag(t);
if (!is_type_tag)
return BTF_FIELD_IGNORE;
/* Reject extra tags */
if (btf_type_is_type_tag(btf_type_by_id(btf, t->type)))
return -EINVAL;
if (!strcmp("kptr_untrusted", __btf_name_by_offset(btf, t->name_off)))
tag_value = __btf_name_by_offset(btf, t->name_off);
if (!strcmp("kptr_untrusted", tag_value))
type = BPF_KPTR_UNREF;
else if (!strcmp("kptr", __btf_name_by_offset(btf, t->name_off)))
else if (!strcmp("kptr", tag_value))
type = BPF_KPTR_REF;
else if (!strcmp("percpu_kptr", __btf_name_by_offset(btf, t->name_off)))
else if (!strcmp("percpu_kptr", tag_value))
type = BPF_KPTR_PERCPU;
else if (!strcmp("uptr", __btf_name_by_offset(btf, t->name_off)))
else if (!strcmp("uptr", tag_value))
type = BPF_UPTR;
else
return -EINVAL;
@ -4944,11 +4948,6 @@ static s32 btf_decl_tag_check_meta(struct btf_verifier_env *env,
return -EINVAL;
}
if (btf_type_kflag(t)) {
btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
return -EINVAL;
}
component_idx = btf_type_decl_tag(t)->component_idx;
if (component_idx < -1) {
btf_verifier_log_type(env, t, "Invalid component_idx");
@ -6507,6 +6506,8 @@ static const struct bpf_raw_tp_null_args raw_tp_null_args[] = {
/* rxrpc */
{ "rxrpc_recvdata", 0x1 },
{ "rxrpc_resend", 0x10 },
{ "rxrpc_tq", 0x10 },
{ "rxrpc_client", 0x1 },
/* skb */
{"kfree_skb", 0x1000},
/* sunrpc */
@ -6529,6 +6530,103 @@ static const struct bpf_raw_tp_null_args raw_tp_null_args[] = {
{ "mr_integ_alloc", 0x2000 },
/* bpf_testmod */
{ "bpf_testmod_test_read", 0x0 },
/* amdgpu */
{ "amdgpu_vm_bo_map", 0x1 },
{ "amdgpu_vm_bo_unmap", 0x1 },
/* netfs */
{ "netfs_folioq", 0x1 },
/* xfs from xfs_defer_pending_class */
{ "xfs_defer_create_intent", 0x1 },
{ "xfs_defer_cancel_list", 0x1 },
{ "xfs_defer_pending_finish", 0x1 },
{ "xfs_defer_pending_abort", 0x1 },
{ "xfs_defer_relog_intent", 0x1 },
{ "xfs_defer_isolate_paused", 0x1 },
{ "xfs_defer_item_pause", 0x1 },
{ "xfs_defer_item_unpause", 0x1 },
/* xfs from xfs_defer_pending_item_class */
{ "xfs_defer_add_item", 0x1 },
{ "xfs_defer_cancel_item", 0x1 },
{ "xfs_defer_finish_item", 0x1 },
/* xfs from xfs_icwalk_class */
{ "xfs_ioc_free_eofblocks", 0x10 },
{ "xfs_blockgc_free_space", 0x10 },
/* xfs from xfs_btree_cur_class */
{ "xfs_btree_updkeys", 0x100 },
{ "xfs_btree_overlapped_query_range", 0x100 },
/* xfs from xfs_imap_class*/
{ "xfs_map_blocks_found", 0x10000 },
{ "xfs_map_blocks_alloc", 0x10000 },
{ "xfs_iomap_alloc", 0x1000 },
{ "xfs_iomap_found", 0x1000 },
/* xfs from xfs_fs_class */
{ "xfs_inodegc_flush", 0x1 },
{ "xfs_inodegc_push", 0x1 },
{ "xfs_inodegc_start", 0x1 },
{ "xfs_inodegc_stop", 0x1 },
{ "xfs_inodegc_queue", 0x1 },
{ "xfs_inodegc_throttle", 0x1 },
{ "xfs_fs_sync_fs", 0x1 },
{ "xfs_blockgc_start", 0x1 },
{ "xfs_blockgc_stop", 0x1 },
{ "xfs_blockgc_worker", 0x1 },
{ "xfs_blockgc_flush_all", 0x1 },
/* xfs_scrub */
{ "xchk_nlinks_live_update", 0x10 },
/* xfs_scrub from xchk_metapath_class */
{ "xchk_metapath_lookup", 0x100 },
/* nfsd */
{ "nfsd_dirent", 0x1 },
{ "nfsd_file_acquire", 0x1001 },
{ "nfsd_file_insert_err", 0x1 },
{ "nfsd_file_cons_err", 0x1 },
/* nfs4 */
{ "nfs4_setup_sequence", 0x1 },
{ "pnfs_update_layout", 0x10000 },
{ "nfs4_inode_callback_event", 0x200 },
{ "nfs4_inode_stateid_callback_event", 0x200 },
/* nfs from pnfs_layout_event */
{ "pnfs_mds_fallback_pg_init_read", 0x10000 },
{ "pnfs_mds_fallback_pg_init_write", 0x10000 },
{ "pnfs_mds_fallback_pg_get_mirror_count", 0x10000 },
{ "pnfs_mds_fallback_read_done", 0x10000 },
{ "pnfs_mds_fallback_write_done", 0x10000 },
{ "pnfs_mds_fallback_read_pagelist", 0x10000 },
{ "pnfs_mds_fallback_write_pagelist", 0x10000 },
/* coda */
{ "coda_dec_pic_run", 0x10 },
{ "coda_dec_pic_done", 0x10 },
/* cfg80211 */
{ "cfg80211_scan_done", 0x11 },
{ "rdev_set_coalesce", 0x10 },
{ "cfg80211_report_wowlan_wakeup", 0x100 },
{ "cfg80211_inform_bss_frame", 0x100 },
{ "cfg80211_michael_mic_failure", 0x10000 },
/* cfg80211 from wiphy_work_event */
{ "wiphy_work_queue", 0x10 },
{ "wiphy_work_run", 0x10 },
{ "wiphy_work_cancel", 0x10 },
{ "wiphy_work_flush", 0x10 },
/* hugetlbfs */
{ "hugetlbfs_alloc_inode", 0x10 },
/* spufs */
{ "spufs_context", 0x10 },
/* kvm_hv */
{ "kvm_page_fault_enter", 0x100 },
/* dpu */
{ "dpu_crtc_setup_mixer", 0x100 },
/* binder */
{ "binder_transaction", 0x100 },
/* bcachefs */
{ "btree_path_free", 0x100 },
/* hfi1_tx */
{ "hfi1_sdma_progress", 0x1000 },
/* iptfs */
{ "iptfs_ingress_postq_event", 0x1000 },
/* neigh */
{ "neigh_update", 0x10 },
/* snd_firewire_lib */
{ "amdtp_packet", 0x100 },
};
bool btf_ctx_access(int off, int size, enum bpf_access_type type,
@ -6679,6 +6777,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
info->reg_type = ctx_arg_info->reg_type;
info->btf = ctx_arg_info->btf ? : btf_vmlinux;
info->btf_id = ctx_arg_info->btf_id;
info->ref_obj_id = ctx_arg_info->ref_obj_id;
return true;
}
}
@ -6745,7 +6844,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
info->btf_id = t->type;
t = btf_type_by_id(btf, t->type);
if (btf_type_is_type_tag(t)) {
if (btf_type_is_type_tag(t) && !btf_type_kflag(t)) {
tag_value = __btf_name_by_offset(btf, t->name_off);
if (strcmp(tag_value, "user") == 0)
info->reg_type |= MEM_USER;
@ -7004,7 +7103,7 @@ error:
/* check type tag */
t = btf_type_by_id(btf, mtype->type);
if (btf_type_is_type_tag(t)) {
if (btf_type_is_type_tag(t) && !btf_type_kflag(t)) {
tag_value = __btf_name_by_offset(btf, t->name_off);
/* check __user tag */
if (strcmp(tag_value, "user") == 0)

33
kernel/bpf/cgroup.c
View File

@ -369,7 +369,7 @@ static struct bpf_prog *prog_list_prog(struct bpf_prog_list *pl)
/* count number of elements in the list.
* it's slow but the list cannot be long
*/
static u32 prog_list_length(struct hlist_head *head)
static u32 prog_list_length(struct hlist_head *head, int *preorder_cnt)
{
struct bpf_prog_list *pl;
u32 cnt = 0;
@ -377,6 +377,8 @@ static u32 prog_list_length(struct hlist_head *head)
hlist_for_each_entry(pl, head, node) {
if (!prog_list_prog(pl))
continue;
if (preorder_cnt && (pl->flags & BPF_F_PREORDER))
(*preorder_cnt)++;
cnt++;
}
return cnt;
@ -400,7 +402,7 @@ static bool hierarchy_allows_attach(struct cgroup *cgrp,
if (flags & BPF_F_ALLOW_MULTI)
return true;
cnt = prog_list_length(&p->bpf.progs[atype]);
cnt = prog_list_length(&p->bpf.progs[atype], NULL);
WARN_ON_ONCE(cnt > 1);
if (cnt == 1)
return !!(flags & BPF_F_ALLOW_OVERRIDE);
@ -423,12 +425,12 @@ static int compute_effective_progs(struct cgroup *cgrp,
struct bpf_prog_array *progs;
struct bpf_prog_list *pl;
struct cgroup *p = cgrp;
int cnt = 0;
int i, j, cnt = 0, preorder_cnt = 0, fstart, bstart, init_bstart;
/* count number of effective programs by walking parents */
do {
if (cnt == 0 || (p->bpf.flags[atype] & BPF_F_ALLOW_MULTI))
cnt += prog_list_length(&p->bpf.progs[atype]);
cnt += prog_list_length(&p->bpf.progs[atype], &preorder_cnt);
p = cgroup_parent(p);
} while (p);
@ -439,20 +441,34 @@ static int compute_effective_progs(struct cgroup *cgrp,
/* populate the array with effective progs */
cnt = 0;
p = cgrp;
fstart = preorder_cnt;
bstart = preorder_cnt - 1;
do {
if (cnt > 0 && !(p->bpf.flags[atype] & BPF_F_ALLOW_MULTI))
continue;
init_bstart = bstart;
hlist_for_each_entry(pl, &p->bpf.progs[atype], node) {
if (!prog_list_prog(pl))
continue;
item = &progs->items[cnt];
if (pl->flags & BPF_F_PREORDER) {
item = &progs->items[bstart];
bstart--;
} else {
item = &progs->items[fstart];
fstart++;
}
item->prog = prog_list_prog(pl);
bpf_cgroup_storages_assign(item->cgroup_storage,
pl->storage);
cnt++;
}
/* reverse pre-ordering progs at this cgroup level */
for (i = bstart + 1, j = init_bstart; i < j; i++, j--)
swap(progs->items[i], progs->items[j]);
} while ((p = cgroup_parent(p)));
*array = progs;
@ -663,7 +679,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp,
*/
return -EPERM;
if (prog_list_length(progs) >= BPF_CGROUP_MAX_PROGS)
if (prog_list_length(progs, NULL) >= BPF_CGROUP_MAX_PROGS)
return -E2BIG;
pl = find_attach_entry(progs, prog, link, replace_prog,
@ -698,6 +714,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp,
pl->prog = prog;
pl->link = link;
pl->flags = flags;
bpf_cgroup_storages_assign(pl->storage, storage);
cgrp->bpf.flags[atype] = saved_flags;
@ -1073,7 +1090,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
lockdep_is_held(&cgroup_mutex));
total_cnt += bpf_prog_array_length(effective);
} else {
total_cnt += prog_list_length(&cgrp->bpf.progs[atype]);
total_cnt += prog_list_length(&cgrp->bpf.progs[atype], NULL);
}
}
@ -1105,7 +1122,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
u32 id;
progs = &cgrp->bpf.progs[atype];
cnt = min_t(int, prog_list_length(progs), total_cnt);
cnt = min_t(int, prog_list_length(progs, NULL), total_cnt);
i = 0;
hlist_for_each_entry(pl, progs, node) {
prog = prog_list_prog(pl);

117
kernel/bpf/core.c
View File

@ -1663,14 +1663,17 @@ EXPORT_SYMBOL_GPL(__bpf_call_base);
INSN_3(JMP, JSET, K), \
INSN_2(JMP, JA), \
INSN_2(JMP32, JA), \
/* Atomic operations. */ \
INSN_3(STX, ATOMIC, B), \
INSN_3(STX, ATOMIC, H), \
INSN_3(STX, ATOMIC, W), \
INSN_3(STX, ATOMIC, DW), \
/* Store instructions. */ \
/* Register based. */ \
INSN_3(STX, MEM, B), \
INSN_3(STX, MEM, H), \
INSN_3(STX, MEM, W), \
INSN_3(STX, MEM, DW), \
INSN_3(STX, ATOMIC, W), \
INSN_3(STX, ATOMIC, DW), \
/* Immediate based. */ \
INSN_3(ST, MEM, B), \
INSN_3(ST, MEM, H), \
@ -2152,24 +2155,33 @@ out:
if (BPF_SIZE(insn->code) == BPF_W) \
atomic_##KOP((u32) SRC, (atomic_t *)(unsigned long) \
(DST + insn->off)); \
else \
else if (BPF_SIZE(insn->code) == BPF_DW) \
atomic64_##KOP((u64) SRC, (atomic64_t *)(unsigned long) \
(DST + insn->off)); \
else \
goto default_label; \
break; \
case BOP | BPF_FETCH: \
if (BPF_SIZE(insn->code) == BPF_W) \
SRC = (u32) atomic_fetch_##KOP( \
(u32) SRC, \
(atomic_t *)(unsigned long) (DST + insn->off)); \
else \
else if (BPF_SIZE(insn->code) == BPF_DW) \
SRC = (u64) atomic64_fetch_##KOP( \
(u64) SRC, \
(atomic64_t *)(unsigned long) (DST + insn->off)); \
else \
goto default_label; \
break;
STX_ATOMIC_DW:
STX_ATOMIC_W:
STX_ATOMIC_H:
STX_ATOMIC_B:
switch (IMM) {
/* Atomic read-modify-write instructions support only W and DW
* size modifiers.
*/
ATOMIC_ALU_OP(BPF_ADD, add)
ATOMIC_ALU_OP(BPF_AND, and)
ATOMIC_ALU_OP(BPF_OR, or)
@ -2181,20 +2193,63 @@ out:
SRC = (u32) atomic_xchg(
(atomic_t *)(unsigned long) (DST + insn->off),
(u32) SRC);
else
else if (BPF_SIZE(insn->code) == BPF_DW)
SRC = (u64) atomic64_xchg(
(atomic64_t *)(unsigned long) (DST + insn->off),
(u64) SRC);
else
goto default_label;
break;
case BPF_CMPXCHG:
if (BPF_SIZE(insn->code) == BPF_W)
BPF_R0 = (u32) atomic_cmpxchg(
(atomic_t *)(unsigned long) (DST + insn->off),
(u32) BPF_R0, (u32) SRC);
else
else if (BPF_SIZE(insn->code) == BPF_DW)
BPF_R0 = (u64) atomic64_cmpxchg(
(atomic64_t *)(unsigned long) (DST + insn->off),
(u64) BPF_R0, (u64) SRC);
else
goto default_label;
break;
/* Atomic load and store instructions support all size
* modifiers.
*/
case BPF_LOAD_ACQ:
switch (BPF_SIZE(insn->code)) {
#define LOAD_ACQUIRE(SIZEOP, SIZE) \
case BPF_##SIZEOP: \
DST = (SIZE)smp_load_acquire( \
(SIZE *)(unsigned long)(SRC + insn->off)); \
break;
LOAD_ACQUIRE(B, u8)
LOAD_ACQUIRE(H, u16)
LOAD_ACQUIRE(W, u32)
#ifdef CONFIG_64BIT
LOAD_ACQUIRE(DW, u64)
#endif
#undef LOAD_ACQUIRE
default:
goto default_label;
}
break;
case BPF_STORE_REL:
switch (BPF_SIZE(insn->code)) {
#define STORE_RELEASE(SIZEOP, SIZE) \
case BPF_##SIZEOP: \
smp_store_release( \
(SIZE *)(unsigned long)(DST + insn->off), (SIZE)SRC); \
break;
STORE_RELEASE(B, u8)
STORE_RELEASE(H, u16)
STORE_RELEASE(W, u32)
#ifdef CONFIG_64BIT
STORE_RELEASE(DW, u64)
#endif
#undef STORE_RELEASE
default:
goto default_label;
}
break;
default:
@ -2290,17 +2345,18 @@ void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth)
insn->code = BPF_JMP | BPF_CALL_ARGS;
}
#endif
#else
#endif
static unsigned int __bpf_prog_ret0_warn(const void *ctx,
const struct bpf_insn *insn)
{
/* If this handler ever gets executed, then BPF_JIT_ALWAYS_ON
* is not working properly, so warn about it!
* is not working properly, or interpreter is being used when
* prog->jit_requested is not 0, so warn about it!
*/
WARN_ON_ONCE(1);
return 0;
}
#endif
bool bpf_prog_map_compatible(struct bpf_map *map,
const struct bpf_prog *fp)
@ -2380,8 +2436,18 @@ static void bpf_prog_select_func(struct bpf_prog *fp)
{
#ifndef CONFIG_BPF_JIT_ALWAYS_ON
u32 stack_depth = max_t(u32, fp->aux->stack_depth, 1);
u32 idx = (round_up(stack_depth, 32) / 32) - 1;
fp->bpf_func = interpreters[(round_up(stack_depth, 32) / 32) - 1];
/* may_goto may cause stack size > 512, leading to idx out-of-bounds.
* But for non-JITed programs, we don't need bpf_func, so no bounds
* check needed.
*/
if (!fp->jit_requested &&
!WARN_ON_ONCE(idx >= ARRAY_SIZE(interpreters))) {
fp->bpf_func = interpreters[idx];
} else {
fp->bpf_func = __bpf_prog_ret0_warn;
}
#else
fp->bpf_func = __bpf_prog_ret0_warn;
#endif
@ -2906,6 +2972,11 @@ const struct bpf_func_proto * __weak bpf_get_trace_vprintk_proto(void)
return NULL;
}
const struct bpf_func_proto * __weak bpf_get_perf_event_read_value_proto(void)
{
return NULL;
}
u64 __weak
bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy)
@ -3058,6 +3129,32 @@ void __weak arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp,
{
}
bool __weak bpf_jit_supports_timed_may_goto(void)
{
return false;
}
u64 __weak arch_bpf_timed_may_goto(void)
{
return 0;
}
u64 bpf_check_timed_may_goto(struct bpf_timed_may_goto *p)
{
u64 time = ktime_get_mono_fast_ns();
/* Populate the timestamp for this stack frame, and refresh count. */
if (!p->timestamp) {
p->timestamp = time;
return BPF_MAX_TIMED_LOOPS;
}
/* Check if we've exhausted our time slice, and zero count. */
if (time - p->timestamp >= (NSEC_PER_SEC / 4))
return 0;
/* Refresh the count for the stack frame. */
return BPF_MAX_TIMED_LOOPS;
}
/* for configs without MMU or 32-bit */
__weak const struct bpf_map_ops arena_map_ops;
__weak u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena)

53
kernel/bpf/cpumask.c
View File

@ -45,6 +45,10 @@ __bpf_kfunc_start_defs();
*
* bpf_cpumask_create() allocates memory using the BPF memory allocator, and
* will not block. It may return NULL if no memory is available.
*
* Return:
* * A pointer to a new struct bpf_cpumask instance on success.
* * NULL if the BPF memory allocator is out of memory.
*/
__bpf_kfunc struct bpf_cpumask *bpf_cpumask_create(void)
{
@ -71,6 +75,10 @@ __bpf_kfunc struct bpf_cpumask *bpf_cpumask_create(void)
* Acquires a reference to a BPF cpumask. The cpumask returned by this function
* must either be embedded in a map as a kptr, or freed with
* bpf_cpumask_release().
*
* Return:
* * The struct bpf_cpumask pointer passed to the function.
*
*/
__bpf_kfunc struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask)
{
@ -106,6 +114,9 @@ CFI_NOSEAL(bpf_cpumask_release_dtor);
*
* Find the index of the first nonzero bit of the cpumask. A struct bpf_cpumask
* pointer may be safely passed to this function.
*
* Return:
* * The index of the first nonzero bit in the struct cpumask.
*/
__bpf_kfunc u32 bpf_cpumask_first(const struct cpumask *cpumask)
{
@ -119,6 +130,9 @@ __bpf_kfunc u32 bpf_cpumask_first(const struct cpumask *cpumask)
*
* Find the index of the first unset bit of the cpumask. A struct bpf_cpumask
* pointer may be safely passed to this function.
*
* Return:
* * The index of the first zero bit in the struct cpumask.
*/
__bpf_kfunc u32 bpf_cpumask_first_zero(const struct cpumask *cpumask)
{
@ -133,6 +147,9 @@ __bpf_kfunc u32 bpf_cpumask_first_zero(const struct cpumask *cpumask)
*
* Find the index of the first nonzero bit of the AND of two cpumasks.
* struct bpf_cpumask pointers may be safely passed to @src1 and @src2.
*
* Return:
* * The index of the first bit that is nonzero in both cpumask instances.
*/
__bpf_kfunc u32 bpf_cpumask_first_and(const struct cpumask *src1,
const struct cpumask *src2)
@ -414,12 +431,47 @@ __bpf_kfunc u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1,
* @cpumask: The cpumask being queried.
*
* Count the number of set bits in the given cpumask.
*
* Return:
* * The number of bits set in the mask.
*/
__bpf_kfunc u32 bpf_cpumask_weight(const struct cpumask *cpumask)
{
return cpumask_weight(cpumask);
}
/**
* bpf_cpumask_populate() - Populate the CPU mask from the contents of
* a BPF memory region.
*
* @cpumask: The cpumask being populated.
* @src: The BPF memory holding the bit pattern.
* @src__sz: Length of the BPF memory region in bytes.
*
* Return:
* * 0 if the struct cpumask * instance was populated successfully.
* * -EACCES if the memory region is too small to populate the cpumask.
* * -EINVAL if the memory region is not aligned to the size of a long
* and the architecture does not support efficient unaligned accesses.
*/
__bpf_kfunc int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t src__sz)
{
unsigned long source = (unsigned long)src;
/* The memory region must be large enough to populate the entire CPU mask. */
if (src__sz < bitmap_size(nr_cpu_ids))
return -EACCES;
/* If avoiding unaligned accesses, the input region must be aligned to the nearest long. */
if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
!IS_ALIGNED(source, sizeof(long)))
return -EINVAL;
bitmap_copy(cpumask_bits(cpumask), src, nr_cpu_ids);
return 0;
}
__bpf_kfunc_end_defs();
BTF_KFUNCS_START(cpumask_kfunc_btf_ids)
@ -448,6 +500,7 @@ BTF_ID_FLAGS(func, bpf_cpumask_copy, KF_RCU)
BTF_ID_FLAGS(func, bpf_cpumask_any_distribute, KF_RCU)
BTF_ID_FLAGS(func, bpf_cpumask_any_and_distribute, KF_RCU)
BTF_ID_FLAGS(func, bpf_cpumask_weight, KF_RCU)
BTF_ID_FLAGS(func, bpf_cpumask_populate, KF_RCU)
BTF_KFUNCS_END(cpumask_kfunc_btf_ids)
static const struct btf_kfunc_id_set cpumask_kfunc_set = {

16
kernel/bpf/disasm.c
View File

@ -202,7 +202,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs,
insn->dst_reg, class == BPF_ALU ? 'w' : 'r',
insn->dst_reg);
} else if (is_addr_space_cast(insn)) {
verbose(cbs->private_data, "(%02x) r%d = addr_space_cast(r%d, %d, %d)\n",
verbose(cbs->private_data, "(%02x) r%d = addr_space_cast(r%d, %u, %u)\n",
insn->code, insn->dst_reg,
insn->src_reg, ((u32)insn->imm) >> 16, (u16)insn->imm);
} else if (is_mov_percpu_addr(insn)) {
@ -267,6 +267,18 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs,
BPF_SIZE(insn->code) == BPF_DW ? "64" : "",
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
insn->dst_reg, insn->off, insn->src_reg);
} else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
insn->imm == BPF_LOAD_ACQ) {
verbose(cbs->private_data, "(%02x) r%d = load_acquire((%s *)(r%d %+d))\n",
insn->code, insn->dst_reg,
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
insn->src_reg, insn->off);
} else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
insn->imm == BPF_STORE_REL) {
verbose(cbs->private_data, "(%02x) store_release((%s *)(r%d %+d), r%d)\n",
insn->code,
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
insn->dst_reg, insn->off, insn->src_reg);
} else {
verbose(cbs->private_data, "BUG_%02x\n", insn->code);
}
@ -369,7 +381,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs,
insn->code, class == BPF_JMP32 ? 'w' : 'r',
insn->dst_reg,
bpf_jmp_string[BPF_OP(insn->code) >> 4],
insn->imm, insn->off);
(u32)insn->imm, insn->off);
}
} else {
verbose(cbs->private_data, "(%02x) %s\n",

9
kernel/bpf/hashtab.c
View File

@ -198,12 +198,12 @@ static bool htab_is_percpu(const struct bpf_htab *htab)
static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size,
void __percpu *pptr)
{
*(void __percpu **)(l->key + key_size) = pptr;
*(void __percpu **)(l->key + roundup(key_size, 8)) = pptr;
}
static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size)
{
return *(void __percpu **)(l->key + key_size);
return *(void __percpu **)(l->key + roundup(key_size, 8));
}
static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l)
@ -787,6 +787,9 @@ static int htab_lru_map_gen_lookup(struct bpf_map *map,
static void check_and_free_fields(struct bpf_htab *htab,
struct htab_elem *elem)
{
if (IS_ERR_OR_NULL(htab->map.record))
return;
if (htab_is_percpu(htab)) {
void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size);
int cpu;
@ -2354,7 +2357,7 @@ static int htab_percpu_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn
*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);
*insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3);
*insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0,
offsetof(struct htab_elem, key) + map->key_size);
offsetof(struct htab_elem, key) + roundup(map->key_size, 8));
*insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);
*insn++ = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);

123
kernel/bpf/helpers.c
View File

@ -1758,8 +1758,8 @@ static const struct bpf_func_proto bpf_dynptr_from_mem_proto = {
.arg4_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL | MEM_UNINIT | MEM_WRITE,
};
BPF_CALL_5(bpf_dynptr_read, void *, dst, u32, len, const struct bpf_dynptr_kern *, src,
u32, offset, u64, flags)
static int __bpf_dynptr_read(void *dst, u32 len, const struct bpf_dynptr_kern *src,
u32 offset, u64 flags)
{
enum bpf_dynptr_type type;
int err;
@ -1792,6 +1792,12 @@ BPF_CALL_5(bpf_dynptr_read, void *, dst, u32, len, const struct bpf_dynptr_kern
}
}
BPF_CALL_5(bpf_dynptr_read, void *, dst, u32, len, const struct bpf_dynptr_kern *, src,
u32, offset, u64, flags)
{
return __bpf_dynptr_read(dst, len, src, offset, flags);
}
static const struct bpf_func_proto bpf_dynptr_read_proto = {
.func = bpf_dynptr_read,
.gpl_only = false,
@ -1803,8 +1809,8 @@ static const struct bpf_func_proto bpf_dynptr_read_proto = {
.arg5_type = ARG_ANYTHING,
};
BPF_CALL_5(bpf_dynptr_write, const struct bpf_dynptr_kern *, dst, u32, offset, void *, src,
u32, len, u64, flags)
static int __bpf_dynptr_write(const struct bpf_dynptr_kern *dst, u32 offset, void *src,
u32 len, u64 flags)
{
enum bpf_dynptr_type type;
int err;
@ -1842,6 +1848,12 @@ BPF_CALL_5(bpf_dynptr_write, const struct bpf_dynptr_kern *, dst, u32, offset, v
}
}
BPF_CALL_5(bpf_dynptr_write, const struct bpf_dynptr_kern *, dst, u32, offset, void *, src,
u32, len, u64, flags)
{
return __bpf_dynptr_write(dst, offset, src, len, flags);
}
static const struct bpf_func_proto bpf_dynptr_write_proto = {
.func = bpf_dynptr_write,
.gpl_only = false,
@ -2043,6 +2055,8 @@ bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
return &bpf_task_pt_regs_proto;
case BPF_FUNC_trace_vprintk:
return bpf_get_trace_vprintk_proto();
case BPF_FUNC_perf_event_read_value:
return bpf_get_perf_event_read_value_proto();
default:
return NULL;
}
@ -2757,6 +2771,61 @@ __bpf_kfunc int bpf_dynptr_clone(const struct bpf_dynptr *p,
return 0;
}
/**
* bpf_dynptr_copy() - Copy data from one dynptr to another.
* @dst_ptr: Destination dynptr - where data should be copied to
* @dst_off: Offset into the destination dynptr
* @src_ptr: Source dynptr - where data should be copied from
* @src_off: Offset into the source dynptr
* @size: Length of the data to copy from source to destination
*
* Copies data from source dynptr to destination dynptr.
* Returns 0 on success; negative error, otherwise.
*/
__bpf_kfunc int bpf_dynptr_copy(struct bpf_dynptr *dst_ptr, u32 dst_off,
struct bpf_dynptr *src_ptr, u32 src_off, u32 size)
{
struct bpf_dynptr_kern *dst = (struct bpf_dynptr_kern *)dst_ptr;
struct bpf_dynptr_kern *src = (struct bpf_dynptr_kern *)src_ptr;
void *src_slice, *dst_slice;
char buf[256];
u32 off;
src_slice = bpf_dynptr_slice(src_ptr, src_off, NULL, size);
dst_slice = bpf_dynptr_slice_rdwr(dst_ptr, dst_off, NULL, size);
if (src_slice && dst_slice) {
memmove(dst_slice, src_slice, size);
return 0;
}
if (src_slice)
return __bpf_dynptr_write(dst, dst_off, src_slice, size, 0);
if (dst_slice)
return __bpf_dynptr_read(dst_slice, size, src, src_off, 0);
if (bpf_dynptr_check_off_len(dst, dst_off, size) ||
bpf_dynptr_check_off_len(src, src_off, size))
return -E2BIG;
off = 0;
while (off < size) {
u32 chunk_sz = min_t(u32, sizeof(buf), size - off);
int err;
err = __bpf_dynptr_read(buf, chunk_sz, src, src_off + off, 0);
if (err)
return err;
err = __bpf_dynptr_write(dst, dst_off + off, buf, chunk_sz, 0);
if (err)
return err;
off += chunk_sz;
}
return 0;
}
__bpf_kfunc void *bpf_cast_to_kern_ctx(void *obj)
{
return obj;
@ -3066,6 +3135,50 @@ __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __user
return ret + 1;
}
/**
* bpf_copy_from_user_task_str() - Copy a string from an task's address space
* @dst: Destination address, in kernel space. This buffer must be
* at least @dst__sz bytes long.
* @dst__sz: Maximum number of bytes to copy, includes the trailing NUL.
* @unsafe_ptr__ign: Source address in the task's address space.
* @tsk: The task whose address space will be used
* @flags: The only supported flag is BPF_F_PAD_ZEROS
*
* Copies a NUL terminated string from a task's address space to @dst__sz
* buffer. If user string is too long this will still ensure zero termination
* in the @dst__sz buffer unless buffer size is 0.
*
* If BPF_F_PAD_ZEROS flag is set, memset the tail of @dst__sz to 0 on success
* and memset all of @dst__sz on failure.
*
* Return: The number of copied bytes on success including the NUL terminator.
* A negative error code on failure.
*/
__bpf_kfunc int bpf_copy_from_user_task_str(void *dst, u32 dst__sz,
const void __user *unsafe_ptr__ign,
struct task_struct *tsk, u64 flags)
{
int ret;
if (unlikely(flags & ~BPF_F_PAD_ZEROS))
return -EINVAL;
if (unlikely(dst__sz == 0))
return 0;
ret = copy_remote_vm_str(tsk, (unsigned long)unsafe_ptr__ign, dst, dst__sz, 0);
if (ret < 0) {
if (flags & BPF_F_PAD_ZEROS)
memset(dst, 0, dst__sz);
return ret;
}
if (flags & BPF_F_PAD_ZEROS)
memset(dst + ret, 0, dst__sz - ret);
return ret + 1;
}
/* Keep unsinged long in prototype so that kfunc is usable when emitted to
* vmlinux.h in BPF programs directly, but note that while in BPF prog, the
* unsigned long always points to 8-byte region on stack, the kernel may only
@ -3161,6 +3274,7 @@ BTF_ID_FLAGS(func, bpf_dynptr_is_null)
BTF_ID_FLAGS(func, bpf_dynptr_is_rdonly)
BTF_ID_FLAGS(func, bpf_dynptr_size)
BTF_ID_FLAGS(func, bpf_dynptr_clone)
BTF_ID_FLAGS(func, bpf_dynptr_copy)
#ifdef CONFIG_NET
BTF_ID_FLAGS(func, bpf_modify_return_test_tp)
#endif
@ -3173,6 +3287,7 @@ BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW)
BTF_ID_FLAGS(func, bpf_iter_bits_next, KF_ITER_NEXT | KF_RET_NULL)
BTF_ID_FLAGS(func, bpf_iter_bits_destroy, KF_ITER_DESTROY)
BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE)
BTF_ID_FLAGS(func, bpf_copy_from_user_task_str, KF_SLEEPABLE)
BTF_ID_FLAGS(func, bpf_get_kmem_cache)
BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE)
BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE)

1
kernel/bpf/preload/bpf_preload_kern.c
View File

@ -90,3 +90,4 @@ static void __exit fini(void)
late_initcall(load);
module_exit(fini);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Embedded BPF programs for introspection in bpffs");

48
kernel/bpf/syscall.c
View File

@ -1315,7 +1315,7 @@ static bool bpf_net_capable(void)
#define BPF_MAP_CREATE_LAST_FIELD map_token_fd
/* called via syscall */
static int map_create(union bpf_attr *attr)
static int map_create(union bpf_attr *attr, bool kernel)
{
const struct bpf_map_ops *ops;
struct bpf_token *token = NULL;
@ -1505,7 +1505,7 @@ static int map_create(union bpf_attr *attr)
attr->btf_vmlinux_value_type_id;
}
err = security_bpf_map_create(map, attr, token);
err = security_bpf_map_create(map, attr, token, kernel);
if (err)
goto free_map_sec;
@ -1593,11 +1593,8 @@ struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref)
struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map)
{
spin_lock_bh(&map_idr_lock);
map = __bpf_map_inc_not_zero(map, false);
spin_unlock_bh(&map_idr_lock);
return map;
lockdep_assert(rcu_read_lock_held());
return __bpf_map_inc_not_zero(map, false);
}
EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero);
@ -2314,6 +2311,7 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred)
kvfree(prog->aux->jited_linfo);
kvfree(prog->aux->linfo);
kfree(prog->aux->kfunc_tab);
kfree(prog->aux->ctx_arg_info);
if (prog->aux->attach_btf)
btf_put(prog->aux->attach_btf);
@ -2944,7 +2942,7 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size)
if (err < 0)
goto free_prog;
err = security_bpf_prog_load(prog, attr, token);
err = security_bpf_prog_load(prog, attr, token, uattr.is_kernel);
if (err)
goto free_prog_sec;
@ -4169,7 +4167,8 @@ static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,
#define BPF_F_ATTACH_MASK_BASE \
(BPF_F_ALLOW_OVERRIDE | \
BPF_F_ALLOW_MULTI | \
BPF_F_REPLACE)
BPF_F_REPLACE | \
BPF_F_PREORDER)
#define BPF_F_ATTACH_MASK_MPROG \
(BPF_F_REPLACE | \
@ -4733,6 +4732,8 @@ static int bpf_prog_get_info_by_fd(struct file *file,
info.recursion_misses = stats.misses;
info.verified_insns = prog->aux->verified_insns;
if (prog->aux->btf)
info.btf_id = btf_obj_id(prog->aux->btf);
if (!bpf_capable()) {
info.jited_prog_len = 0;
@ -4879,8 +4880,6 @@ static int bpf_prog_get_info_by_fd(struct file *file,
}
}
if (prog->aux->btf)
info.btf_id = btf_obj_id(prog->aux->btf);
info.attach_btf_id = prog->aux->attach_btf_id;
if (attach_btf)
info.attach_btf_obj_id = btf_obj_id(attach_btf);
@ -5121,15 +5120,34 @@ static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_
return btf_new_fd(attr, uattr, uattr_size);
}
#define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id
#define BPF_BTF_GET_FD_BY_ID_LAST_FIELD fd_by_id_token_fd
static int bpf_btf_get_fd_by_id(const union bpf_attr *attr)
{
struct bpf_token *token = NULL;
if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID))
return -EINVAL;
if (!capable(CAP_SYS_ADMIN))
if (attr->open_flags & ~BPF_F_TOKEN_FD)
return -EINVAL;
if (attr->open_flags & BPF_F_TOKEN_FD) {
token = bpf_token_get_from_fd(attr->fd_by_id_token_fd);
if (IS_ERR(token))
return PTR_ERR(token);
if (!bpf_token_allow_cmd(token, BPF_BTF_GET_FD_BY_ID)) {
bpf_token_put(token);
token = NULL;
}
}
if (!bpf_token_capable(token, CAP_SYS_ADMIN)) {
bpf_token_put(token);
return -EPERM;
}
bpf_token_put(token);
return btf_get_fd_by_id(attr->btf_id);
}
@ -5768,13 +5786,13 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size)
if (copy_from_bpfptr(&attr, uattr, size) != 0)
return -EFAULT;
err = security_bpf(cmd, &attr, size);
err = security_bpf(cmd, &attr, size, uattr.is_kernel);
if (err < 0)
return err;
switch (cmd) {
case BPF_MAP_CREATE:
err = map_create(&attr);
err = map_create(&attr, uattr.is_kernel);
break;
case BPF_MAP_LOOKUP_ELEM:
err = map_lookup_elem(&attr);

1360
kernel/bpf/verifier.c
View File

File diff suppressed because it is too large Load Diff

14
kernel/trace/bpf_trace.c
View File

@ -392,7 +392,7 @@ static const struct bpf_func_proto bpf_trace_printk_proto = {
.arg2_type = ARG_CONST_SIZE,
};
static void __set_printk_clr_event(void)
static void __set_printk_clr_event(struct work_struct *work)
{
/*
* This program might be calling bpf_trace_printk,
@ -405,10 +405,11 @@ static void __set_printk_clr_event(void)
if (trace_set_clr_event("bpf_trace", "bpf_trace_printk", 1))
pr_warn_ratelimited("could not enable bpf_trace_printk events");
}
static DECLARE_WORK(set_printk_work, __set_printk_clr_event);
const struct bpf_func_proto *bpf_get_trace_printk_proto(void)
{
__set_printk_clr_event();
schedule_work(&set_printk_work);
return &bpf_trace_printk_proto;
}
@ -451,7 +452,7 @@ static const struct bpf_func_proto bpf_trace_vprintk_proto = {
const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void)
{
__set_printk_clr_event();
schedule_work(&set_printk_work);
return &bpf_trace_vprintk_proto;
}
@ -606,6 +607,11 @@ static const struct bpf_func_proto bpf_perf_event_read_value_proto = {
.arg4_type = ARG_CONST_SIZE,
};
const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void)
{
return &bpf_perf_event_read_value_proto;
}
static __always_inline u64
__bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,
u64 flags, struct perf_raw_record *raw,
@ -843,7 +849,7 @@ static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struc
if (unlikely(is_global_init(task)))
return -EPERM;
if (!preemptible()) {
if (preempt_count() != 0 || irqs_disabled()) {
/* Do an early check on signal validity. Otherwise,
* the error is lost in deferred irq_work.
*/

118
mm/memory.c
View File

@ -6807,6 +6807,124 @@ int access_process_vm(struct task_struct *tsk, unsigned long addr,
}
EXPORT_SYMBOL_GPL(access_process_vm);
#ifdef CONFIG_BPF_SYSCALL
/*
* Copy a string from another process's address space as given in mm.
* If there is any error return -EFAULT.
*/
static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr,
void *buf, int len, unsigned int gup_flags)
{
void *old_buf = buf;
int err = 0;
*(char *)buf = '\0';
if (mmap_read_lock_killable(mm))
return -EFAULT;
addr = untagged_addr_remote(mm, addr);
/* Avoid triggering the temporary warning in __get_user_pages */
if (!vma_lookup(mm, addr)) {
err = -EFAULT;
goto out;
}
while (len) {
int bytes, offset, retval;
void *maddr;
struct page *page;
struct vm_area_struct *vma = NULL;
page = get_user_page_vma_remote(mm, addr, gup_flags, &vma);
if (IS_ERR(page)) {
/*
* Treat as a total failure for now until we decide how
* to handle the CONFIG_HAVE_IOREMAP_PROT case and
* stack expansion.
*/
*(char *)buf = '\0';
err = -EFAULT;
goto out;
}
bytes = len;
offset = addr & (PAGE_SIZE - 1);
if (bytes > PAGE_SIZE - offset)
bytes = PAGE_SIZE - offset;
maddr = kmap_local_page(page);
retval = strscpy(buf, maddr + offset, bytes);
if (retval >= 0) {
/* Found the end of the string */
buf += retval;
unmap_and_put_page(page, maddr);
break;
}
buf += bytes - 1;
/*
* Because strscpy always NUL terminates we need to
* copy the last byte in the page if we are going to
* load more pages
*/
if (bytes != len) {
addr += bytes - 1;
copy_from_user_page(vma, page, addr, buf, maddr + (PAGE_SIZE - 1), 1);
buf += 1;
addr += 1;
}
len -= bytes;
unmap_and_put_page(page, maddr);
}
out:
mmap_read_unlock(mm);
if (err)
return err;
return buf - old_buf;
}
/**
* copy_remote_vm_str - copy a string from another process's address space.
* @tsk: the task of the target address space
* @addr: start address to read from
* @buf: destination buffer
* @len: number of bytes to copy
* @gup_flags: flags modifying lookup behaviour
*
* The caller must hold a reference on @mm.
*
* Return: number of bytes copied from @addr (source) to @buf (destination);
* not including the trailing NUL. Always guaranteed to leave NUL-terminated
* buffer. On any error, return -EFAULT.
*/
int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr,
void *buf, int len, unsigned int gup_flags)
{
struct mm_struct *mm;
int ret;
if (unlikely(len == 0))
return 0;
mm = get_task_mm(tsk);
if (!mm) {
*(char *)buf = '\0';
return -EFAULT;
}
ret = __copy_remote_vm_str(mm, addr, buf, len, gup_flags);
mmput(mm);
return ret;
}
EXPORT_SYMBOL_GPL(copy_remote_vm_str);
#endif /* CONFIG_BPF_SYSCALL */
/*
* Print the name of a VMA.
*/

79
mm/nommu.c
View File

@ -1714,6 +1714,85 @@ int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, in
}
EXPORT_SYMBOL_GPL(access_process_vm);
#ifdef CONFIG_BPF_SYSCALL
/*
* Copy a string from another process's address space as given in mm.
* If there is any error return -EFAULT.
*/
static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr,
void *buf, int len)
{
unsigned long addr_end;
struct vm_area_struct *vma;
int ret = -EFAULT;
*(char *)buf = '\0';
if (mmap_read_lock_killable(mm))
return ret;
/* the access must start within one of the target process's mappings */
vma = find_vma(mm, addr);
if (!vma)
goto out;
if (check_add_overflow(addr, len, &addr_end))
goto out;
/* don't overrun this mapping */
if (addr_end > vma->vm_end)
len = vma->vm_end - addr;
/* only read mappings where it is permitted */
if (vma->vm_flags & VM_MAYREAD) {
ret = strscpy(buf, (char *)addr, len);
if (ret < 0)
ret = len - 1;
}
out:
mmap_read_unlock(mm);
return ret;
}
/**
* copy_remote_vm_str - copy a string from another process's address space.
* @tsk: the task of the target address space
* @addr: start address to read from
* @buf: destination buffer
* @len: number of bytes to copy
* @gup_flags: flags modifying lookup behaviour (unused)
*
* The caller must hold a reference on @mm.
*
* Return: number of bytes copied from @addr (source) to @buf (destination);
* not including the trailing NUL. Always guaranteed to leave NUL-terminated
* buffer. On any error, return -EFAULT.
*/
int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr,
void *buf, int len, unsigned int gup_flags)
{
struct mm_struct *mm;
int ret;
if (unlikely(len == 0))
return 0;
mm = get_task_mm(tsk);
if (!mm) {
*(char *)buf = '\0';
return -EFAULT;
}
ret = __copy_remote_vm_str(mm, addr, buf, len);
mmput(mm);
return ret;
}
EXPORT_SYMBOL_GPL(copy_remote_vm_str);
#endif /* CONFIG_BPF_SYSCALL */
/**
* nommu_shrink_inode_mappings - Shrink the shared mappings on an inode
* @inode: The inode to check

6
net/core/filter.c
View File

@ -8137,6 +8137,8 @@ sk_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
return &bpf_skb_load_bytes_relative_proto;
case BPF_FUNC_get_socket_cookie:
return &bpf_get_socket_cookie_proto;
case BPF_FUNC_get_netns_cookie:
return &bpf_get_netns_cookie_proto;
case BPF_FUNC_get_socket_uid:
return &bpf_get_socket_uid_proto;
case BPF_FUNC_perf_event_output:
@ -9697,7 +9699,7 @@ static u32 bpf_convert_ctx_access(enum bpf_access_type type,
case offsetof(struct __sk_buff, queue_mapping):
if (type == BPF_WRITE) {
u32 off = bpf_target_off(struct sk_buff, queue_mapping, 2, target_size);
u32 offset = bpf_target_off(struct sk_buff, queue_mapping, 2, target_size);
if (BPF_CLASS(si->code) == BPF_ST && si->imm >= NO_QUEUE_MAPPING) {
*insn++ = BPF_JMP_A(0); /* noop */
@ -9706,7 +9708,7 @@ static u32 bpf_convert_ctx_access(enum bpf_access_type type,
if (BPF_CLASS(si->code) == BPF_STX)
*insn++ = BPF_JMP_IMM(BPF_JGE, si->src_reg, NO_QUEUE_MAPPING, 1);
*insn++ = BPF_EMIT_STORE(BPF_H, si, off);
*insn++ = BPF_EMIT_STORE(BPF_H, si, offset);
} else {
*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
bpf_target_off(struct sk_buff,

2
samples/bpf/Makefile
View File

@ -307,7 +307,7 @@ $(obj)/$(TRACE_HELPERS): TPROGS_CFLAGS := $(TPROGS_CFLAGS) -D__must_check=
VMLINUX_BTF_PATHS ?= $(abspath $(if $(O),$(O)/vmlinux)) \
$(abspath $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux)) \
$(abspath ./vmlinux)
$(abspath $(objtree)/vmlinux)
VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS))))
$(obj)/vmlinux.h: $(VMLINUX_BTF) $(BPFTOOL)

2
scripts/Makefile.btf
View File

@ -24,7 +24,7 @@ else
pahole-flags-$(call test-ge, $(pahole-ver), 126) = -j$(JOBS) --btf_features=encode_force,var,float,enum64,decl_tag,type_tag,optimized_func,consistent_func,decl_tag_kfuncs
ifneq ($(KBUILD_EXTMOD),)
module-pahole-flags-$(call test-ge, $(pahole-ver), 126) += --btf_features=distilled_base
module-pahole-flags-$(call test-ge, $(pahole-ver), 128) += --btf_features=distilled_base
endif
endif

15
security/security.c
View File

@ -5627,6 +5627,7 @@ int security_audit_rule_match(struct lsm_prop *prop, u32 field, u32 op,
* @cmd: command
* @attr: bpf attribute
* @size: size
* @kernel: whether or not call originated from kernel
*
* Do a initial check for all bpf syscalls after the attribute is copied into
* the kernel. The actual security module can implement their own rules to
@ -5634,9 +5635,9 @@ int security_audit_rule_match(struct lsm_prop *prop, u32 field, u32 op,
*
* Return: Returns 0 if permission is granted.
*/
int security_bpf(int cmd, union bpf_attr *attr, unsigned int size)
int security_bpf(int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
{
return call_int_hook(bpf, cmd, attr, size);
return call_int_hook(bpf, cmd, attr, size, kernel);
}
/**
@ -5673,6 +5674,7 @@ int security_bpf_prog(struct bpf_prog *prog)
* @map: BPF map object
* @attr: BPF syscall attributes used to create BPF map
* @token: BPF token used to grant user access
* @kernel: whether or not call originated from kernel
*
* Do a check when the kernel creates a new BPF map. This is also the
* point where LSM blob is allocated for LSMs that need them.
@ -5680,9 +5682,9 @@ int security_bpf_prog(struct bpf_prog *prog)
* Return: Returns 0 on success, error on failure.
*/
int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr,
struct bpf_token *token)
struct bpf_token *token, bool kernel)
{
return call_int_hook(bpf_map_create, map, attr, token);
return call_int_hook(bpf_map_create, map, attr, token, kernel);
}
/**
@ -5690,6 +5692,7 @@ int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr,
* @prog: BPF program object
* @attr: BPF syscall attributes used to create BPF program
* @token: BPF token used to grant user access to BPF subsystem
* @kernel: whether or not call originated from kernel
*
* Perform an access control check when the kernel loads a BPF program and
* allocates associated BPF program object. This hook is also responsible for
@ -5698,9 +5701,9 @@ int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr,
* Return: Returns 0 on success, error on failure.
*/
int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr,
struct bpf_token *token)
struct bpf_token *token, bool kernel)
{
return call_int_hook(bpf_prog_load, prog, attr, token);
return call_int_hook(bpf_prog_load, prog, attr, token, kernel);
}
/**

6
security/selinux/hooks.c
View File

@ -6907,7 +6907,7 @@ static int selinux_ib_alloc_security(void *ib_sec)
#ifdef CONFIG_BPF_SYSCALL
static int selinux_bpf(int cmd, union bpf_attr *attr,
unsigned int size)
unsigned int size, bool kernel)
{
u32 sid = current_sid();
int ret;
@ -6994,7 +6994,7 @@ static int selinux_bpf_prog(struct bpf_prog *prog)
}
static int selinux_bpf_map_create(struct bpf_map *map, union bpf_attr *attr,
struct bpf_token *token)
struct bpf_token *token, bool kernel)
{
struct bpf_security_struct *bpfsec;
@ -7017,7 +7017,7 @@ static void selinux_bpf_map_free(struct bpf_map *map)
}
static int selinux_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr,
struct bpf_token *token)
struct bpf_token *token, bool kernel)
{
struct bpf_security_struct *bpfsec;

7
tools/bpf/bpftool/Makefile
View File

@ -65,7 +65,12 @@ prefix ?= /usr/local
bash_compdir ?= /usr/share/bash-completion/completions
CFLAGS += -O2
CFLAGS += -W -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers
CFLAGS += -W
CFLAGS += -Wall
CFLAGS += -Wextra
CFLAGS += -Wformat-signedness
CFLAGS += -Wno-unused-parameter
CFLAGS += -Wno-missing-field-initializers
CFLAGS += $(filter-out -Wswitch-enum -Wnested-externs,$(EXTRA_WARNINGS))
CFLAGS += -DPACKAGE='"bpftool"' -D__EXPORTED_HEADERS__ \
-I$(or $(OUTPUT),.) \

14
tools/bpf/bpftool/btf.c
View File

@ -253,7 +253,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id,
if (btf_kflag(t))
printf("\n\t'%s' val=%d", name, v->val);
else
printf("\n\t'%s' val=%u", name, v->val);
printf("\n\t'%s' val=%u", name, (__u32)v->val);
}
}
if (json_output)
@ -1022,7 +1022,7 @@ static int do_dump(int argc, char **argv)
for (i = 0; i < root_type_cnt; i++) {
if (root_type_ids[i] == root_id) {
err = -EINVAL;
p_err("duplicate root_id %d supplied", root_id);
p_err("duplicate root_id %u supplied", root_id);
goto done;
}
}
@ -1132,7 +1132,7 @@ build_btf_type_table(struct hashmap *tab, enum bpf_obj_type type,
break;
default:
err = -1;
p_err("unexpected object type: %d", type);
p_err("unexpected object type: %u", type);
goto err_free;
}
if (err) {
@ -1155,7 +1155,7 @@ build_btf_type_table(struct hashmap *tab, enum bpf_obj_type type,
break;
default:
err = -1;
p_err("unexpected object type: %d", type);
p_err("unexpected object type: %u", type);
goto err_free;
}
if (fd < 0) {
@ -1188,7 +1188,7 @@ build_btf_type_table(struct hashmap *tab, enum bpf_obj_type type,
break;
default:
err = -1;
p_err("unexpected object type: %d", type);
p_err("unexpected object type: %u", type);
goto err_free;
}
if (!btf_id)
@ -1254,12 +1254,12 @@ show_btf_plain(struct bpf_btf_info *info, int fd,
n = 0;
hashmap__for_each_key_entry(btf_prog_table, entry, info->id) {
printf("%s%lu", n++ == 0 ? " prog_ids " : ",", entry->value);
printf("%s%lu", n++ == 0 ? " prog_ids " : ",", (unsigned long)entry->value);
}
n = 0;
hashmap__for_each_key_entry(btf_map_table, entry, info->id) {
printf("%s%lu", n++ == 0 ? " map_ids " : ",", entry->value);
printf("%s%lu", n++ == 0 ? " map_ids " : ",", (unsigned long)entry->value);
}
emit_obj_refs_plain(refs_table, info->id, "\n\tpids ");

2
tools/bpf/bpftool/btf_dumper.c
View File

@ -653,7 +653,7 @@ static int __btf_dumper_type_only(const struct btf *btf, __u32 type_id,
case BTF_KIND_ARRAY:
array = (struct btf_array *)(t + 1);
BTF_PRINT_TYPE(array->type);
BTF_PRINT_ARG("[%d]", array->nelems);
BTF_PRINT_ARG("[%u]", array->nelems);
break;
case BTF_KIND_PTR:
BTF_PRINT_TYPE(t->type);

2
tools/bpf/bpftool/cgroup.c
View File

@ -191,7 +191,7 @@ static int show_bpf_prog(int id, enum bpf_attach_type attach_type,
if (attach_btf_name)
printf(" %-15s", attach_btf_name);
else if (info.attach_btf_id)
printf(" attach_btf_obj_id=%d attach_btf_id=%d",
printf(" attach_btf_obj_id=%u attach_btf_id=%u",
info.attach_btf_obj_id, info.attach_btf_id);
printf("\n");
}

7
tools/bpf/bpftool/common.c
View File

@ -461,10 +461,11 @@ int get_fd_type(int fd)
p_err("can't read link type: %s", strerror(errno));
return -1;
}
if (n == sizeof(path)) {
if (n == sizeof(buf)) {
p_err("can't read link type: path too long!");
return -1;
}
buf[n] = '\0';
if (strstr(buf, "bpf-map"))
return BPF_OBJ_MAP;
@ -713,7 +714,7 @@ ifindex_to_arch(__u32 ifindex, __u64 ns_dev, __u64 ns_ino, const char **opt)
int vendor_id;
if (!ifindex_to_name_ns(ifindex, ns_dev, ns_ino, devname)) {
p_err("Can't get net device name for ifindex %d: %s", ifindex,
p_err("Can't get net device name for ifindex %u: %s", ifindex,
strerror(errno));
return NULL;
}
@ -738,7 +739,7 @@ ifindex_to_arch(__u32 ifindex, __u64 ns_dev, __u64 ns_ino, const char **opt)
/* No NFP support in LLVM, we have no valid triple to return. */
default:
p_err("Can't get arch name for device vendor id 0x%04x",
vendor_id);
(unsigned int)vendor_id);
return NULL;
}
}

12
tools/bpf/bpftool/gen.c
View File

@ -670,7 +670,7 @@ static void codegen_destroy(struct bpf_object *obj, const char *obj_name)
continue;
if (bpf_map__is_internal(map) &&
(bpf_map__map_flags(map) & BPF_F_MMAPABLE))
printf("\tskel_free_map_data(skel->%1$s, skel->maps.%1$s.initial_value, %2$zd);\n",
printf("\tskel_free_map_data(skel->%1$s, skel->maps.%1$s.initial_value, %2$zu);\n",
ident, bpf_map_mmap_sz(map));
codegen("\
\n\
@ -984,7 +984,7 @@ static int walk_st_ops_shadow_vars(struct btf *btf, const char *ident,
offset = m->offset / 8;
if (next_offset < offset)
printf("\t\t\tchar __padding_%d[%d];\n", i, offset - next_offset);
printf("\t\t\tchar __padding_%d[%u];\n", i, offset - next_offset);
switch (btf_kind(member_type)) {
case BTF_KIND_INT:
@ -1052,7 +1052,7 @@ static int walk_st_ops_shadow_vars(struct btf *btf, const char *ident,
/* Cannot fail since it must be a struct type */
size = btf__resolve_size(btf, map_type_id);
if (next_offset < (__u32)size)
printf("\t\t\tchar __padding_end[%d];\n", size - next_offset);
printf("\t\t\tchar __padding_end[%u];\n", size - next_offset);
out:
btf_dump__free(d);
@ -2095,7 +2095,7 @@ btfgen_mark_type(struct btfgen_info *info, unsigned int type_id, bool follow_poi
break;
/* tells if some other type needs to be handled */
default:
p_err("unsupported kind: %s (%d)", btf_kind_str(btf_type), type_id);
p_err("unsupported kind: %s (%u)", btf_kind_str(btf_type), type_id);
return -EINVAL;
}
@ -2147,7 +2147,7 @@ static int btfgen_record_field_relo(struct btfgen_info *info, struct bpf_core_sp
btf_type = btf__type_by_id(btf, type_id);
break;
default:
p_err("unsupported kind: %s (%d)",
p_err("unsupported kind: %s (%u)",
btf_kind_str(btf_type), btf_type->type);
return -EINVAL;
}
@ -2246,7 +2246,7 @@ static int btfgen_mark_type_match(struct btfgen_info *info, __u32 type_id, bool
}
/* tells if some other type needs to be handled */
default:
p_err("unsupported kind: %s (%d)", btf_kind_str(btf_type), type_id);
p_err("unsupported kind: %s (%u)", btf_kind_str(btf_type), type_id);
return -EINVAL;
}

3
tools/bpf/bpftool/jit_disasm.c
View File

@ -343,7 +343,8 @@ int disasm_print_insn(unsigned char *image, ssize_t len, int opcodes,
{
const struct bpf_line_info *linfo = NULL;
unsigned int nr_skip = 0;
int count, i, pc = 0;
int count, i;
unsigned int pc = 0;
disasm_ctx_t ctx;
if (!len)

14
tools/bpf/bpftool/link.c
View File

@ -107,7 +107,7 @@ static int link_parse_fd(int *argc, char ***argv)
fd = bpf_link_get_fd_by_id(id);
if (fd < 0)
p_err("failed to get link with ID %d: %s", id, strerror(errno));
p_err("failed to get link with ID %u: %s", id, strerror(errno));
return fd;
} else if (is_prefix(**argv, "pinned")) {
char *path;
@ -404,7 +404,7 @@ static char *perf_config_hw_cache_str(__u64 config)
if (hw_cache)
snprintf(str, PERF_HW_CACHE_LEN, "%s-", hw_cache);
else
snprintf(str, PERF_HW_CACHE_LEN, "%lld-", config & 0xff);
snprintf(str, PERF_HW_CACHE_LEN, "%llu-", config & 0xff);
op = perf_event_name(evsel__hw_cache_op, (config >> 8) & 0xff);
if (op)
@ -412,7 +412,7 @@ static char *perf_config_hw_cache_str(__u64 config)
"%s-", op);
else
snprintf(str + strlen(str), PERF_HW_CACHE_LEN - strlen(str),
"%lld-", (config >> 8) & 0xff);
"%llu-", (config >> 8) & 0xff);
result = perf_event_name(evsel__hw_cache_result, config >> 16);
if (result)
@ -420,7 +420,7 @@ static char *perf_config_hw_cache_str(__u64 config)
"%s", result);
else
snprintf(str + strlen(str), PERF_HW_CACHE_LEN - strlen(str),
"%lld", config >> 16);
"%llu", config >> 16);
return str;
}
@ -623,7 +623,7 @@ static void show_link_ifindex_plain(__u32 ifindex)
else
snprintf(devname, sizeof(devname), "(detached)");
if (ret)
snprintf(devname, sizeof(devname), "%s(%d)",
snprintf(devname, sizeof(devname), "%s(%u)",
tmpname, ifindex);
printf("ifindex %s ", devname);
}
@ -699,7 +699,7 @@ void netfilter_dump_plain(const struct bpf_link_info *info)
if (pfname)
printf("\n\t%s", pfname);
else
printf("\n\tpf: %d", pf);
printf("\n\tpf: %u", pf);
if (hookname)
printf(" %s", hookname);
@ -773,7 +773,7 @@ static void show_uprobe_multi_plain(struct bpf_link_info *info)
printf("func_cnt %u ", info->uprobe_multi.count);
if (info->uprobe_multi.pid)
printf("pid %d ", info->uprobe_multi.pid);
printf("pid %u ", info->uprobe_multi.pid);
printf("\n\t%-16s %-16s %-16s", "offset", "ref_ctr_offset", "cookies");
for (i = 0; i < info->uprobe_multi.count; i++) {

8
tools/bpf/bpftool/main.c
View File

@ -152,7 +152,7 @@ static int do_version(int argc, char **argv)
BPFTOOL_MINOR_VERSION, BPFTOOL_PATCH_VERSION);
#endif
jsonw_name(json_wtr, "libbpf_version");
jsonw_printf(json_wtr, "\"%d.%d\"",
jsonw_printf(json_wtr, "\"%u.%u\"",
libbpf_major_version(), libbpf_minor_version());
jsonw_name(json_wtr, "features");
@ -370,7 +370,7 @@ static int do_batch(int argc, char **argv)
while ((cp = strstr(buf, "\\\n")) != NULL) {
if (!fgets(contline, sizeof(contline), fp) ||
strlen(contline) == 0) {
p_err("missing continuation line on command %d",
p_err("missing continuation line on command %u",
lines);
err = -1;
goto err_close;
@ -381,7 +381,7 @@ static int do_batch(int argc, char **argv)
*cp = '\0';
if (strlen(buf) + strlen(contline) + 1 > sizeof(buf)) {
p_err("command %d is too long", lines);
p_err("command %u is too long", lines);
err = -1;
goto err_close;
}
@ -423,7 +423,7 @@ static int do_batch(int argc, char **argv)
err = -1;
} else {
if (!json_output)
printf("processed %d commands\n", lines);
printf("processed %u commands\n", lines);
}
err_close:
if (fp != stdin)

14
tools/bpf/bpftool/map.c
View File

@ -285,7 +285,7 @@ static void print_entry_plain(struct bpf_map_info *info, unsigned char *key,
}
if (info->value_size) {
for (i = 0; i < n; i++) {
printf("value (CPU %02d):%c",
printf("value (CPU %02u):%c",
i, info->value_size > 16 ? '\n' : ' ');
fprint_hex(stdout, value + i * step,
info->value_size, " ");
@ -316,7 +316,7 @@ static char **parse_bytes(char **argv, const char *name, unsigned char *val,
}
if (i != n) {
p_err("%s expected %d bytes got %d", name, n, i);
p_err("%s expected %u bytes got %u", name, n, i);
return NULL;
}
@ -462,7 +462,7 @@ static void show_map_header_json(struct bpf_map_info *info, json_writer_t *wtr)
jsonw_string_field(wtr, "name", info->name);
jsonw_name(wtr, "flags");
jsonw_printf(wtr, "%d", info->map_flags);
jsonw_printf(wtr, "%u", info->map_flags);
}
static int show_map_close_json(int fd, struct bpf_map_info *info)
@ -588,7 +588,7 @@ static int show_map_close_plain(int fd, struct bpf_map_info *info)
if (prog_type_str)
printf("owner_prog_type %s ", prog_type_str);
else
printf("owner_prog_type %d ", prog_type);
printf("owner_prog_type %u ", prog_type);
}
if (owner_jited)
printf("owner%s jited",
@ -615,7 +615,7 @@ static int show_map_close_plain(int fd, struct bpf_map_info *info)
printf("\n\t");
if (info->btf_id)
printf("btf_id %d", info->btf_id);
printf("btf_id %u", info->btf_id);
if (frozen)
printf("%sfrozen", info->btf_id ? " " : "");
@ -1270,6 +1270,10 @@ static int do_create(int argc, char **argv)
} else if (is_prefix(*argv, "name")) {
NEXT_ARG();
map_name = GET_ARG();
if (strlen(map_name) > BPF_OBJ_NAME_LEN - 1) {
p_info("Warning: map name is longer than %u characters, it will be truncated.",
BPF_OBJ_NAME_LEN - 1);
}
} else if (is_prefix(*argv, "key")) {
if (parse_u32_arg(&argc, &argv, &key_size,
"key size"))

6
tools/bpf/bpftool/map_perf_ring.c
View File

@ -91,15 +91,15 @@ print_bpf_output(void *private_data, int cpu, struct perf_event_header *event)
jsonw_end_object(json_wtr);
} else {
if (e->header.type == PERF_RECORD_SAMPLE) {
printf("== @%lld.%09lld CPU: %d index: %d =====\n",
printf("== @%llu.%09llu CPU: %d index: %d =====\n",
e->time / 1000000000ULL, e->time % 1000000000ULL,
cpu, idx);
fprint_hex(stdout, e->data, e->size, " ");
printf("\n");
} else if (e->header.type == PERF_RECORD_LOST) {
printf("lost %lld events\n", lost->lost);
printf("lost %llu events\n", lost->lost);
} else {
printf("unknown event type=%d size=%d\n",
printf("unknown event type=%u size=%u\n",
e->header.type, e->header.size);
}
}

4
tools/bpf/bpftool/net.c
View File

@ -476,7 +476,7 @@ static void __show_dev_tc_bpf(const struct ip_devname_ifindex *dev,
for (i = 0; i < optq.count; i++) {
NET_START_OBJECT;
NET_DUMP_STR("devname", "%s", dev->devname);
NET_DUMP_UINT("ifindex", "(%u)", dev->ifindex);
NET_DUMP_UINT("ifindex", "(%u)", (unsigned int)dev->ifindex);
NET_DUMP_STR("kind", " %s", attach_loc_strings[loc]);
ret = __show_dev_tc_bpf_name(prog_ids[i], prog_name,
sizeof(prog_name));
@ -831,7 +831,7 @@ static void show_link_netfilter(void)
if (err) {
if (errno == ENOENT)
break;
p_err("can't get next link: %s (id %d)", strerror(errno), id);
p_err("can't get next link: %s (id %u)", strerror(errno), id);
break;
}

6
tools/bpf/bpftool/netlink_dumper.c
View File

@ -45,7 +45,7 @@ static int do_xdp_dump_one(struct nlattr *attr, unsigned int ifindex,
NET_START_OBJECT;
if (name)
NET_DUMP_STR("devname", "%s", name);
NET_DUMP_UINT("ifindex", "(%d)", ifindex);
NET_DUMP_UINT("ifindex", "(%u)", ifindex);
if (mode == XDP_ATTACHED_MULTI) {
if (json_output) {
@ -74,7 +74,7 @@ int do_xdp_dump(struct ifinfomsg *ifinfo, struct nlattr **tb)
if (!tb[IFLA_XDP])
return 0;
return do_xdp_dump_one(tb[IFLA_XDP], ifinfo->ifi_index,
return do_xdp_dump_one(tb[IFLA_XDP], (unsigned int)ifinfo->ifi_index,
libbpf_nla_getattr_str(tb[IFLA_IFNAME]));
}
@ -168,7 +168,7 @@ int do_filter_dump(struct tcmsg *info, struct nlattr **tb, const char *kind,
NET_START_OBJECT;
if (devname[0] != '\0')
NET_DUMP_STR("devname", "%s", devname);
NET_DUMP_UINT("ifindex", "(%u)", ifindex);
NET_DUMP_UINT("ifindex", "(%u)", (unsigned int)ifindex);
NET_DUMP_STR("kind", " %s", kind);
ret = do_bpf_filter_dump(tb[TCA_OPTIONS]);
NET_END_OBJECT_FINAL;

13
tools/bpf/bpftool/prog.c
View File

@ -521,10 +521,10 @@ static void print_prog_header_plain(struct bpf_prog_info *info, int fd)
print_dev_plain(info->ifindex, info->netns_dev, info->netns_ino);
printf("%s", info->gpl_compatible ? " gpl" : "");
if (info->run_time_ns)
printf(" run_time_ns %lld run_cnt %lld",
printf(" run_time_ns %llu run_cnt %llu",
info->run_time_ns, info->run_cnt);
if (info->recursion_misses)
printf(" recursion_misses %lld", info->recursion_misses);
printf(" recursion_misses %llu", info->recursion_misses);
printf("\n");
}
@ -569,7 +569,7 @@ static void print_prog_plain(struct bpf_prog_info *info, int fd, bool orphaned)
}
if (info->btf_id)
printf("\n\tbtf_id %d", info->btf_id);
printf("\n\tbtf_id %u", info->btf_id);
emit_obj_refs_plain(refs_table, info->id, "\n\tpids ");
@ -1164,7 +1164,7 @@ static int get_run_data(const char *fname, void **data_ptr, unsigned int *size)
}
if (nb_read > buf_size - block_size) {
if (buf_size == UINT32_MAX) {
p_err("data_in/ctx_in is too long (max: %d)",
p_err("data_in/ctx_in is too long (max: %u)",
UINT32_MAX);
goto err_free;
}
@ -1928,6 +1928,7 @@ static int do_loader(int argc, char **argv)
obj = bpf_object__open_file(file, &open_opts);
if (!obj) {
err = -1;
p_err("failed to open object file");
goto err_close_obj;
}
@ -2251,7 +2252,7 @@ static char *profile_target_name(int tgt_fd)
t = btf__type_by_id(btf, func_info.type_id);
if (!t) {
p_err("btf %d doesn't have type %d",
p_err("btf %u doesn't have type %u",
info.btf_id, func_info.type_id);
goto out;
}
@ -2329,7 +2330,7 @@ static int profile_open_perf_events(struct profiler_bpf *obj)
continue;
for (cpu = 0; cpu < obj->rodata->num_cpu; cpu++) {
if (profile_open_perf_event(m, cpu, map_fd)) {
p_err("failed to create event %s on cpu %d",
p_err("failed to create event %s on cpu %u",
metrics[m].name, cpu);
return -1;
}

2
tools/bpf/bpftool/tracelog.c
View File

@ -78,7 +78,7 @@ static bool get_tracefs_pipe(char *mnt)
return false;
/* Allow room for NULL terminating byte and pipe file name */
snprintf(format, sizeof(format), "%%*s %%%zds %%99s %%*s %%*d %%*d\\n",
snprintf(format, sizeof(format), "%%*s %%%zus %%99s %%*s %%*d %%*d\\n",
PATH_MAX - strlen(pipe_name) - 1);
while (fscanf(fp, format, mnt, type) == 2)
if (strcmp(type, fstype) == 0) {

6
tools/bpf/bpftool/xlated_dumper.c
View File

@ -199,13 +199,13 @@ static const char *print_imm(void *private_data,
if (insn->src_reg == BPF_PSEUDO_MAP_FD)
snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
"map[id:%u]", insn->imm);
"map[id:%d]", insn->imm);
else if (insn->src_reg == BPF_PSEUDO_MAP_VALUE)
snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
"map[id:%u][0]+%u", insn->imm, (insn + 1)->imm);
"map[id:%d][0]+%d", insn->imm, (insn + 1)->imm);
else if (insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE)
snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
"map[idx:%u]+%u", insn->imm, (insn + 1)->imm);
"map[idx:%d]+%d", insn->imm, (insn + 1)->imm);
else if (insn->src_reg == BPF_PSEUDO_FUNC)
snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
"subprog[%+d]", insn->imm);

3
tools/bpf/runqslower/Makefile
View File

@ -6,6 +6,7 @@ OUTPUT ?= $(abspath .output)/
BPFTOOL_OUTPUT := $(OUTPUT)bpftool/
DEFAULT_BPFTOOL := $(BPFTOOL_OUTPUT)bootstrap/bpftool
BPFTOOL ?= $(DEFAULT_BPFTOOL)
BPF_TARGET_ENDIAN ?= --target=bpf
LIBBPF_SRC := $(abspath ../../lib/bpf)
BPFOBJ_OUTPUT := $(OUTPUT)libbpf/
BPFOBJ := $(BPFOBJ_OUTPUT)libbpf.a
@ -60,7 +61,7 @@ $(OUTPUT)/%.skel.h: $(OUTPUT)/%.bpf.o | $(BPFTOOL)
$(QUIET_GEN)$(BPFTOOL) gen skeleton $< > $@
$(OUTPUT)/%.bpf.o: %.bpf.c $(BPFOBJ) | $(OUTPUT)
$(QUIET_GEN)$(CLANG) -g -O2 --target=bpf $(INCLUDES) \
$(QUIET_GEN)$(CLANG) -g -O2 $(BPF_TARGET_ENDIAN) $(INCLUDES) \
-c $(filter %.c,$^) -o $@ && \
$(LLVM_STRIP) -g $@

10
tools/include/uapi/linux/bpf.h
View File

@ -51,6 +51,9 @@
#define BPF_XCHG (0xe0 | BPF_FETCH) /* atomic exchange */
#define BPF_CMPXCHG (0xf0 | BPF_FETCH) /* atomic compare-and-write */
#define BPF_LOAD_ACQ 0x100 /* load-acquire */
#define BPF_STORE_REL 0x110 /* store-release */
enum bpf_cond_pseudo_jmp {
BPF_MAY_GOTO = 0,
};
@ -1207,6 +1210,7 @@ enum bpf_perf_event_type {
#define BPF_F_BEFORE (1U << 3)
#define BPF_F_AFTER (1U << 4)
#define BPF_F_ID (1U << 5)
#define BPF_F_PREORDER (1U << 6)
#define BPF_F_LINK BPF_F_LINK /* 1 << 13 */
/* If BPF_F_STRICT_ALIGNMENT is used in BPF_PROG_LOAD command, the
@ -1648,6 +1652,7 @@ union bpf_attr {
};
__u32 next_id;
__u32 open_flags;
__s32 fd_by_id_token_fd;
};
struct { /* anonymous struct used by BPF_OBJ_GET_INFO_BY_FD */
@ -6019,7 +6024,10 @@ union bpf_attr {
FN(user_ringbuf_drain, 209, ##ctx) \
FN(cgrp_storage_get, 210, ##ctx) \
FN(cgrp_storage_delete, 211, ##ctx) \
/* */
/* This helper list is effectively frozen. If you are trying to \
* add a new helper, you should add a kfunc instead which has \
* less stability guarantees. See Documentation/bpf/kfuncs.rst \
*/
/* backwards-compatibility macros for users of __BPF_FUNC_MAPPER that don't
* know or care about integer value that is now passed as second argument

3
tools/include/uapi/linux/btf.h
View File

@ -36,7 +36,8 @@ struct btf_type {
* bits 24-28: kind (e.g. int, ptr, array...etc)
* bits 29-30: unused
* bit 31: kind_flag, currently used by
* struct, union, enum, fwd and enum64
* struct, union, enum, fwd, enum64,
* decl_tag and type_tag
*/
__u32 info;
/* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC and ENUM64.

3
tools/lib/bpf/bpf.c
View File

@ -1097,7 +1097,7 @@ int bpf_map_get_fd_by_id(__u32 id)
int bpf_btf_get_fd_by_id_opts(__u32 id,
const struct bpf_get_fd_by_id_opts *opts)
{
const size_t attr_sz = offsetofend(union bpf_attr, open_flags);
const size_t attr_sz = offsetofend(union bpf_attr, fd_by_id_token_fd);
union bpf_attr attr;
int fd;
@ -1107,6 +1107,7 @@ int bpf_btf_get_fd_by_id_opts(__u32 id,
memset(&attr, 0, attr_sz);
attr.btf_id = id;
attr.open_flags = OPTS_GET(opts, open_flags, 0);
attr.fd_by_id_token_fd = OPTS_GET(opts, token_fd, 0);
fd = sys_bpf_fd(BPF_BTF_GET_FD_BY_ID, &attr, attr_sz);
return libbpf_err_errno(fd);

3
tools/lib/bpf/bpf.h
View File

@ -487,9 +487,10 @@ LIBBPF_API int bpf_link_get_next_id(__u32 start_id, __u32 *next_id);
struct bpf_get_fd_by_id_opts {
size_t sz; /* size of this struct for forward/backward compatibility */
__u32 open_flags; /* permissions requested for the operation on fd */
__u32 token_fd;
size_t :0;
};
#define bpf_get_fd_by_id_opts__last_field open_flags
#define bpf_get_fd_by_id_opts__last_field token_fd
LIBBPF_API int bpf_prog_get_fd_by_id(__u32 id);
LIBBPF_API int bpf_prog_get_fd_by_id_opts(__u32 id,

105
tools/lib/bpf/btf.c
View File

@ -1619,12 +1619,18 @@ exit_free:
return btf;
}
struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf)
struct btf *btf_load_from_kernel(__u32 id, struct btf *base_btf, int token_fd)
{
struct btf *btf;
int btf_fd;
LIBBPF_OPTS(bpf_get_fd_by_id_opts, opts);
btf_fd = bpf_btf_get_fd_by_id(id);
if (token_fd) {
opts.open_flags |= BPF_F_TOKEN_FD;
opts.token_fd = token_fd;
}
btf_fd = bpf_btf_get_fd_by_id_opts(id, &opts);
if (btf_fd < 0)
return libbpf_err_ptr(-errno);
@ -1634,6 +1640,11 @@ struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf)
return libbpf_ptr(btf);
}
struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf)
{
return btf_load_from_kernel(id, base_btf, 0);
}
struct btf *btf__load_from_kernel_by_id(__u32 id)
{
return btf__load_from_kernel_by_id_split(id, NULL);
@ -2090,7 +2101,7 @@ static int validate_type_id(int id)
}
/* generic append function for PTR, TYPEDEF, CONST/VOLATILE/RESTRICT */
static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref_type_id)
static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref_type_id, int kflag)
{
struct btf_type *t;
int sz, name_off = 0;
@ -2113,7 +2124,7 @@ static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref
}
t->name_off = name_off;
t->info = btf_type_info(kind, 0, 0);
t->info = btf_type_info(kind, 0, kflag);
t->type = ref_type_id;
return btf_commit_type(btf, sz);
@ -2128,7 +2139,7 @@ static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref
*/
int btf__add_ptr(struct btf *btf, int ref_type_id)
{
return btf_add_ref_kind(btf, BTF_KIND_PTR, NULL, ref_type_id);
return btf_add_ref_kind(btf, BTF_KIND_PTR, NULL, ref_type_id, 0);
}
/*
@ -2506,7 +2517,7 @@ int btf__add_fwd(struct btf *btf, const char *name, enum btf_fwd_kind fwd_kind)
struct btf_type *t;
int id;
id = btf_add_ref_kind(btf, BTF_KIND_FWD, name, 0);
id = btf_add_ref_kind(btf, BTF_KIND_FWD, name, 0, 0);
if (id <= 0)
return id;
t = btf_type_by_id(btf, id);
@ -2536,7 +2547,7 @@ int btf__add_typedef(struct btf *btf, const char *name, int ref_type_id)
if (!name || !name[0])
return libbpf_err(-EINVAL);
return btf_add_ref_kind(btf, BTF_KIND_TYPEDEF, name, ref_type_id);
return btf_add_ref_kind(btf, BTF_KIND_TYPEDEF, name, ref_type_id, 0);
}
/*
@ -2548,7 +2559,7 @@ int btf__add_typedef(struct btf *btf, const char *name, int ref_type_id)
*/
int btf__add_volatile(struct btf *btf, int ref_type_id)
{
return btf_add_ref_kind(btf, BTF_KIND_VOLATILE, NULL, ref_type_id);
return btf_add_ref_kind(btf, BTF_KIND_VOLATILE, NULL, ref_type_id, 0);
}
/*
@ -2560,7 +2571,7 @@ int btf__add_volatile(struct btf *btf, int ref_type_id)
*/
int btf__add_const(struct btf *btf, int ref_type_id)
{
return btf_add_ref_kind(btf, BTF_KIND_CONST, NULL, ref_type_id);
return btf_add_ref_kind(btf, BTF_KIND_CONST, NULL, ref_type_id, 0);
}
/*
@ -2572,7 +2583,7 @@ int btf__add_const(struct btf *btf, int ref_type_id)
*/
int btf__add_restrict(struct btf *btf, int ref_type_id)
{
return btf_add_ref_kind(btf, BTF_KIND_RESTRICT, NULL, ref_type_id);
return btf_add_ref_kind(btf, BTF_KIND_RESTRICT, NULL, ref_type_id, 0);
}
/*
@ -2588,7 +2599,24 @@ int btf__add_type_tag(struct btf *btf, const char *value, int ref_type_id)
if (!value || !value[0])
return libbpf_err(-EINVAL);
return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id);
return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id, 0);
}
/*
* Append new BTF_KIND_TYPE_TAG type with:
* - *value*, non-empty/non-NULL tag value;
* - *ref_type_id* - referenced type ID, it might not exist yet;
* Set info->kflag to 1, indicating this tag is an __attribute__
* Returns:
* - >0, type ID of newly added BTF type;
* - <0, on error.
*/
int btf__add_type_attr(struct btf *btf, const char *value, int ref_type_id)
{
if (!value || !value[0])
return libbpf_err(-EINVAL);
return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id, 1);
}
/*
@ -2610,7 +2638,7 @@ int btf__add_func(struct btf *btf, const char *name,
linkage != BTF_FUNC_EXTERN)
return libbpf_err(-EINVAL);
id = btf_add_ref_kind(btf, BTF_KIND_FUNC, name, proto_type_id);
id = btf_add_ref_kind(btf, BTF_KIND_FUNC, name, proto_type_id, 0);
if (id > 0) {
struct btf_type *t = btf_type_by_id(btf, id);
@ -2845,18 +2873,8 @@ int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __
return 0;
}
/*
* Append new BTF_KIND_DECL_TAG type with:
* - *value* - non-empty/non-NULL string;
* - *ref_type_id* - referenced type ID, it might not exist yet;
* - *component_idx* - -1 for tagging reference type, otherwise struct/union
* member or function argument index;
* Returns:
* - >0, type ID of newly added BTF type;
* - <0, on error.
*/
int btf__add_decl_tag(struct btf *btf, const char *value, int ref_type_id,
int component_idx)
static int btf_add_decl_tag(struct btf *btf, const char *value, int ref_type_id,
int component_idx, int kflag)
{
struct btf_type *t;
int sz, value_off;
@ -2880,13 +2898,46 @@ int btf__add_decl_tag(struct btf *btf, const char *value, int ref_type_id,
return value_off;
t->name_off = value_off;
t->info = btf_type_info(BTF_KIND_DECL_TAG, 0, false);
t->info = btf_type_info(BTF_KIND_DECL_TAG, 0, kflag);
t->type = ref_type_id;
btf_decl_tag(t)->component_idx = component_idx;
return btf_commit_type(btf, sz);
}
/*
* Append new BTF_KIND_DECL_TAG type with:
* - *value* - non-empty/non-NULL string;
* - *ref_type_id* - referenced type ID, it might not exist yet;
* - *component_idx* - -1 for tagging reference type, otherwise struct/union
* member or function argument index;
* Returns:
* - >0, type ID of newly added BTF type;
* - <0, on error.
*/
int btf__add_decl_tag(struct btf *btf, const char *value, int ref_type_id,
int component_idx)
{
return btf_add_decl_tag(btf, value, ref_type_id, component_idx, 0);
}
/*
* Append new BTF_KIND_DECL_TAG type with:
* - *value* - non-empty/non-NULL string;
* - *ref_type_id* - referenced type ID, it might not exist yet;
* - *component_idx* - -1 for tagging reference type, otherwise struct/union
* member or function argument index;
* Set info->kflag to 1, indicating this tag is an __attribute__
* Returns:
* - >0, type ID of newly added BTF type;
* - <0, on error.
*/
int btf__add_decl_attr(struct btf *btf, const char *value, int ref_type_id,
int component_idx)
{
return btf_add_decl_tag(btf, value, ref_type_id, component_idx, 1);
}
struct btf_ext_sec_info_param {
__u32 off;
__u32 len;
@ -3015,8 +3066,6 @@ static int btf_ext_parse_info(struct btf_ext *btf_ext, bool is_native)
.desc = "line_info",
};
struct btf_ext_sec_info_param core_relo = {
.off = btf_ext->hdr->core_relo_off,
.len = btf_ext->hdr->core_relo_len,
.min_rec_size = sizeof(struct bpf_core_relo),
.ext_info = &btf_ext->core_relo_info,
.desc = "core_relo",
@ -3034,6 +3083,8 @@ static int btf_ext_parse_info(struct btf_ext *btf_ext, bool is_native)
if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
return 0; /* skip core relos parsing */
core_relo.off = btf_ext->hdr->core_relo_off;
core_relo.len = btf_ext->hdr->core_relo_len;
err = btf_ext_parse_sec_info(btf_ext, &core_relo, is_native);
if (err)
return err;

3
tools/lib/bpf/btf.h
View File

@ -227,6 +227,7 @@ LIBBPF_API int btf__add_volatile(struct btf *btf, int ref_type_id);
LIBBPF_API int btf__add_const(struct btf *btf, int ref_type_id);
LIBBPF_API int btf__add_restrict(struct btf *btf, int ref_type_id);
LIBBPF_API int btf__add_type_tag(struct btf *btf, const char *value, int ref_type_id);
LIBBPF_API int btf__add_type_attr(struct btf *btf, const char *value, int ref_type_id);
/* func and func_proto construction APIs */
LIBBPF_API int btf__add_func(struct btf *btf, const char *name,
@ -243,6 +244,8 @@ LIBBPF_API int btf__add_datasec_var_info(struct btf *btf, int var_type_id,
/* tag construction API */
LIBBPF_API int btf__add_decl_tag(struct btf *btf, const char *value, int ref_type_id,
int component_idx);
LIBBPF_API int btf__add_decl_attr(struct btf *btf, const char *value, int ref_type_id,
int component_idx);
struct btf_dedup_opts {
size_t sz;

5
tools/lib/bpf/btf_dump.c
View File

@ -1494,7 +1494,10 @@ static void btf_dump_emit_type_chain(struct btf_dump *d,
case BTF_KIND_TYPE_TAG:
btf_dump_emit_mods(d, decls);
name = btf_name_of(d, t->name_off);
btf_dump_printf(d, " __attribute__((btf_type_tag(\"%s\")))", name);
if (btf_kflag(t))
btf_dump_printf(d, " __attribute__((%s))", name);
else
btf_dump_printf(d, " __attribute__((btf_type_tag(\"%s\")))", name);
break;
case BTF_KIND_ARRAY: {
const struct btf_array *a = btf_array(t);

237
tools/lib/bpf/libbpf.c
View File

@ -670,11 +670,18 @@ struct elf_state {
struct usdt_manager;
enum bpf_object_state {
OBJ_OPEN,
OBJ_PREPARED,
OBJ_LOADED,
};
struct bpf_object {
char name[BPF_OBJ_NAME_LEN];
char license[64];
__u32 kern_version;
enum bpf_object_state state;
struct bpf_program *programs;
size_t nr_programs;
struct bpf_map *maps;
@ -686,7 +693,6 @@ struct bpf_object {
int nr_extern;
int kconfig_map_idx;
bool loaded;
bool has_subcalls;
bool has_rodata;
@ -1511,7 +1517,7 @@ static struct bpf_object *bpf_object__new(const char *path,
obj->kconfig_map_idx = -1;
obj->kern_version = get_kernel_version();
obj->loaded = false;
obj->state = OBJ_OPEN;
return obj;
}
@ -2106,7 +2112,7 @@ static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,
}
len = strlen(value);
if (value[len - 1] != '"') {
if (len < 2 || value[len - 1] != '"') {
pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
ext->name, value);
return -EINVAL;
@ -4845,6 +4851,11 @@ static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info)
return 0;
}
static bool map_is_created(const struct bpf_map *map)
{
return map->obj->state >= OBJ_PREPARED || map->reused;
}
bool bpf_map__autocreate(const struct bpf_map *map)
{
return map->autocreate;
@ -4852,7 +4863,7 @@ bool bpf_map__autocreate(const struct bpf_map *map)
int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate)
{
if (map->obj->loaded)
if (map_is_created(map))
return libbpf_err(-EBUSY);
map->autocreate = autocreate;
@ -4946,7 +4957,7 @@ struct bpf_map *bpf_map__inner_map(struct bpf_map *map)
int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
{
if (map->obj->loaded)
if (map_is_created(map))
return libbpf_err(-EBUSY);
map->def.max_entries = max_entries;
@ -5191,11 +5202,6 @@ bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
static void bpf_map__destroy(struct bpf_map *map);
static bool map_is_created(const struct bpf_map *map)
{
return map->obj->loaded || map->reused;
}
static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner)
{
LIBBPF_OPTS(bpf_map_create_opts, create_attr);
@ -7895,13 +7901,6 @@ bpf_object__load_progs(struct bpf_object *obj, int log_level)
size_t i;
int err;
for (i = 0; i < obj->nr_programs; i++) {
prog = &obj->programs[i];
err = bpf_object__sanitize_prog(obj, prog);
if (err)
return err;
}
for (i = 0; i < obj->nr_programs; i++) {
prog = &obj->programs[i];
if (prog_is_subprog(obj, prog))
@ -7927,6 +7926,21 @@ bpf_object__load_progs(struct bpf_object *obj, int log_level)
return 0;
}
static int bpf_object_prepare_progs(struct bpf_object *obj)
{
struct bpf_program *prog;
size_t i;
int err;
for (i = 0; i < obj->nr_programs; i++) {
prog = &obj->programs[i];
err = bpf_object__sanitize_prog(obj, prog);
if (err)
return err;
}
return 0;
}
static const struct bpf_sec_def *find_sec_def(const char *sec_name);
static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object_open_opts *opts)
@ -8543,14 +8557,77 @@ static int bpf_object_prepare_struct_ops(struct bpf_object *obj)
return 0;
}
static void bpf_object_unpin(struct bpf_object *obj)
{
int i;
/* unpin any maps that were auto-pinned during load */
for (i = 0; i < obj->nr_maps; i++)
if (obj->maps[i].pinned && !obj->maps[i].reused)
bpf_map__unpin(&obj->maps[i], NULL);
}
static void bpf_object_post_load_cleanup(struct bpf_object *obj)
{
int i;
/* clean up fd_array */
zfree(&obj->fd_array);
/* clean up module BTFs */
for (i = 0; i < obj->btf_module_cnt; i++) {
close(obj->btf_modules[i].fd);
btf__free(obj->btf_modules[i].btf);
free(obj->btf_modules[i].name);
}
obj->btf_module_cnt = 0;
zfree(&obj->btf_modules);
/* clean up vmlinux BTF */
btf__free(obj->btf_vmlinux);
obj->btf_vmlinux = NULL;
}
static int bpf_object_prepare(struct bpf_object *obj, const char *target_btf_path)
{
int err;
if (obj->state >= OBJ_PREPARED) {
pr_warn("object '%s': prepare loading can't be attempted twice\n", obj->name);
return -EINVAL;
}
err = bpf_object_prepare_token(obj);
err = err ? : bpf_object__probe_loading(obj);
err = err ? : bpf_object__load_vmlinux_btf(obj, false);
err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
err = err ? : bpf_object__sanitize_maps(obj);
err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
err = err ? : bpf_object_adjust_struct_ops_autoload(obj);
err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
err = err ? : bpf_object__sanitize_and_load_btf(obj);
err = err ? : bpf_object__create_maps(obj);
err = err ? : bpf_object_prepare_progs(obj);
if (err) {
bpf_object_unpin(obj);
bpf_object_unload(obj);
obj->state = OBJ_LOADED;
return err;
}
obj->state = OBJ_PREPARED;
return 0;
}
static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const char *target_btf_path)
{
int err, i;
int err;
if (!obj)
return libbpf_err(-EINVAL);
if (obj->loaded) {
if (obj->state >= OBJ_LOADED) {
pr_warn("object '%s': load can't be attempted twice\n", obj->name);
return libbpf_err(-EINVAL);
}
@ -8565,17 +8642,12 @@ static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const ch
return libbpf_err(-LIBBPF_ERRNO__ENDIAN);
}
err = bpf_object_prepare_token(obj);
err = err ? : bpf_object__probe_loading(obj);
err = err ? : bpf_object__load_vmlinux_btf(obj, false);
err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
err = err ? : bpf_object__sanitize_maps(obj);
err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
err = err ? : bpf_object_adjust_struct_ops_autoload(obj);
err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
err = err ? : bpf_object__sanitize_and_load_btf(obj);
err = err ? : bpf_object__create_maps(obj);
err = err ? : bpf_object__load_progs(obj, extra_log_level);
if (obj->state < OBJ_PREPARED) {
err = bpf_object_prepare(obj, target_btf_path);
if (err)
return libbpf_err(err);
}
err = bpf_object__load_progs(obj, extra_log_level);
err = err ? : bpf_object_init_prog_arrays(obj);
err = err ? : bpf_object_prepare_struct_ops(obj);
@ -8587,36 +8659,22 @@ static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const ch
err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps);
}
/* clean up fd_array */
zfree(&obj->fd_array);
bpf_object_post_load_cleanup(obj);
obj->state = OBJ_LOADED; /* doesn't matter if successfully or not */
/* clean up module BTFs */
for (i = 0; i < obj->btf_module_cnt; i++) {
close(obj->btf_modules[i].fd);
btf__free(obj->btf_modules[i].btf);
free(obj->btf_modules[i].name);
if (err) {
bpf_object_unpin(obj);
bpf_object_unload(obj);
pr_warn("failed to load object '%s'\n", obj->path);
return libbpf_err(err);
}
free(obj->btf_modules);
/* clean up vmlinux BTF */
btf__free(obj->btf_vmlinux);
obj->btf_vmlinux = NULL;
obj->loaded = true; /* doesn't matter if successfully or not */
if (err)
goto out;
return 0;
out:
/* unpin any maps that were auto-pinned during load */
for (i = 0; i < obj->nr_maps; i++)
if (obj->maps[i].pinned && !obj->maps[i].reused)
bpf_map__unpin(&obj->maps[i], NULL);
}
bpf_object_unload(obj);
pr_warn("failed to load object '%s'\n", obj->path);
return libbpf_err(err);
int bpf_object__prepare(struct bpf_object *obj)
{
return libbpf_err(bpf_object_prepare(obj, NULL));
}
int bpf_object__load(struct bpf_object *obj)
@ -8866,7 +8924,7 @@ int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
if (!obj)
return libbpf_err(-ENOENT);
if (!obj->loaded) {
if (obj->state < OBJ_PREPARED) {
pr_warn("object not yet loaded; load it first\n");
return libbpf_err(-ENOENT);
}
@ -8945,7 +9003,7 @@ int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
if (!obj)
return libbpf_err(-ENOENT);
if (!obj->loaded) {
if (obj->state < OBJ_LOADED) {
pr_warn("object not yet loaded; load it first\n");
return libbpf_err(-ENOENT);
}
@ -9064,6 +9122,13 @@ void bpf_object__close(struct bpf_object *obj)
if (IS_ERR_OR_NULL(obj))
return;
/*
* if user called bpf_object__prepare() without ever getting to
* bpf_object__load(), we need to clean up stuff that is normally
* cleaned up at the end of loading step
*/
bpf_object_post_load_cleanup(obj);
usdt_manager_free(obj->usdt_man);
obj->usdt_man = NULL;
@ -9132,7 +9197,7 @@ int bpf_object__btf_fd(const struct bpf_object *obj)
int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version)
{
if (obj->loaded)
if (obj->state >= OBJ_LOADED)
return libbpf_err(-EINVAL);
obj->kern_version = kern_version;
@ -9145,12 +9210,12 @@ int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts)
struct bpf_gen *gen;
if (!opts)
return -EFAULT;
return libbpf_err(-EFAULT);
if (!OPTS_VALID(opts, gen_loader_opts))
return -EINVAL;
return libbpf_err(-EINVAL);
gen = calloc(sizeof(*gen), 1);
if (!gen)
return -ENOMEM;
return libbpf_err(-ENOMEM);
gen->opts = opts;
gen->swapped_endian = !is_native_endianness(obj);
obj->gen_loader = gen;
@ -9229,7 +9294,7 @@ bool bpf_program__autoload(const struct bpf_program *prog)
int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
{
if (prog->obj->loaded)
if (prog->obj->state >= OBJ_LOADED)
return libbpf_err(-EINVAL);
prog->autoload = autoload;
@ -9261,14 +9326,14 @@ int bpf_program__set_insns(struct bpf_program *prog,
{
struct bpf_insn *insns;
if (prog->obj->loaded)
return -EBUSY;
if (prog->obj->state >= OBJ_LOADED)
return libbpf_err(-EBUSY);
insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
/* NULL is a valid return from reallocarray if the new count is zero */
if (!insns && new_insn_cnt) {
pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
return -ENOMEM;
return libbpf_err(-ENOMEM);
}
memcpy(insns, new_insns, new_insn_cnt * sizeof(*insns));
@ -9304,7 +9369,7 @@ static int last_custom_sec_def_handler_id;
int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
{
if (prog->obj->loaded)
if (prog->obj->state >= OBJ_LOADED)
return libbpf_err(-EBUSY);
/* if type is not changed, do nothing */
@ -9335,7 +9400,7 @@ enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program
int bpf_program__set_expected_attach_type(struct bpf_program *prog,
enum bpf_attach_type type)
{
if (prog->obj->loaded)
if (prog->obj->state >= OBJ_LOADED)
return libbpf_err(-EBUSY);
prog->expected_attach_type = type;
@ -9349,7 +9414,7 @@ __u32 bpf_program__flags(const struct bpf_program *prog)
int bpf_program__set_flags(struct bpf_program *prog, __u32 flags)
{
if (prog->obj->loaded)
if (prog->obj->state >= OBJ_LOADED)
return libbpf_err(-EBUSY);
prog->prog_flags = flags;
@ -9363,7 +9428,7 @@ __u32 bpf_program__log_level(const struct bpf_program *prog)
int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level)
{
if (prog->obj->loaded)
if (prog->obj->state >= OBJ_LOADED)
return libbpf_err(-EBUSY);
prog->log_level = log_level;
@ -9379,11 +9444,11 @@ const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_siz
int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size)
{
if (log_size && !log_buf)
return -EINVAL;
return libbpf_err(-EINVAL);
if (prog->log_size > UINT_MAX)
return -EINVAL;
if (prog->obj->loaded)
return -EBUSY;
return libbpf_err(-EINVAL);
if (prog->obj->state >= OBJ_LOADED)
return libbpf_err(-EBUSY);
prog->log_buf = log_buf;
prog->log_size = log_size;
@ -9959,7 +10024,7 @@ int libbpf_find_vmlinux_btf_id(const char *name,
return libbpf_err(err);
}
static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd, int token_fd)
{
struct bpf_prog_info info;
__u32 info_len = sizeof(info);
@ -9979,7 +10044,7 @@ static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
pr_warn("The target program doesn't have BTF\n");
goto out;
}
btf = btf__load_from_kernel_by_id(info.btf_id);
btf = btf_load_from_kernel(info.btf_id, NULL, token_fd);
err = libbpf_get_error(btf);
if (err) {
pr_warn("Failed to get BTF %d of the program: %s\n", info.btf_id, errstr(err));
@ -10062,7 +10127,7 @@ static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attac
pr_warn("prog '%s': attach program FD is not set\n", prog->name);
return -EINVAL;
}
err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd, prog->obj->token_fd);
if (err < 0) {
pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %s\n",
prog->name, attach_prog_fd, attach_name, errstr(err));
@ -10299,7 +10364,7 @@ static int map_btf_datasec_resize(struct bpf_map *map, __u32 size)
int bpf_map__set_value_size(struct bpf_map *map, __u32 size)
{
if (map->obj->loaded || map->reused)
if (map_is_created(map))
return libbpf_err(-EBUSY);
if (map->mmaped) {
@ -10307,7 +10372,7 @@ int bpf_map__set_value_size(struct bpf_map *map, __u32 size)
int err;
if (map->def.type != BPF_MAP_TYPE_ARRAY)
return -EOPNOTSUPP;
return libbpf_err(-EOPNOTSUPP);
mmap_old_sz = bpf_map_mmap_sz(map);
mmap_new_sz = array_map_mmap_sz(size, map->def.max_entries);
@ -10315,7 +10380,7 @@ int bpf_map__set_value_size(struct bpf_map *map, __u32 size)
if (err) {
pr_warn("map '%s': failed to resize memory-mapped region: %s\n",
bpf_map__name(map), errstr(err));
return err;
return libbpf_err(err);
}
err = map_btf_datasec_resize(map, size);
if (err && err != -ENOENT) {
@ -10345,7 +10410,7 @@ int bpf_map__set_initial_value(struct bpf_map *map,
{
size_t actual_sz;
if (map->obj->loaded || map->reused)
if (map_is_created(map))
return libbpf_err(-EBUSY);
if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG)
@ -12858,7 +12923,7 @@ struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog,
if (target_fd) {
LIBBPF_OPTS(bpf_link_create_opts, target_opts);
btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd);
btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd, prog->obj->token_fd);
if (btf_id < 0)
return libbpf_err_ptr(btf_id);
@ -13070,17 +13135,17 @@ int bpf_link__update_map(struct bpf_link *link, const struct bpf_map *map)
int err;
if (!bpf_map__is_struct_ops(map))
return -EINVAL;
return libbpf_err(-EINVAL);
if (map->fd < 0) {
pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);
return -EINVAL;
return libbpf_err(-EINVAL);
}
st_ops_link = container_of(link, struct bpf_link_struct_ops, link);
/* Ensure the type of a link is correct */
if (st_ops_link->map_fd < 0)
return -EINVAL;
return libbpf_err(-EINVAL);
err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);
/* It can be EBUSY if the map has been used to create or
@ -13666,7 +13731,7 @@ int bpf_program__set_attach_target(struct bpf_program *prog,
if (!prog || attach_prog_fd < 0)
return libbpf_err(-EINVAL);
if (prog->obj->loaded)
if (prog->obj->state >= OBJ_LOADED)
return libbpf_err(-EINVAL);
if (attach_prog_fd && !attach_func_name) {
@ -13679,7 +13744,7 @@ int bpf_program__set_attach_target(struct bpf_program *prog,
if (attach_prog_fd) {
btf_id = libbpf_find_prog_btf_id(attach_func_name,
attach_prog_fd);
attach_prog_fd, prog->obj->token_fd);
if (btf_id < 0)
return libbpf_err(btf_id);
} else {

13
tools/lib/bpf/libbpf.h
View File

@ -241,6 +241,19 @@ LIBBPF_API struct bpf_object *
bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
const struct bpf_object_open_opts *opts);
/**
* @brief **bpf_object__prepare()** prepares BPF object for loading:
* performs ELF processing, relocations, prepares final state of BPF program
* instructions (accessible with bpf_program__insns()), creates and
* (potentially) pins maps. Leaves BPF object in the state ready for program
* loading.
* @param obj Pointer to a valid BPF object instance returned by
* **bpf_object__open*()** API
* @return 0, on success; negative error code, otherwise, error code is
* stored in errno
*/
int bpf_object__prepare(struct bpf_object *obj);
/**
* @brief **bpf_object__load()** loads BPF object into kernel.
* @param obj Pointer to a valid BPF object instance returned by

3
tools/lib/bpf/libbpf.map
View File

@ -436,4 +436,7 @@ LIBBPF_1.6.0 {
bpf_linker__add_buf;
bpf_linker__add_fd;
bpf_linker__new_fd;
bpf_object__prepare;
btf__add_decl_attr;
btf__add_type_attr;
} LIBBPF_1.5.0;

1
tools/lib/bpf/libbpf_internal.h
View File

@ -409,6 +409,7 @@ int libbpf__load_raw_btf(const char *raw_types, size_t types_len,
int btf_load_into_kernel(struct btf *btf,
char *log_buf, size_t log_sz, __u32 log_level,
int token_fd);
struct btf *btf_load_from_kernel(__u32 id, struct btf *base_btf, int token_fd);
struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf);
void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type,

2
tools/lib/bpf/linker.c
View File

@ -2163,7 +2163,7 @@ add_sym:
obj->sym_map[src_sym_idx] = dst_sym_idx;
if (sym_type == STT_SECTION && dst_sym) {
if (sym_type == STT_SECTION && dst_sec) {
dst_sec->sec_sym_idx = dst_sym_idx;
dst_sym->st_value = 0;
}

24
tools/lib/bpf/relo_core.c
View File

@ -683,7 +683,7 @@ static int bpf_core_calc_field_relo(const char *prog_name,
{
const struct bpf_core_accessor *acc;
const struct btf_type *t;
__u32 byte_off, byte_sz, bit_off, bit_sz, field_type_id;
__u32 byte_off, byte_sz, bit_off, bit_sz, field_type_id, elem_id;
const struct btf_member *m;
const struct btf_type *mt;
bool bitfield;
@ -706,8 +706,14 @@ static int bpf_core_calc_field_relo(const char *prog_name,
if (!acc->name) {
if (relo->kind == BPF_CORE_FIELD_BYTE_OFFSET) {
*val = spec->bit_offset / 8;
/* remember field size for load/store mem size */
sz = btf__resolve_size(spec->btf, acc->type_id);
/* remember field size for load/store mem size;
* note, for arrays we care about individual element
* sizes, not the overall array size
*/
t = skip_mods_and_typedefs(spec->btf, acc->type_id, &elem_id);
while (btf_is_array(t))
t = skip_mods_and_typedefs(spec->btf, btf_array(t)->type, &elem_id);
sz = btf__resolve_size(spec->btf, elem_id);
if (sz < 0)
return -EINVAL;
*field_sz = sz;
@ -767,7 +773,17 @@ static int bpf_core_calc_field_relo(const char *prog_name,
case BPF_CORE_FIELD_BYTE_OFFSET:
*val = byte_off;
if (!bitfield) {
*field_sz = byte_sz;
/* remember field size for load/store mem size;
* note, for arrays we care about individual element
* sizes, not the overall array size
*/
t = skip_mods_and_typedefs(spec->btf, field_type_id, &elem_id);
while (btf_is_array(t))
t = skip_mods_and_typedefs(spec->btf, btf_array(t)->type, &elem_id);
sz = btf__resolve_size(spec->btf, elem_id);
if (sz < 0)
return -EINVAL;
*field_sz = sz;
*type_id = field_type_id;
}
break;

2
tools/lib/bpf/str_error.c
View File

@ -36,7 +36,7 @@ char *libbpf_strerror_r(int err, char *dst, int len)
return dst;
}
const char *errstr(int err)
const char *libbpf_errstr(int err)
{
static __thread char buf[12];

7
tools/lib/bpf/str_error.h
View File

@ -7,10 +7,13 @@
char *libbpf_strerror_r(int err, char *dst, int len);
/**
* @brief **errstr()** returns string corresponding to numeric errno
* @brief **libbpf_errstr()** returns string corresponding to numeric errno
* @param err negative numeric errno
* @return pointer to string representation of the errno, that is invalidated
* upon the next call.
*/
const char *errstr(int err);
const char *libbpf_errstr(int err);
#define errstr(err) libbpf_errstr(err)
#endif /* __LIBBPF_STR_ERROR_H */

32
tools/lib/bpf/usdt.bpf.h
View File

@ -108,6 +108,38 @@ int bpf_usdt_arg_cnt(struct pt_regs *ctx)
return spec->arg_cnt;
}
/* Returns the size in bytes of the #*arg_num* (zero-indexed) USDT argument.
* Returns negative error if argument is not found or arg_num is invalid.
*/
static __always_inline
int bpf_usdt_arg_size(struct pt_regs *ctx, __u64 arg_num)
{
struct __bpf_usdt_arg_spec *arg_spec;
struct __bpf_usdt_spec *spec;
int spec_id;
spec_id = __bpf_usdt_spec_id(ctx);
if (spec_id < 0)
return -ESRCH;
spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
if (!spec)
return -ESRCH;
if (arg_num >= BPF_USDT_MAX_ARG_CNT)
return -ENOENT;
barrier_var(arg_num);
if (arg_num >= spec->arg_cnt)
return -ENOENT;
arg_spec = &spec->args[arg_num];
/* arg_spec->arg_bitshift = 64 - arg_sz * 8
* so: arg_sz = (64 - arg_spec->arg_bitshift) / 8
*/
return (unsigned int)(64 - arg_spec->arg_bitshift) / 8;
}
/* Fetch USDT argument #*arg_num* (zero-indexed) and put its value into *res.
* Returns 0 on success; negative error, otherwise.
* On error *res is guaranteed to be set to zero.

9
tools/testing/selftests/bpf/DENYLIST.aarch64
View File

@ -1,12 +1,3 @@
bpf_cookie/multi_kprobe_attach_api # kprobe_multi_link_api_subtest:FAIL:fentry_raw_skel_load unexpected error: -3
bpf_cookie/multi_kprobe_link_api # kprobe_multi_link_api_subtest:FAIL:fentry_raw_skel_load unexpected error: -3
kprobe_multi_bench_attach # needs CONFIG_FPROBE
kprobe_multi_test # needs CONFIG_FPROBE
module_attach # prog 'kprobe_multi': failed to auto-attach: -95
fentry_test/fentry_many_args # fentry_many_args:FAIL:fentry_many_args_attach unexpected error: -524
fexit_test/fexit_many_args # fexit_many_args:FAIL:fexit_many_args_attach unexpected error: -524
tracing_struct/struct_many_args # struct_many_args:FAIL:tracing_struct_many_args__attach unexpected error: -524
fill_link_info/kprobe_multi_link_info # bpf_program__attach_kprobe_multi_opts unexpected error: -95
fill_link_info/kretprobe_multi_link_info # bpf_program__attach_kprobe_multi_opts unexpected error: -95
fill_link_info/kprobe_multi_invalid_ubuff # bpf_program__attach_kprobe_multi_opts unexpected error: -95
missed/kprobe_recursion # missed_kprobe_recursion__attach unexpected error: -95 (errno 95)

28
tools/testing/selftests/bpf/Makefile
View File

@ -95,18 +95,12 @@ TEST_GEN_PROGS += test_progs-cpuv4
TEST_INST_SUBDIRS += cpuv4
endif
TEST_GEN_FILES = test_lwt_ip_encap.bpf.o test_tc_edt.bpf.o
TEST_GEN_FILES = test_tc_edt.bpf.o
TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c)
# Order correspond to 'make run_tests' order
TEST_PROGS := test_kmod.sh \
test_xdp_redirect_multi.sh \
test_tunnel.sh \
test_lwt_seg6local.sh \
test_lirc_mode2.sh \
test_xdp_vlan_mode_generic.sh \
test_xdp_vlan_mode_native.sh \
test_lwt_ip_encap.sh \
test_tc_tunnel.sh \
test_tc_edt.sh \
test_xdping.sh \
@ -117,9 +111,9 @@ TEST_PROGS := test_kmod.sh \
test_xsk.sh \
test_xdp_features.sh
TEST_PROGS_EXTENDED := with_addr.sh \
with_tunnels.sh ima_setup.sh verify_sig_setup.sh \
test_xdp_vlan.sh test_bpftool.py
TEST_PROGS_EXTENDED := \
ima_setup.sh verify_sig_setup.sh \
test_bpftool.py
TEST_KMODS := bpf_testmod.ko bpf_test_no_cfi.ko bpf_test_modorder_x.ko \
bpf_test_modorder_y.ko
@ -135,7 +129,6 @@ TEST_GEN_PROGS_EXTENDED = \
veristat \
xdp_features \
xdp_hw_metadata \
xdp_redirect_multi \
xdp_synproxy \
xdping \
xskxceiver
@ -184,9 +177,14 @@ ifeq ($(feature-llvm),1)
LLVM_CONFIG_LIB_COMPONENTS := mcdisassembler all-targets
# both llvm-config and lib.mk add -D_GNU_SOURCE, which ends up as conflict
LLVM_CFLAGS += $(filter-out -D_GNU_SOURCE,$(shell $(LLVM_CONFIG) --cflags))
LLVM_LDLIBS += $(shell $(LLVM_CONFIG) --link-static --libs $(LLVM_CONFIG_LIB_COMPONENTS))
LLVM_LDLIBS += $(shell $(LLVM_CONFIG) --link-static --system-libs $(LLVM_CONFIG_LIB_COMPONENTS))
LLVM_LDLIBS += -lstdc++
# Prefer linking statically if it's available, otherwise fallback to shared
ifeq ($(shell $(LLVM_CONFIG) --link-static --libs >/dev/null 2>&1 && echo static),static)
LLVM_LDLIBS += $(shell $(LLVM_CONFIG) --link-static --libs $(LLVM_CONFIG_LIB_COMPONENTS))
LLVM_LDLIBS += $(shell $(LLVM_CONFIG) --link-static --system-libs $(LLVM_CONFIG_LIB_COMPONENTS))
LLVM_LDLIBS += -lstdc++
else
LLVM_LDLIBS += $(shell $(LLVM_CONFIG) --link-shared --libs $(LLVM_CONFIG_LIB_COMPONENTS))
endif
LLVM_LDFLAGS += $(shell $(LLVM_CONFIG) --ldflags)
endif
@ -306,6 +304,7 @@ $(OUTPUT)/runqslower: $(BPFOBJ) | $(DEFAULT_BPFTOOL) $(RUNQSLOWER_OUTPUT)
BPFTOOL_OUTPUT=$(HOST_BUILD_DIR)/bpftool/ \
BPFOBJ_OUTPUT=$(BUILD_DIR)/libbpf/ \
BPFOBJ=$(BPFOBJ) BPF_INCLUDE=$(INCLUDE_DIR) \
BPF_TARGET_ENDIAN=$(BPF_TARGET_ENDIAN) \
EXTRA_CFLAGS='-g $(OPT_FLAGS) $(SAN_CFLAGS) $(EXTRA_CFLAGS)' \
EXTRA_LDFLAGS='$(SAN_LDFLAGS) $(EXTRA_LDFLAGS)' && \
cp $(RUNQSLOWER_OUTPUT)runqslower $@
@ -684,6 +683,7 @@ $(OUTPUT)/$(TRUNNER_BINARY): $(TRUNNER_TEST_OBJS) \
$(TRUNNER_EXTRA_OBJS) $$(BPFOBJ) \
$(RESOLVE_BTFIDS) \
$(TRUNNER_BPFTOOL) \
$(OUTPUT)/veristat \
| $(TRUNNER_BINARY)-extras
$$(call msg,BINARY,,$$@)
$(Q)$$(CC) $$(CFLAGS) $$(filter %.a %.o,$$^) $$(LDLIBS) $$(LDFLAGS) -o $$@

533
tools/testing/selftests/bpf/bpf_arena_spin_lock.h Normal file
View File

@ -0,0 +1,533 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
#ifndef BPF_ARENA_SPIN_LOCK_H
#define BPF_ARENA_SPIN_LOCK_H
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include "bpf_atomic.h"
#define arch_mcs_spin_lock_contended_label(l, label) smp_cond_load_acquire_label(l, VAL, label)
#define arch_mcs_spin_unlock_contended(l) smp_store_release((l), 1)
#if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST)
#define EBUSY 16
#define EOPNOTSUPP 95
#define ETIMEDOUT 110
#ifndef __arena
#define __arena __attribute__((address_space(1)))
#endif
extern unsigned long CONFIG_NR_CPUS __kconfig;
/*
* Typically, we'd just rely on the definition in vmlinux.h for qspinlock, but
* PowerPC overrides the definition to define lock->val as u32 instead of
* atomic_t, leading to compilation errors. Import a local definition below so
* that we don't depend on the vmlinux.h version.
*/
struct __qspinlock {
union {
atomic_t val;
struct {
u8 locked;
u8 pending;
};
struct {
u16 locked_pending;
u16 tail;
};
};
};
#define arena_spinlock_t struct __qspinlock
/* FIXME: Using typedef causes CO-RE relocation error */
/* typedef struct qspinlock arena_spinlock_t; */
struct arena_mcs_spinlock {
struct arena_mcs_spinlock __arena *next;
int locked;
int count;
};
struct arena_qnode {
struct arena_mcs_spinlock mcs;
};
#define _Q_MAX_NODES 4
#define _Q_PENDING_LOOPS 1
/*
* Bitfields in the atomic value:
*
* 0- 7: locked byte
* 8: pending
* 9-15: not used
* 16-17: tail index
* 18-31: tail cpu (+1)
*/
#define _Q_MAX_CPUS 1024
#define _Q_SET_MASK(type) (((1U << _Q_ ## type ## _BITS) - 1)\
<< _Q_ ## type ## _OFFSET)
#define _Q_LOCKED_OFFSET 0
#define _Q_LOCKED_BITS 8
#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
#define _Q_PENDING_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
#define _Q_PENDING_BITS 8
#define _Q_PENDING_MASK _Q_SET_MASK(PENDING)
#define _Q_TAIL_IDX_OFFSET (_Q_PENDING_OFFSET + _Q_PENDING_BITS)
#define _Q_TAIL_IDX_BITS 2
#define _Q_TAIL_IDX_MASK _Q_SET_MASK(TAIL_IDX)
#define _Q_TAIL_CPU_OFFSET (_Q_TAIL_IDX_OFFSET + _Q_TAIL_IDX_BITS)
#define _Q_TAIL_CPU_BITS (32 - _Q_TAIL_CPU_OFFSET)
#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
#define _Q_TAIL_OFFSET _Q_TAIL_IDX_OFFSET
#define _Q_TAIL_MASK (_Q_TAIL_IDX_MASK | _Q_TAIL_CPU_MASK)
#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
#define _Q_PENDING_VAL (1U << _Q_PENDING_OFFSET)
#define likely(x) __builtin_expect(!!(x), 1)
#define unlikely(x) __builtin_expect(!!(x), 0)
struct arena_qnode __arena qnodes[_Q_MAX_CPUS][_Q_MAX_NODES];
static inline u32 encode_tail(int cpu, int idx)
{
u32 tail;
tail = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
tail |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */
return tail;
}
static inline struct arena_mcs_spinlock __arena *decode_tail(u32 tail)
{
u32 cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1;
u32 idx = (tail & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
return &qnodes[cpu][idx].mcs;
}
static inline
struct arena_mcs_spinlock __arena *grab_mcs_node(struct arena_mcs_spinlock __arena *base, int idx)
{
return &((struct arena_qnode __arena *)base + idx)->mcs;
}
#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
/**
* xchg_tail - Put in the new queue tail code word & retrieve previous one
* @lock : Pointer to queued spinlock structure
* @tail : The new queue tail code word
* Return: The previous queue tail code word
*
* xchg(lock, tail)
*
* p,*,* -> n,*,* ; prev = xchg(lock, node)
*/
static __always_inline u32 xchg_tail(arena_spinlock_t __arena *lock, u32 tail)
{
u32 old, new;
old = atomic_read(&lock->val);
do {
new = (old & _Q_LOCKED_PENDING_MASK) | tail;
/*
* We can use relaxed semantics since the caller ensures that
* the MCS node is properly initialized before updating the
* tail.
*/
/* These loops are not expected to stall, but we still need to
* prove to the verifier they will terminate eventually.
*/
cond_break_label(out);
} while (!atomic_try_cmpxchg_relaxed(&lock->val, &old, new));
return old;
out:
bpf_printk("RUNTIME ERROR: %s unexpected cond_break exit!!!", __func__);
return old;
}
/**
* clear_pending - clear the pending bit.
* @lock: Pointer to queued spinlock structure
*
* *,1,* -> *,0,*
*/
static __always_inline void clear_pending(arena_spinlock_t __arena *lock)
{
WRITE_ONCE(lock->pending, 0);
}
/**
* clear_pending_set_locked - take ownership and clear the pending bit.
* @lock: Pointer to queued spinlock structure
*
* *,1,0 -> *,0,1
*
* Lock stealing is not allowed if this function is used.
*/
static __always_inline void clear_pending_set_locked(arena_spinlock_t __arena *lock)
{
WRITE_ONCE(lock->locked_pending, _Q_LOCKED_VAL);
}
/**
* set_locked - Set the lock bit and own the lock
* @lock: Pointer to queued spinlock structure
*
* *,*,0 -> *,0,1
*/
static __always_inline void set_locked(arena_spinlock_t __arena *lock)
{
WRITE_ONCE(lock->locked, _Q_LOCKED_VAL);
}
static __always_inline
u32 arena_fetch_set_pending_acquire(arena_spinlock_t __arena *lock)
{
u32 old, new;
old = atomic_read(&lock->val);
do {
new = old | _Q_PENDING_VAL;
/*
* These loops are not expected to stall, but we still need to
* prove to the verifier they will terminate eventually.
*/
cond_break_label(out);
} while (!atomic_try_cmpxchg_acquire(&lock->val, &old, new));
return old;
out:
bpf_printk("RUNTIME ERROR: %s unexpected cond_break exit!!!", __func__);
return old;
}
/**
* arena_spin_trylock - try to acquire the queued spinlock
* @lock : Pointer to queued spinlock structure
* Return: 1 if lock acquired, 0 if failed
*/
static __always_inline int arena_spin_trylock(arena_spinlock_t __arena *lock)
{
int val = atomic_read(&lock->val);
if (unlikely(val))
return 0;
return likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL));
}
__noinline
int arena_spin_lock_slowpath(arena_spinlock_t __arena __arg_arena *lock, u32 val)
{
struct arena_mcs_spinlock __arena *prev, *next, *node0, *node;
int ret = -ETIMEDOUT;
u32 old, tail;
int idx;
/*
* Wait for in-progress pending->locked hand-overs with a bounded
* number of spins so that we guarantee forward progress.
*
* 0,1,0 -> 0,0,1
*/
if (val == _Q_PENDING_VAL) {
int cnt = _Q_PENDING_LOOPS;
val = atomic_cond_read_relaxed_label(&lock->val,
(VAL != _Q_PENDING_VAL) || !cnt--,
release_err);
}
/*
* If we observe any contention; queue.
*/
if (val & ~_Q_LOCKED_MASK)
goto queue;
/*
* trylock || pending
*
* 0,0,* -> 0,1,* -> 0,0,1 pending, trylock
*/
val = arena_fetch_set_pending_acquire(lock);
/*
* If we observe contention, there is a concurrent locker.
*
* Undo and queue; our setting of PENDING might have made the
* n,0,0 -> 0,0,0 transition fail and it will now be waiting
* on @next to become !NULL.
*/
if (unlikely(val & ~_Q_LOCKED_MASK)) {
/* Undo PENDING if we set it. */
if (!(val & _Q_PENDING_MASK))
clear_pending(lock);
goto queue;
}
/*
* We're pending, wait for the owner to go away.
*
* 0,1,1 -> *,1,0
*
* this wait loop must be a load-acquire such that we match the
* store-release that clears the locked bit and create lock
* sequentiality; this is because not all
* clear_pending_set_locked() implementations imply full
* barriers.
*/
if (val & _Q_LOCKED_MASK)
smp_cond_load_acquire_label(&lock->locked, !VAL, release_err);
/*
* take ownership and clear the pending bit.
*
* 0,1,0 -> 0,0,1
*/
clear_pending_set_locked(lock);
return 0;
/*
* End of pending bit optimistic spinning and beginning of MCS
* queuing.
*/
queue:
node0 = &(qnodes[bpf_get_smp_processor_id()])[0].mcs;
idx = node0->count++;
tail = encode_tail(bpf_get_smp_processor_id(), idx);
/*
* 4 nodes are allocated based on the assumption that there will not be
* nested NMIs taking spinlocks. That may not be true in some
* architectures even though the chance of needing more than 4 nodes
* will still be extremely unlikely. When that happens, we simply return
* an error. Original qspinlock has a trylock fallback in this case.
*/
if (unlikely(idx >= _Q_MAX_NODES)) {
ret = -EBUSY;
goto release_node_err;
}
node = grab_mcs_node(node0, idx);
/*
* Ensure that we increment the head node->count before initialising
* the actual node. If the compiler is kind enough to reorder these
* stores, then an IRQ could overwrite our assignments.
*/
barrier();
node->locked = 0;
node->next = NULL;
/*
* We touched a (possibly) cold cacheline in the per-cpu queue node;
* attempt the trylock once more in the hope someone let go while we
* weren't watching.
*/
if (arena_spin_trylock(lock))
goto release;
/*
* Ensure that the initialisation of @node is complete before we
* publish the updated tail via xchg_tail() and potentially link
* @node into the waitqueue via WRITE_ONCE(prev->next, node) below.
*/
smp_wmb();
/*
* Publish the updated tail.
* We have already touched the queueing cacheline; don't bother with
* pending stuff.
*
* p,*,* -> n,*,*
*/
old = xchg_tail(lock, tail);
next = NULL;
/*
* if there was a previous node; link it and wait until reaching the
* head of the waitqueue.
*/
if (old & _Q_TAIL_MASK) {
prev = decode_tail(old);
/* Link @node into the waitqueue. */
WRITE_ONCE(prev->next, node);
arch_mcs_spin_lock_contended_label(&node->locked, release_node_err);
/*
* While waiting for the MCS lock, the next pointer may have
* been set by another lock waiter. We cannot prefetch here
* due to lack of equivalent instruction in BPF ISA.
*/
next = READ_ONCE(node->next);
}
/*
* we're at the head of the waitqueue, wait for the owner & pending to
* go away.
*
* *,x,y -> *,0,0
*
* this wait loop must use a load-acquire such that we match the
* store-release that clears the locked bit and create lock
* sequentiality; this is because the set_locked() function below
* does not imply a full barrier.
*/
val = atomic_cond_read_acquire_label(&lock->val, !(VAL & _Q_LOCKED_PENDING_MASK),
release_node_err);
/*
* claim the lock:
*
* n,0,0 -> 0,0,1 : lock, uncontended
* *,*,0 -> *,*,1 : lock, contended
*
* If the queue head is the only one in the queue (lock value == tail)
* and nobody is pending, clear the tail code and grab the lock.
* Otherwise, we only need to grab the lock.
*/
/*
* In the PV case we might already have _Q_LOCKED_VAL set, because
* of lock stealing; therefore we must also allow:
*
* n,0,1 -> 0,0,1
*
* Note: at this point: (val & _Q_PENDING_MASK) == 0, because of the
* above wait condition, therefore any concurrent setting of
* PENDING will make the uncontended transition fail.
*/
if ((val & _Q_TAIL_MASK) == tail) {
if (atomic_try_cmpxchg_relaxed(&lock->val, &val, _Q_LOCKED_VAL))
goto release; /* No contention */
}
/*
* Either somebody is queued behind us or _Q_PENDING_VAL got set
* which will then detect the remaining tail and queue behind us
* ensuring we'll see a @next.
*/
set_locked(lock);
/*
* contended path; wait for next if not observed yet, release.
*/
if (!next)
next = smp_cond_load_relaxed_label(&node->next, (VAL), release_node_err);
arch_mcs_spin_unlock_contended(&next->locked);
release:;
/*
* release the node
*
* Doing a normal dec vs this_cpu_dec is fine. An upper context always
* decrements count it incremented before returning, thus we're fine.
* For contexts interrupting us, they either observe our dec or not.
* Just ensure the compiler doesn't reorder this statement, as a
* this_cpu_dec implicitly implied that.
*/
barrier();
node0->count--;
return 0;
release_node_err:
barrier();
node0->count--;
goto release_err;
release_err:
return ret;
}
/**
* arena_spin_lock - acquire a queued spinlock
* @lock: Pointer to queued spinlock structure
*
* On error, returned value will be negative.
* On success, zero is returned.
*
* The return value _must_ be tested against zero for success,
* instead of checking it against negative, for passing the
* BPF verifier.
*
* The user should do:
* if (arena_spin_lock(...) != 0) // failure
* or
* if (arena_spin_lock(...) == 0) // success
* or
* if (arena_spin_lock(...)) // failure
* or
* if (!arena_spin_lock(...)) // success
* instead of:
* if (arena_spin_lock(...) < 0) // failure
*
* The return value can still be inspected later.
*/
static __always_inline int arena_spin_lock(arena_spinlock_t __arena *lock)
{
int val = 0;
if (CONFIG_NR_CPUS > 1024)
return -EOPNOTSUPP;
bpf_preempt_disable();
if (likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL)))
return 0;
val = arena_spin_lock_slowpath(lock, val);
/* FIXME: bpf_assert_range(-MAX_ERRNO, 0) once we have it working for all cases. */
if (val)
bpf_preempt_enable();
return val;
}
/**
* arena_spin_unlock - release a queued spinlock
* @lock : Pointer to queued spinlock structure
*/
static __always_inline void arena_spin_unlock(arena_spinlock_t __arena *lock)
{
/*
* unlock() needs release semantics:
*/
smp_store_release(&lock->locked, 0);
bpf_preempt_enable();
}
#define arena_spin_lock_irqsave(lock, flags) \
({ \
int __ret; \
bpf_local_irq_save(&(flags)); \
__ret = arena_spin_lock((lock)); \
if (__ret) \
bpf_local_irq_restore(&(flags)); \
(__ret); \
})
#define arena_spin_unlock_irqrestore(lock, flags) \
({ \
arena_spin_unlock((lock)); \
bpf_local_irq_restore(&(flags)); \
})
#endif
#endif /* BPF_ARENA_SPIN_LOCK_H */

140
tools/testing/selftests/bpf/bpf_atomic.h Normal file
View File

@ -0,0 +1,140 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
#ifndef BPF_ATOMIC_H
#define BPF_ATOMIC_H
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include "bpf_experimental.h"
extern bool CONFIG_X86_64 __kconfig __weak;
/*
* __unqual_typeof(x) - Declare an unqualified scalar type, leaving
* non-scalar types unchanged,
*
* Prefer C11 _Generic for better compile-times and simpler code. Note: 'char'
* is not type-compatible with 'signed char', and we define a separate case.
*
* This is copied verbatim from kernel's include/linux/compiler_types.h, but
* with default expression (for pointers) changed from (x) to (typeof(x)0).
*
* This is because LLVM has a bug where for lvalue (x), it does not get rid of
* an extra address_space qualifier, but does in case of rvalue (typeof(x)0).
* Hence, for pointers, we need to create an rvalue expression to get the
* desired type. See https://github.com/llvm/llvm-project/issues/53400.
*/
#define __scalar_type_to_expr_cases(type) \
unsigned type : (unsigned type)0, signed type : (signed type)0
#define __unqual_typeof(x) \
typeof(_Generic((x), \
char: (char)0, \
__scalar_type_to_expr_cases(char), \
__scalar_type_to_expr_cases(short), \
__scalar_type_to_expr_cases(int), \
__scalar_type_to_expr_cases(long), \
__scalar_type_to_expr_cases(long long), \
default: (typeof(x))0))
/* No-op for BPF */
#define cpu_relax() ({})
#define READ_ONCE(x) (*(volatile typeof(x) *)&(x))
#define WRITE_ONCE(x, val) ((*(volatile typeof(x) *)&(x)) = (val))
#define cmpxchg(p, old, new) __sync_val_compare_and_swap((p), old, new)
#define try_cmpxchg(p, pold, new) \
({ \
__unqual_typeof(*(pold)) __o = *(pold); \
__unqual_typeof(*(p)) __r = cmpxchg(p, __o, new); \
if (__r != __o) \
*(pold) = __r; \
__r == __o; \
})
#define try_cmpxchg_relaxed(p, pold, new) try_cmpxchg(p, pold, new)
#define try_cmpxchg_acquire(p, pold, new) try_cmpxchg(p, pold, new)
#define smp_mb() \
({ \
unsigned long __val; \
__sync_fetch_and_add(&__val, 0); \
})
#define smp_rmb() \
({ \
if (!CONFIG_X86_64) \
smp_mb(); \
else \
barrier(); \
})
#define smp_wmb() \
({ \
if (!CONFIG_X86_64) \
smp_mb(); \
else \
barrier(); \
})
/* Control dependency provides LOAD->STORE, provide LOAD->LOAD */
#define smp_acquire__after_ctrl_dep() ({ smp_rmb(); })
#define smp_load_acquire(p) \
({ \
__unqual_typeof(*(p)) __v = READ_ONCE(*(p)); \
if (!CONFIG_X86_64) \
smp_mb(); \
barrier(); \
__v; \
})
#define smp_store_release(p, val) \
({ \
if (!CONFIG_X86_64) \
smp_mb(); \
barrier(); \
WRITE_ONCE(*(p), val); \
})
#define smp_cond_load_relaxed_label(p, cond_expr, label) \
({ \
typeof(p) __ptr = (p); \
__unqual_typeof(*(p)) VAL; \
for (;;) { \
VAL = (__unqual_typeof(*(p)))READ_ONCE(*__ptr); \
if (cond_expr) \
break; \
cond_break_label(label); \
cpu_relax(); \
} \
(typeof(*(p)))VAL; \
})
#define smp_cond_load_acquire_label(p, cond_expr, label) \
({ \
__unqual_typeof(*p) __val = \
smp_cond_load_relaxed_label(p, cond_expr, label); \
smp_acquire__after_ctrl_dep(); \
(typeof(*(p)))__val; \
})
#define atomic_read(p) READ_ONCE((p)->counter)
#define atomic_cond_read_relaxed_label(p, cond_expr, label) \
smp_cond_load_relaxed_label(&(p)->counter, cond_expr, label)
#define atomic_cond_read_acquire_label(p, cond_expr, label) \
smp_cond_load_acquire_label(&(p)->counter, cond_expr, label)
#define atomic_try_cmpxchg_relaxed(p, pold, new) \
try_cmpxchg_relaxed(&(p)->counter, pold, new)
#define atomic_try_cmpxchg_acquire(p, pold, new) \
try_cmpxchg_acquire(&(p)->counter, pold, new)
#endif /* BPF_ATOMIC_H */

15
tools/testing/selftests/bpf/bpf_experimental.h
View File

@ -368,12 +368,12 @@ l_true: \
ret; \
})
#define cond_break \
#define __cond_break(expr) \
({ __label__ l_break, l_continue; \
asm volatile goto("may_goto %l[l_break]" \
:::: l_break); \
goto l_continue; \
l_break: break; \
l_break: expr; \
l_continue:; \
})
#else
@ -392,7 +392,7 @@ l_true: \
ret; \
})
#define cond_break \
#define __cond_break(expr) \
({ __label__ l_break, l_continue; \
asm volatile goto("1:.byte 0xe5; \
.byte 0; \
@ -400,7 +400,7 @@ l_true: \
.short 0" \
:::: l_break); \
goto l_continue; \
l_break: break; \
l_break: expr; \
l_continue:; \
})
#else
@ -418,7 +418,7 @@ l_true: \
ret; \
})
#define cond_break \
#define __cond_break(expr) \
({ __label__ l_break, l_continue; \
asm volatile goto("1:.byte 0xe5; \
.byte 0; \
@ -426,12 +426,15 @@ l_true: \
.short 0" \
:::: l_break); \
goto l_continue; \
l_break: break; \
l_break: expr; \
l_continue:; \
})
#endif
#endif
#define cond_break __cond_break(break)
#define cond_break_label(label) __cond_break(goto label)
#ifndef bpf_nop_mov
#define bpf_nop_mov(var) \
asm volatile("%[reg]=%[reg]"::[reg]"r"((short)var))

5
tools/testing/selftests/bpf/bpf_kfuncs.h
View File

@ -87,4 +87,9 @@ struct dentry;
*/
extern int bpf_get_dentry_xattr(struct dentry *dentry, const char *name,
struct bpf_dynptr *value_ptr) __ksym __weak;
extern int bpf_set_dentry_xattr(struct dentry *dentry, const char *name__str,
const struct bpf_dynptr *value_p, int flags) __ksym __weak;
extern int bpf_remove_dentry_xattr(struct dentry *dentry, const char *name__str) __ksym __weak;
#endif

8
tools/testing/selftests/bpf/cap_helpers.c
View File

@ -19,7 +19,7 @@ int cap_enable_effective(__u64 caps, __u64 *old_caps)
err = capget(&hdr, data);
if (err)
return err;
return -errno;
if (old_caps)
*old_caps = (__u64)(data[1].effective) << 32 | data[0].effective;
@ -32,7 +32,7 @@ int cap_enable_effective(__u64 caps, __u64 *old_caps)
data[1].effective |= cap1;
err = capset(&hdr, data);
if (err)
return err;
return -errno;
return 0;
}
@ -49,7 +49,7 @@ int cap_disable_effective(__u64 caps, __u64 *old_caps)
err = capget(&hdr, data);
if (err)
return err;
return -errno;
if (old_caps)
*old_caps = (__u64)(data[1].effective) << 32 | data[0].effective;
@ -61,7 +61,7 @@ int cap_disable_effective(__u64 caps, __u64 *old_caps)
data[1].effective &= ~cap1;
err = capset(&hdr, data);
if (err)
return err;
return -errno;
return 0;
}

1
tools/testing/selftests/bpf/cap_helpers.h
View File

@ -4,6 +4,7 @@
#include <linux/types.h>
#include <linux/capability.h>
#include <errno.h>
#ifndef CAP_PERFMON
#define CAP_PERFMON 38

111
tools/testing/selftests/bpf/network_helpers.c
View File

@ -446,6 +446,23 @@ char *ping_command(int family)
return "ping";
}
int append_tid(char *str, size_t sz)
{
size_t end;
if (!str)
return -1;
end = strlen(str);
if (end + 8 > sz)
return -1;
sprintf(&str[end], "%07d", gettid());
str[end + 7] = '\0';
return 0;
}
int remove_netns(const char *name)
{
char *cmd;
@ -761,6 +778,36 @@ struct tmonitor_ctx {
int pcap_fd;
};
static int __base_pr(const char *format, va_list args)
{
return vfprintf(stdout, format, args);
}
static tm_print_fn_t __tm_pr = __base_pr;
tm_print_fn_t traffic_monitor_set_print(tm_print_fn_t fn)
{
tm_print_fn_t old_print_fn;
old_print_fn = __atomic_exchange_n(&__tm_pr, fn, __ATOMIC_RELAXED);
return old_print_fn;
}
void tm_print(const char *format, ...)
{
tm_print_fn_t print_fn;
va_list args;
print_fn = __atomic_load_n(&__tm_pr, __ATOMIC_RELAXED);
if (!print_fn)
return;
va_start(args, format);
print_fn(format, args);
va_end(args);
}
/* Is this packet captured with a Ethernet protocol type? */
static bool is_ethernet(const u_char *packet)
{
@ -778,7 +825,7 @@ static bool is_ethernet(const u_char *packet)
case 770: /* ARPHRD_FRAD */
case 778: /* ARPHDR_IPGRE */
case 803: /* ARPHRD_IEEE80211_RADIOTAP */
printf("Packet captured: arphdr_type=%d\n", arphdr_type);
tm_print("Packet captured: arphdr_type=%d\n", arphdr_type);
return false;
}
return true;
@ -799,12 +846,13 @@ static const char *pkt_type_str(u16 pkt_type)
return "Unknown";
}
#define MAX_FLAGS_STRLEN 21
/* Show the information of the transport layer in the packet */
static void show_transport(const u_char *packet, u16 len, u32 ifindex,
const char *src_addr, const char *dst_addr,
u16 proto, bool ipv6, u8 pkt_type)
{
char *ifname, _ifname[IF_NAMESIZE];
char *ifname, _ifname[IF_NAMESIZE], flags[MAX_FLAGS_STRLEN] = "";
const char *transport_str;
u16 src_port, dst_port;
struct udphdr *udp;
@ -827,47 +875,39 @@ static void show_transport(const u_char *packet, u16 len, u32 ifindex,
dst_port = ntohs(tcp->dest);
transport_str = "TCP";
} else if (proto == IPPROTO_ICMP) {
printf("%-7s %-3s IPv4 %s > %s: ICMP, length %d, type %d, code %d\n",
ifname, pkt_type_str(pkt_type), src_addr, dst_addr, len,
packet[0], packet[1]);
tm_print("%-7s %-3s IPv4 %s > %s: ICMP, length %d, type %d, code %d\n",
ifname, pkt_type_str(pkt_type), src_addr, dst_addr, len,
packet[0], packet[1]);
return;
} else if (proto == IPPROTO_ICMPV6) {
printf("%-7s %-3s IPv6 %s > %s: ICMPv6, length %d, type %d, code %d\n",
ifname, pkt_type_str(pkt_type), src_addr, dst_addr, len,
packet[0], packet[1]);
tm_print("%-7s %-3s IPv6 %s > %s: ICMPv6, length %d, type %d, code %d\n",
ifname, pkt_type_str(pkt_type), src_addr, dst_addr, len,
packet[0], packet[1]);
return;
} else {
printf("%-7s %-3s %s %s > %s: protocol %d\n",
ifname, pkt_type_str(pkt_type), ipv6 ? "IPv6" : "IPv4",
src_addr, dst_addr, proto);
tm_print("%-7s %-3s %s %s > %s: protocol %d\n",
ifname, pkt_type_str(pkt_type), ipv6 ? "IPv6" : "IPv4",
src_addr, dst_addr, proto);
return;
}
/* TCP or UDP*/
flockfile(stdout);
if (proto == IPPROTO_TCP)
snprintf(flags, MAX_FLAGS_STRLEN, "%s%s%s%s",
tcp->fin ? ", FIN" : "",
tcp->syn ? ", SYN" : "",
tcp->rst ? ", RST" : "",
tcp->ack ? ", ACK" : "");
if (ipv6)
printf("%-7s %-3s IPv6 %s.%d > %s.%d: %s, length %d",
ifname, pkt_type_str(pkt_type), src_addr, src_port,
dst_addr, dst_port, transport_str, len);
tm_print("%-7s %-3s IPv6 %s.%d > %s.%d: %s, length %d%s\n",
ifname, pkt_type_str(pkt_type), src_addr, src_port,
dst_addr, dst_port, transport_str, len, flags);
else
printf("%-7s %-3s IPv4 %s:%d > %s:%d: %s, length %d",
ifname, pkt_type_str(pkt_type), src_addr, src_port,
dst_addr, dst_port, transport_str, len);
if (proto == IPPROTO_TCP) {
if (tcp->fin)
printf(", FIN");
if (tcp->syn)
printf(", SYN");
if (tcp->rst)
printf(", RST");
if (tcp->ack)
printf(", ACK");
}
printf("\n");
funlockfile(stdout);
tm_print("%-7s %-3s IPv4 %s:%d > %s:%d: %s, length %d%s\n",
ifname, pkt_type_str(pkt_type), src_addr, src_port,
dst_addr, dst_port, transport_str, len, flags);
}
static void show_ipv6_packet(const u_char *packet, u32 ifindex, u8 pkt_type)
@ -982,8 +1022,8 @@ static void *traffic_monitor_thread(void *arg)
ifname = _ifname;
}
printf("%-7s %-3s Unknown network protocol type 0x%x\n",
ifname, pkt_type_str(ptype), proto);
tm_print("%-7s %-3s Unknown network protocol type 0x%x\n",
ifname, pkt_type_str(ptype), proto);
}
}
@ -1183,8 +1223,9 @@ void traffic_monitor_stop(struct tmonitor_ctx *ctx)
write(ctx->wake_fd, &w, sizeof(w));
pthread_join(ctx->thread, NULL);
printf("Packet file: %s\n", strrchr(ctx->pkt_fname, '/') + 1);
tm_print("Packet file: %s\n", strrchr(ctx->pkt_fname, '/') + 1);
traffic_monitor_release(ctx);
}
#endif /* TRAFFIC_MONITOR */

21
tools/testing/selftests/bpf/network_helpers.h
View File

@ -18,6 +18,7 @@ typedef __u16 __sum16;
#include <netinet/udp.h>
#include <bpf/bpf_endian.h>
#include <net/if.h>
#include <stdio.h>
#define MAGIC_VAL 0x1234
#define NUM_ITER 100000
@ -101,6 +102,18 @@ int send_recv_data(int lfd, int fd, uint32_t total_bytes);
int make_netns(const char *name);
int remove_netns(const char *name);
/**
* append_tid() - Append thread ID to the given string.
*
* @str: string to extend
* @sz: string's size
*
* 8 characters are used to append the thread ID (7 digits + '\0')
*
* Returns -1 on errors, 0 otherwise
*/
int append_tid(char *str, size_t sz);
static __u16 csum_fold(__u32 csum)
{
csum = (csum & 0xffff) + (csum >> 16);
@ -240,10 +253,13 @@ static inline __sum16 build_udp_v6_csum(const struct ipv6hdr *ip6h,
struct tmonitor_ctx;
typedef int (*tm_print_fn_t)(const char *format, va_list args);
#ifdef TRAFFIC_MONITOR
struct tmonitor_ctx *traffic_monitor_start(const char *netns, const char *test_name,
const char *subtest_name);
void traffic_monitor_stop(struct tmonitor_ctx *ctx);
tm_print_fn_t traffic_monitor_set_print(tm_print_fn_t fn);
#else
static inline struct tmonitor_ctx *traffic_monitor_start(const char *netns, const char *test_name,
const char *subtest_name)
@ -254,6 +270,11 @@ static inline struct tmonitor_ctx *traffic_monitor_start(const char *netns, cons
static inline void traffic_monitor_stop(struct tmonitor_ctx *ctx)
{
}
static inline tm_print_fn_t traffic_monitor_set_print(tm_print_fn_t fn)
{
return NULL;
}
#endif
#endif

11
tools/testing/selftests/bpf/prog_tests/align.c
View File

@ -610,9 +610,11 @@ static int do_test_single(struct bpf_align_test *test)
.log_size = sizeof(bpf_vlog),
.log_level = 2,
);
const char *main_pass_start = "0: R1=ctx() R10=fp0";
const char *line_ptr;
int cur_line = -1;
int prog_len, i;
char *start;
int fd_prog;
int ret;
@ -632,7 +634,13 @@ static int do_test_single(struct bpf_align_test *test)
ret = 0;
/* We make a local copy so that we can strtok() it */
strncpy(bpf_vlog_copy, bpf_vlog, sizeof(bpf_vlog_copy));
line_ptr = strtok(bpf_vlog_copy, "\n");
start = strstr(bpf_vlog_copy, main_pass_start);
if (!start) {
ret = 1;
printf("Can't find initial line '%s'\n", main_pass_start);
goto out;
}
line_ptr = strtok(start, "\n");
for (i = 0; i < MAX_MATCHES; i++) {
struct bpf_reg_match m = test->matches[i];
const char *p;
@ -682,6 +690,7 @@ static int do_test_single(struct bpf_align_test *test)
break;
}
}
out:
if (fd_prog >= 0)
close(fd_prog);
}

66
tools/testing/selftests/bpf/prog_tests/arena_atomics.c
View File

@ -162,6 +162,66 @@ static void test_uaf(struct arena_atomics *skel)
ASSERT_EQ(skel->arena->uaf_recovery_fails, 0, "uaf_recovery_fails");
}
static void test_load_acquire(struct arena_atomics *skel)
{
LIBBPF_OPTS(bpf_test_run_opts, topts);
int err, prog_fd;
if (skel->data->skip_lacq_srel_tests) {
printf("%s:SKIP: ENABLE_ATOMICS_TESTS not defined, Clang doesn't support addr_space_cast, and/or JIT doesn't support load-acquire\n",
__func__);
test__skip();
return;
}
/* No need to attach it, just run it directly */
prog_fd = bpf_program__fd(skel->progs.load_acquire);
err = bpf_prog_test_run_opts(prog_fd, &topts);
if (!ASSERT_OK(err, "test_run_opts err"))
return;
if (!ASSERT_OK(topts.retval, "test_run_opts retval"))
return;
ASSERT_EQ(skel->arena->load_acquire8_result, 0x12,
"load_acquire8_result");
ASSERT_EQ(skel->arena->load_acquire16_result, 0x1234,
"load_acquire16_result");
ASSERT_EQ(skel->arena->load_acquire32_result, 0x12345678,
"load_acquire32_result");
ASSERT_EQ(skel->arena->load_acquire64_result, 0x1234567890abcdef,
"load_acquire64_result");
}
static void test_store_release(struct arena_atomics *skel)
{
LIBBPF_OPTS(bpf_test_run_opts, topts);
int err, prog_fd;
if (skel->data->skip_lacq_srel_tests) {
printf("%s:SKIP: ENABLE_ATOMICS_TESTS not defined, Clang doesn't support addr_space_cast, and/or JIT doesn't support store-release\n",
__func__);
test__skip();
return;
}
/* No need to attach it, just run it directly */
prog_fd = bpf_program__fd(skel->progs.store_release);
err = bpf_prog_test_run_opts(prog_fd, &topts);
if (!ASSERT_OK(err, "test_run_opts err"))
return;
if (!ASSERT_OK(topts.retval, "test_run_opts retval"))
return;
ASSERT_EQ(skel->arena->store_release8_result, 0x12,
"store_release8_result");
ASSERT_EQ(skel->arena->store_release16_result, 0x1234,
"store_release16_result");
ASSERT_EQ(skel->arena->store_release32_result, 0x12345678,
"store_release32_result");
ASSERT_EQ(skel->arena->store_release64_result, 0x1234567890abcdef,
"store_release64_result");
}
void test_arena_atomics(void)
{
struct arena_atomics *skel;
@ -171,7 +231,7 @@ void test_arena_atomics(void)
if (!ASSERT_OK_PTR(skel, "arena atomics skeleton open"))
return;
if (skel->data->skip_tests) {
if (skel->data->skip_all_tests) {
printf("%s:SKIP:no ENABLE_ATOMICS_TESTS or no addr_space_cast support in clang",
__func__);
test__skip();
@ -198,6 +258,10 @@ void test_arena_atomics(void)
test_xchg(skel);
if (test__start_subtest("uaf"))
test_uaf(skel);
if (test__start_subtest("load_acquire"))
test_load_acquire(skel);
if (test__start_subtest("store_release"))
test_store_release(skel);
cleanup:
arena_atomics__destroy(skel);

108
tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c Normal file
View File

@ -0,0 +1,108 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
#include <test_progs.h>
#include <network_helpers.h>
#include <sys/sysinfo.h>
struct __qspinlock { int val; };
typedef struct __qspinlock arena_spinlock_t;
struct arena_qnode {
unsigned long next;
int count;
int locked;
};
#include "arena_spin_lock.skel.h"
static long cpu;
static int repeat;
pthread_barrier_t barrier;
static void *spin_lock_thread(void *arg)
{
int err, prog_fd = *(u32 *)arg;
LIBBPF_OPTS(bpf_test_run_opts, topts,
.data_in = &pkt_v4,
.data_size_in = sizeof(pkt_v4),
.repeat = repeat,
);
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(__sync_fetch_and_add(&cpu, 1), &cpuset);
ASSERT_OK(pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset), "cpu affinity");
err = pthread_barrier_wait(&barrier);
if (err != PTHREAD_BARRIER_SERIAL_THREAD && err != 0)
ASSERT_FALSE(true, "pthread_barrier");
err = bpf_prog_test_run_opts(prog_fd, &topts);
ASSERT_OK(err, "test_run err");
ASSERT_EQ((int)topts.retval, 0, "test_run retval");
pthread_exit(arg);
}
static void test_arena_spin_lock_size(int size)
{
LIBBPF_OPTS(bpf_test_run_opts, topts);
struct arena_spin_lock *skel;
pthread_t thread_id[16];
int prog_fd, i, err;
void *ret;
if (get_nprocs() < 2) {
test__skip();
return;
}
skel = arena_spin_lock__open_and_load();
if (!ASSERT_OK_PTR(skel, "arena_spin_lock__open_and_load"))
return;
if (skel->data->test_skip == 2) {
test__skip();
goto end;
}
skel->bss->cs_count = size;
skel->bss->limit = repeat * 16;
ASSERT_OK(pthread_barrier_init(&barrier, NULL, 16), "barrier init");
prog_fd = bpf_program__fd(skel->progs.prog);
for (i = 0; i < 16; i++) {
err = pthread_create(&thread_id[i], NULL, &spin_lock_thread, &prog_fd);
if (!ASSERT_OK(err, "pthread_create"))
goto end_barrier;
}
for (i = 0; i < 16; i++) {
if (!ASSERT_OK(pthread_join(thread_id[i], &ret), "pthread_join"))
goto end_barrier;
if (!ASSERT_EQ(ret, &prog_fd, "ret == prog_fd"))
goto end_barrier;
}
ASSERT_EQ(skel->bss->counter, repeat * 16, "check counter value");
end_barrier:
pthread_barrier_destroy(&barrier);
end:
arena_spin_lock__destroy(skel);
return;
}
void test_arena_spin_lock(void)
{
repeat = 1000;
if (test__start_subtest("arena_spin_lock_1"))
test_arena_spin_lock_size(1);
cpu = 0;
if (test__start_subtest("arena_spin_lock_1000"))
test_arena_spin_lock_size(1000);
cpu = 0;
repeat = 100;
if (test__start_subtest("arena_spin_lock_50000"))
test_arena_spin_lock_size(50000);
}

5
tools/testing/selftests/bpf/prog_tests/bloom_filter_map.c
View File

@ -6,6 +6,10 @@
#include <test_progs.h>
#include "bloom_filter_map.skel.h"
#ifndef NUMA_NO_NODE
#define NUMA_NO_NODE (-1)
#endif
static void test_fail_cases(void)
{
LIBBPF_OPTS(bpf_map_create_opts, opts);
@ -69,6 +73,7 @@ static void test_success_cases(void)
/* Create a map */
opts.map_flags = BPF_F_ZERO_SEED | BPF_F_NUMA_NODE;
opts.numa_node = NUMA_NO_NODE;
fd = bpf_map_create(BPF_MAP_TYPE_BLOOM_FILTER, NULL, 0, sizeof(value), 100, &opts);
if (!ASSERT_GE(fd, 0, "bpf_map_create bloom filter success case"))
return;

68
tools/testing/selftests/bpf/prog_tests/bpf_iter.c
View File

@ -323,19 +323,87 @@ static void test_task_pidfd(void)
static void test_task_sleepable(void)
{
struct bpf_iter_tasks *skel;
int pid, status, err, data_pipe[2], finish_pipe[2], c;
char *test_data = NULL;
char *test_data_long = NULL;
char *data[2];
if (!ASSERT_OK(pipe(data_pipe), "data_pipe") ||
!ASSERT_OK(pipe(finish_pipe), "finish_pipe"))
return;
skel = bpf_iter_tasks__open_and_load();
if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))
return;
pid = fork();
if (!ASSERT_GE(pid, 0, "fork"))
return;
if (pid == 0) {
/* child */
close(data_pipe[0]);
close(finish_pipe[1]);
test_data = malloc(sizeof(char) * 10);
strncpy(test_data, "test_data", 10);
test_data[9] = '\0';
test_data_long = malloc(sizeof(char) * 5000);
for (int i = 0; i < 5000; ++i) {
if (i % 2 == 0)
test_data_long[i] = 'b';
else
test_data_long[i] = 'a';
}
test_data_long[4999] = '\0';
data[0] = test_data;
data[1] = test_data_long;
write(data_pipe[1], &data, sizeof(data));
/* keep child alive until after the test */
err = read(finish_pipe[0], &c, 1);
if (err != 1)
exit(-1);
close(data_pipe[1]);
close(finish_pipe[0]);
_exit(0);
}
/* parent */
close(data_pipe[1]);
close(finish_pipe[0]);
err = read(data_pipe[0], &data, sizeof(data));
ASSERT_EQ(err, sizeof(data), "read_check");
skel->bss->user_ptr = data[0];
skel->bss->user_ptr_long = data[1];
skel->bss->pid = pid;
do_dummy_read(skel->progs.dump_task_sleepable);
ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task, 0,
"num_expected_failure_copy_from_user_task");
ASSERT_GT(skel->bss->num_success_copy_from_user_task, 0,
"num_success_copy_from_user_task");
ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task_str, 0,
"num_expected_failure_copy_from_user_task_str");
ASSERT_GT(skel->bss->num_success_copy_from_user_task_str, 0,
"num_success_copy_from_user_task_str");
bpf_iter_tasks__destroy(skel);
write(finish_pipe[1], &c, 1);
err = waitpid(pid, &status, 0);
ASSERT_EQ(err, pid, "waitpid");
ASSERT_EQ(status, 0, "zero_child_exit");
close(data_pipe[0]);
close(finish_pipe[1]);
}
static void test_task_stack(void)

9
tools/testing/selftests/bpf/prog_tests/bpf_nf.c
View File

@ -72,11 +72,14 @@ static void test_bpf_nf_ct(int mode)
if (!ASSERT_OK(system(cmd), cmd))
goto end;
srv_port = (mode == TEST_XDP) ? 5005 : 5006;
srv_fd = start_server(AF_INET, SOCK_STREAM, "127.0.0.1", srv_port, TIMEOUT_MS);
srv_fd = start_server(AF_INET, SOCK_STREAM, "127.0.0.1", 0, TIMEOUT_MS);
if (!ASSERT_GE(srv_fd, 0, "start_server"))
goto end;
srv_port = get_socket_local_port(srv_fd);
if (!ASSERT_GE(srv_port, 0, "get_sock_local_port"))
goto end;
client_fd = connect_to_server(srv_fd);
if (!ASSERT_GE(client_fd, 0, "connect_to_server"))
goto end;
@ -91,7 +94,7 @@ static void test_bpf_nf_ct(int mode)
skel->bss->saddr = peer_addr.sin_addr.s_addr;
skel->bss->sport = peer_addr.sin_port;
skel->bss->daddr = peer_addr.sin_addr.s_addr;
skel->bss->dport = htons(srv_port);
skel->bss->dport = srv_port;
if (mode == TEST_XDP)
prog_fd = bpf_program__fd(skel->progs.nf_xdp_ct_test);

23
tools/testing/selftests/bpf/prog_tests/btf.c
View File

@ -3866,11 +3866,11 @@ static struct btf_raw_test raw_tests[] = {
.err_str = "vlen != 0",
},
{
.descr = "decl_tag test #8, invalid kflag",
.descr = "decl_tag test #8, tag with kflag",
.raw_types = {
BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
BTF_VAR_ENC(NAME_TBD, 1, 0), /* [2] */
BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_DECL_TAG, 1, 0), 2), (-1),
BTF_DECL_ATTR_ENC(NAME_TBD, 2, -1),
BTF_END_RAW,
},
BTF_STR_SEC("\0local\0tag1"),
@ -3881,8 +3881,6 @@ static struct btf_raw_test raw_tests[] = {
.key_type_id = 1,
.value_type_id = 1,
.max_entries = 1,
.btf_load_err = true,
.err_str = "Invalid btf_info kind_flag",
},
{
.descr = "decl_tag test #9, var, invalid component_idx",
@ -4206,6 +4204,23 @@ static struct btf_raw_test raw_tests[] = {
.btf_load_err = true,
.err_str = "Type tags don't precede modifiers",
},
{
.descr = "type_tag test #7, tag with kflag",
.raw_types = {
BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
BTF_TYPE_ATTR_ENC(NAME_TBD, 1), /* [2] */
BTF_PTR_ENC(2), /* [3] */
BTF_END_RAW,
},
BTF_STR_SEC("\0tag"),
.map_type = BPF_MAP_TYPE_ARRAY,
.map_name = "tag_type_check_btf",
.key_size = sizeof(int),
.value_size = 4,
.key_type_id = 1,
.value_type_id = 1,
.max_entries = 1,
},
{
.descr = "enum64 test #1, unsigned, size 8",
.raw_types = {

147
tools/testing/selftests/bpf/prog_tests/btf_dump.c
View File

@ -126,25 +126,68 @@ done:
return err;
}
static char *dump_buf;
static size_t dump_buf_sz;
static FILE *dump_buf_file;
struct test_ctx {
struct btf *btf;
struct btf_dump *d;
char *dump_buf;
size_t dump_buf_sz;
FILE *dump_buf_file;
};
static void test_ctx__free(struct test_ctx *t)
{
fclose(t->dump_buf_file);
free(t->dump_buf);
btf_dump__free(t->d);
btf__free(t->btf);
}
static int test_ctx__init(struct test_ctx *t)
{
t->dump_buf_file = open_memstream(&t->dump_buf, &t->dump_buf_sz);
if (!ASSERT_OK_PTR(t->dump_buf_file, "dump_memstream"))
return -1;
t->btf = btf__new_empty();
if (!ASSERT_OK_PTR(t->btf, "new_empty"))
goto err_out;
t->d = btf_dump__new(t->btf, btf_dump_printf, t->dump_buf_file, NULL);
if (!ASSERT_OK(libbpf_get_error(t->d), "btf_dump__new"))
goto err_out;
return 0;
err_out:
test_ctx__free(t);
return -1;
}
static void test_ctx__dump_and_compare(struct test_ctx *t,
const char *expected_output,
const char *message)
{
int i, err;
for (i = 1; i < btf__type_cnt(t->btf); i++) {
err = btf_dump__dump_type(t->d, i);
ASSERT_OK(err, "dump_type_ok");
}
fflush(t->dump_buf_file);
t->dump_buf[t->dump_buf_sz] = 0; /* some libc implementations don't do this */
ASSERT_STREQ(t->dump_buf, expected_output, message);
}
static void test_btf_dump_incremental(void)
{
struct btf *btf = NULL;
struct btf_dump *d = NULL;
int id, err, i;
struct test_ctx t = {};
struct btf *btf;
int id, err;
dump_buf_file = open_memstream(&dump_buf, &dump_buf_sz);
if (!ASSERT_OK_PTR(dump_buf_file, "dump_memstream"))
if (test_ctx__init(&t))
return;
btf = btf__new_empty();
if (!ASSERT_OK_PTR(btf, "new_empty"))
goto err_out;
d = btf_dump__new(btf, btf_dump_printf, dump_buf_file, NULL);
if (!ASSERT_OK(libbpf_get_error(d), "btf_dump__new"))
goto err_out;
btf = t.btf;
/* First, generate BTF corresponding to the following C code:
*
@ -182,15 +225,7 @@ static void test_btf_dump_incremental(void)
err = btf__add_field(btf, "x", 4, 0, 0);
ASSERT_OK(err, "field_ok");
for (i = 1; i < btf__type_cnt(btf); i++) {
err = btf_dump__dump_type(d, i);
ASSERT_OK(err, "dump_type_ok");
}
fflush(dump_buf_file);
dump_buf[dump_buf_sz] = 0; /* some libc implementations don't do this */
ASSERT_STREQ(dump_buf,
test_ctx__dump_and_compare(&t,
"enum x;\n"
"\n"
"enum x {\n"
@ -221,7 +256,7 @@ static void test_btf_dump_incremental(void)
* enum values don't conflict;
*
*/
fseek(dump_buf_file, 0, SEEK_SET);
fseek(t.dump_buf_file, 0, SEEK_SET);
id = btf__add_struct(btf, "s", 4);
ASSERT_EQ(id, 7, "struct_id");
@ -232,14 +267,7 @@ static void test_btf_dump_incremental(void)
err = btf__add_field(btf, "s", 6, 64, 0);
ASSERT_OK(err, "field_ok");
for (i = 1; i < btf__type_cnt(btf); i++) {
err = btf_dump__dump_type(d, i);
ASSERT_OK(err, "dump_type_ok");
}
fflush(dump_buf_file);
dump_buf[dump_buf_sz] = 0; /* some libc implementations don't do this */
ASSERT_STREQ(dump_buf,
test_ctx__dump_and_compare(&t,
"struct s___2 {\n"
" enum x x;\n"
" enum {\n"
@ -248,11 +276,53 @@ static void test_btf_dump_incremental(void)
" struct s s;\n"
"};\n\n" , "c_dump1");
err_out:
fclose(dump_buf_file);
free(dump_buf);
btf_dump__free(d);
btf__free(btf);
test_ctx__free(&t);
}
static void test_btf_dump_type_tags(void)
{
struct test_ctx t = {};
struct btf *btf;
int id, err;
if (test_ctx__init(&t))
return;
btf = t.btf;
/* Generate BTF corresponding to the following C code:
*
* struct s {
* void __attribute__((btf_type_tag(\"void_tag\"))) *p1;
* void __attribute__((void_attr)) *p2;
* };
*
*/
id = btf__add_type_tag(btf, "void_tag", 0);
ASSERT_EQ(id, 1, "type_tag_id");
id = btf__add_ptr(btf, id);
ASSERT_EQ(id, 2, "void_ptr_id1");
id = btf__add_type_attr(btf, "void_attr", 0);
ASSERT_EQ(id, 3, "type_attr_id");
id = btf__add_ptr(btf, id);
ASSERT_EQ(id, 4, "void_ptr_id2");
id = btf__add_struct(btf, "s", 8);
ASSERT_EQ(id, 5, "struct_id");
err = btf__add_field(btf, "p1", 2, 0, 0);
ASSERT_OK(err, "field_ok1");
err = btf__add_field(btf, "p2", 4, 0, 0);
ASSERT_OK(err, "field_ok2");
test_ctx__dump_and_compare(&t,
"struct s {\n"
" void __attribute__((btf_type_tag(\"void_tag\"))) *p1;\n"
" void __attribute__((void_attr)) *p2;\n"
"};\n\n", "dump_and_compare");
test_ctx__free(&t);
}
#define STRSIZE 4096
@ -874,6 +944,9 @@ void test_btf_dump() {
if (test__start_subtest("btf_dump: incremental"))
test_btf_dump_incremental();
if (test__start_subtest("btf_dump: type_tags"))
test_btf_dump_type_tags();
btf = libbpf_find_kernel_btf();
if (!ASSERT_OK_PTR(btf, "no kernel BTF found"))
return;

128
tools/testing/selftests/bpf/prog_tests/cgroup_preorder.c Normal file
View File

@ -0,0 +1,128 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
#include <test_progs.h>
#include "cgroup_helpers.h"
#include "cgroup_preorder.skel.h"
static int run_getsockopt_test(int cg_parent, int cg_child, int sock_fd, bool all_preorder)
{
LIBBPF_OPTS(bpf_prog_attach_opts, opts);
enum bpf_attach_type prog_c_atype, prog_c2_atype, prog_p_atype, prog_p2_atype;
int prog_c_fd, prog_c2_fd, prog_p_fd, prog_p2_fd;
struct cgroup_preorder *skel = NULL;
struct bpf_program *prog;
__u8 *result, buf;
socklen_t optlen;
int err = 0;
skel = cgroup_preorder__open_and_load();
if (!ASSERT_OK_PTR(skel, "cgroup_preorder__open_and_load"))
return 0;
buf = 0x00;
err = setsockopt(sock_fd, SOL_IP, IP_TOS, &buf, 1);
if (!ASSERT_OK(err, "setsockopt"))
goto close_skel;
opts.flags = BPF_F_ALLOW_MULTI;
if (all_preorder)
opts.flags |= BPF_F_PREORDER;
prog = skel->progs.child;
prog_c_fd = bpf_program__fd(prog);
prog_c_atype = bpf_program__expected_attach_type(prog);
err = bpf_prog_attach_opts(prog_c_fd, cg_child, prog_c_atype, &opts);
if (!ASSERT_OK(err, "bpf_prog_attach_opts-child"))
goto close_skel;
opts.flags = BPF_F_ALLOW_MULTI | BPF_F_PREORDER;
prog = skel->progs.child_2;
prog_c2_fd = bpf_program__fd(prog);
prog_c2_atype = bpf_program__expected_attach_type(prog);
err = bpf_prog_attach_opts(prog_c2_fd, cg_child, prog_c2_atype, &opts);
if (!ASSERT_OK(err, "bpf_prog_attach_opts-child_2"))
goto detach_child;
optlen = 1;
err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen);
if (!ASSERT_OK(err, "getsockopt"))
goto detach_child_2;
result = skel->bss->result;
if (all_preorder)
ASSERT_TRUE(result[0] == 1 && result[1] == 2, "child only");
else
ASSERT_TRUE(result[0] == 2 && result[1] == 1, "child only");
skel->bss->idx = 0;
memset(result, 0, 4);
opts.flags = BPF_F_ALLOW_MULTI;
if (all_preorder)
opts.flags |= BPF_F_PREORDER;
prog = skel->progs.parent;
prog_p_fd = bpf_program__fd(prog);
prog_p_atype = bpf_program__expected_attach_type(prog);
err = bpf_prog_attach_opts(prog_p_fd, cg_parent, prog_p_atype, &opts);
if (!ASSERT_OK(err, "bpf_prog_attach_opts-parent"))
goto detach_child_2;
opts.flags = BPF_F_ALLOW_MULTI | BPF_F_PREORDER;
prog = skel->progs.parent_2;
prog_p2_fd = bpf_program__fd(prog);
prog_p2_atype = bpf_program__expected_attach_type(prog);
err = bpf_prog_attach_opts(prog_p2_fd, cg_parent, prog_p2_atype, &opts);
if (!ASSERT_OK(err, "bpf_prog_attach_opts-parent_2"))
goto detach_parent;
err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen);
if (!ASSERT_OK(err, "getsockopt"))
goto detach_parent_2;
if (all_preorder)
ASSERT_TRUE(result[0] == 3 && result[1] == 4 && result[2] == 1 && result[3] == 2,
"parent and child");
else
ASSERT_TRUE(result[0] == 4 && result[1] == 2 && result[2] == 1 && result[3] == 3,
"parent and child");
detach_parent_2:
ASSERT_OK(bpf_prog_detach2(prog_p2_fd, cg_parent, prog_p2_atype),
"bpf_prog_detach2-parent_2");
detach_parent:
ASSERT_OK(bpf_prog_detach2(prog_p_fd, cg_parent, prog_p_atype),
"bpf_prog_detach2-parent");
detach_child_2:
ASSERT_OK(bpf_prog_detach2(prog_c2_fd, cg_child, prog_c2_atype),
"bpf_prog_detach2-child_2");
detach_child:
ASSERT_OK(bpf_prog_detach2(prog_c_fd, cg_child, prog_c_atype),
"bpf_prog_detach2-child");
close_skel:
cgroup_preorder__destroy(skel);
return err;
}
void test_cgroup_preorder(void)
{
int cg_parent = -1, cg_child = -1, sock_fd = -1;
cg_parent = test__join_cgroup("/parent");
if (!ASSERT_GE(cg_parent, 0, "join_cgroup /parent"))
goto out;
cg_child = test__join_cgroup("/parent/child");
if (!ASSERT_GE(cg_child, 0, "join_cgroup /parent/child"))
goto out;
sock_fd = socket(AF_INET, SOCK_STREAM, 0);
if (!ASSERT_GE(sock_fd, 0, "socket"))
goto out;
ASSERT_OK(run_getsockopt_test(cg_parent, cg_child, sock_fd, false), "getsockopt_test_1");
ASSERT_OK(run_getsockopt_test(cg_parent, cg_child, sock_fd, true), "getsockopt_test_2");
out:
close(sock_fd);
close(cg_child);
close(cg_parent);
}

13
tools/testing/selftests/bpf/prog_tests/cgroup_v1v2.c
View File

@ -10,12 +10,18 @@
static int run_test(int cgroup_fd, int server_fd, bool classid)
{
struct connect4_dropper *skel;
int fd, err = 0;
int fd, err = 0, port;
skel = connect4_dropper__open_and_load();
if (!ASSERT_OK_PTR(skel, "skel_open"))
return -1;
port = get_socket_local_port(server_fd);
if (!ASSERT_GE(port, 0, "get_socket_local_port"))
return -1;
skel->bss->port = ntohs(port);
skel->links.connect_v4_dropper =
bpf_program__attach_cgroup(skel->progs.connect_v4_dropper,
cgroup_fd);
@ -48,10 +54,9 @@ void test_cgroup_v1v2(void)
{
struct network_helper_opts opts = {};
int server_fd, client_fd, cgroup_fd;
static const int port = 60120;
/* Step 1: Check base connectivity works without any BPF. */
server_fd = start_server(AF_INET, SOCK_STREAM, NULL, port, 0);
server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0);
if (!ASSERT_GE(server_fd, 0, "server_fd"))
return;
client_fd = connect_to_fd_opts(server_fd, &opts);
@ -66,7 +71,7 @@ void test_cgroup_v1v2(void)
cgroup_fd = test__join_cgroup("/connect_dropper");
if (!ASSERT_GE(cgroup_fd, 0, "cgroup_fd"))
return;
server_fd = start_server(AF_INET, SOCK_STREAM, NULL, port, 0);
server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0);
if (!ASSERT_GE(server_fd, 0, "server_fd")) {
close(cgroup_fd);
return;

Some files were not shown because too many files have changed in this diff Show More