original_kernel/arch/arm64/kernel/patch-scs.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2022 - Google LLC
 * Author: Ard Biesheuvel <ardb@google.com>
 */

#include <linux/bug.h>
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/linkage.h>
#include <linux/printk.h>
#include <linux/types.h>

#include <asm/cacheflush.h>
#include <asm/scs.h>

//
// This minimal DWARF CFI parser is partially based on the code in
// arch/arc/kernel/unwind.c, and on the document below:
// https://refspecs.linuxbase.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html
//

#define DW_CFA_nop                          0x00
#define DW_CFA_set_loc                      0x01
#define DW_CFA_advance_loc1                 0x02
#define DW_CFA_advance_loc2                 0x03
#define DW_CFA_advance_loc4                 0x04
#define DW_CFA_offset_extended              0x05
#define DW_CFA_restore_extended             0x06
#define DW_CFA_undefined                    0x07
#define DW_CFA_same_value                   0x08
#define DW_CFA_register                     0x09
#define DW_CFA_remember_state               0x0a
#define DW_CFA_restore_state                0x0b
#define DW_CFA_def_cfa                      0x0c
#define DW_CFA_def_cfa_register             0x0d
#define DW_CFA_def_cfa_offset               0x0e
#define DW_CFA_def_cfa_expression           0x0f
#define DW_CFA_expression                   0x10
#define DW_CFA_offset_extended_sf           0x11
#define DW_CFA_def_cfa_sf                   0x12
#define DW_CFA_def_cfa_offset_sf            0x13
#define DW_CFA_val_offset                   0x14
#define DW_CFA_val_offset_sf                0x15
#define DW_CFA_val_expression               0x16
#define DW_CFA_lo_user                      0x1c
#define DW_CFA_negate_ra_state              0x2d
#define DW_CFA_GNU_args_size                0x2e
#define DW_CFA_GNU_negative_offset_extended 0x2f
#define DW_CFA_hi_user                      0x3f

extern const u8 __eh_frame_start[], __eh_frame_end[];

enum {
	PACIASP		= 0xd503233f,
	AUTIASP		= 0xd50323bf,
	SCS_PUSH	= 0xf800865e,
	SCS_POP		= 0xf85f8e5e,
};

static void __always_inline scs_patch_loc(u64 loc)
{
	u32 insn = le32_to_cpup((void *)loc);

	switch (insn) {
	case PACIASP:
		*(u32 *)loc = cpu_to_le32(SCS_PUSH);
		break;
	case AUTIASP:
		*(u32 *)loc = cpu_to_le32(SCS_POP);
		break;
	default:
		/*
		 * While the DW_CFA_negate_ra_state directive is guaranteed to
		 * appear right after a PACIASP/AUTIASP instruction, it may
		 * also appear after a DW_CFA_restore_state directive that
		 * restores a state that is only partially accurate, and is
		 * followed by DW_CFA_negate_ra_state directive to toggle the
		 * PAC bit again. So we permit other instructions here, and ignore
		 * them.
		 */
		return;
	}
	dcache_clean_pou(loc, loc + sizeof(u32));
}

/*
 * Skip one uleb128/sleb128 encoded quantity from the opcode stream. All bytes
 * except the last one have bit #7 set.
 */
static int __always_inline skip_xleb128(const u8 **opcode, int size)
{
	u8 c;

	do {
		c = *(*opcode)++;
		size--;
	} while (c & BIT(7));

	return size;
}

struct eh_frame {
	/*
	 * The size of this frame if 0 < size < U32_MAX, 0 terminates the list.
	 */
	u32	size;

	/*
	 * The first frame is a Common Information Entry (CIE) frame, followed
	 * by one or more Frame Description Entry (FDE) frames. In the former
	 * case, this field is 0, otherwise it is the negated offset relative
	 * to the associated CIE frame.
	 */
	u32	cie_id_or_pointer;

	union {
		struct { // CIE
			u8	version;
			u8	augmentation_string[];
		};

		struct { // FDE
			s32	initial_loc;
			s32	range;
			u8	opcodes[];
		};
	};
};

static int noinstr scs_handle_fde_frame(const struct eh_frame *frame,
					bool fde_has_augmentation_data,
					int code_alignment_factor,
					bool dry_run)
{
	int size = frame->size - offsetof(struct eh_frame, opcodes) + 4;
	u64 loc = (u64)offset_to_ptr(&frame->initial_loc);
	const u8 *opcode = frame->opcodes;

	if (fde_has_augmentation_data) {
		int l;

		// assume single byte uleb128_t
		if (WARN_ON(*opcode & BIT(7)))
			return -ENOEXEC;

		l = *opcode++;
		opcode += l;
		size -= l + 1;
	}

	/*
	 * Starting from 'loc', apply the CFA opcodes that advance the location
	 * pointer, and identify the locations of the PAC instructions.
	 */
	while (size-- > 0) {
		switch (*opcode++) {
		case DW_CFA_nop:
		case DW_CFA_remember_state:
		case DW_CFA_restore_state:
			break;

		case DW_CFA_advance_loc1:
			loc += *opcode++ * code_alignment_factor;
			size--;
			break;

		case DW_CFA_advance_loc2:
			loc += *opcode++ * code_alignment_factor;
			loc += (*opcode++ << 8) * code_alignment_factor;
			size -= 2;
			break;

		case DW_CFA_def_cfa:
		case DW_CFA_offset_extended:
			size = skip_xleb128(&opcode, size);
			fallthrough;
		case DW_CFA_def_cfa_offset:
		case DW_CFA_def_cfa_offset_sf:
		case DW_CFA_def_cfa_register:
		case DW_CFA_same_value:
		case DW_CFA_restore_extended:
		case 0x80 ... 0xbf:
			size = skip_xleb128(&opcode, size);
			break;

		case DW_CFA_negate_ra_state:
			if (!dry_run)
				scs_patch_loc(loc - 4);
			break;

		case 0x40 ... 0x7f:
			// advance loc
			loc += (opcode[-1] & 0x3f) * code_alignment_factor;
			break;

		case 0xc0 ... 0xff:
			break;

		default:
			pr_err("unhandled opcode: %02x in FDE frame %lx\n", opcode[-1], (uintptr_t)frame);
			return -ENOEXEC;
		}
	}
	return 0;
}

int noinstr scs_patch(const u8 eh_frame[], int size)
{
	const u8 *p = eh_frame;

	while (size > 4) {
		const struct eh_frame *frame = (const void *)p;
		bool fde_has_augmentation_data = true;
		int code_alignment_factor = 1;
		int ret;

		if (frame->size == 0 ||
		    frame->size == U32_MAX ||
		    frame->size > size)
			break;

		if (frame->cie_id_or_pointer == 0) {
			const u8 *p = frame->augmentation_string;

			/* a 'z' in the augmentation string must come first */
			fde_has_augmentation_data = *p == 'z';

			/*
			 * The code alignment factor is a uleb128 encoded field
			 * but given that the only sensible values are 1 or 4,
			 * there is no point in decoding the whole thing.
			 */
			p += strlen(p) + 1;
			if (!WARN_ON(*p & BIT(7)))
				code_alignment_factor = *p;
		} else {
			ret = scs_handle_fde_frame(frame,
						   fde_has_augmentation_data,
						   code_alignment_factor,
						   true);
			if (ret)
				return ret;
			scs_handle_fde_frame(frame, fde_has_augmentation_data,
					     code_alignment_factor, false);
		}

		p += sizeof(frame->size) + frame->size;
		size -= sizeof(frame->size) + frame->size;
	}
	return 0;
}

asmlinkage void __init scs_patch_vmlinux(void)
{
	if (!should_patch_pac_into_scs())
		return;

	WARN_ON(scs_patch(__eh_frame_start, __eh_frame_end - __eh_frame_start));
	icache_inval_all_pou();
	isb();
}