x86-64: Fix XMM8-XMM15 register encoding for memlocals

This patch fixes XMM8-XMM15 register encoding for memlocals on x86-64. I
noticed the problem while looking at assembly dumps for caller-save register
saving and restoring.

Signed-off-by: Pekka Enberg <penberg@kernel.org>

3 files changed, 32 insertions, 4 deletions

diff --git a/arch/x86/encode.c b/arch/x86/encode.c
index ff6a4117..0f281330 100644
--- a/arch/x86/encode.c
+++ b/arch/x86/encode.c
@@ -591,9 +591,6 @@ static uint8_t insn_rex_prefix(struct insn *self, uint64_t flags)
 {
 	uint8_t ret;
 
-	if (flags & (SRC_MEMLOCAL|DST_MEMLOCAL))
-		return 0;
-
 	ret	= insn_rex_operand_64(self, flags);
 
 	if (flags & DIR_REVERSED) {
diff --git a/arch/x86/include/arch/instruction.h b/arch/x86/include/arch/instruction.h
index 4dd16fb1..907d33ea 100644
--- a/arch/x86/include/arch/instruction.h
+++ b/arch/x86/include/arch/instruction.h
@@ -63,9 +63,9 @@ static inline bool operand_is_reg(struct operand *operand)
 	switch (operand->type) {
 	case OPERAND_MEMBASE:
 	case OPERAND_MEMINDEX:
-	case OPERAND_MEMLOCAL:
 	case OPERAND_REG:
 		return true;
+	case OPERAND_MEMLOCAL:
 	case OPERAND_MEMDISP:
 	case OPERAND_NONE:
 	case OPERAND_BRANCH:
diff --git a/test/unit/arch-x86/encode-test.c b/test/unit/arch-x86/encode-test.c
index e417d890..e663d6ca 100644
--- a/test/unit/arch-x86/encode-test.c
+++ b/test/unit/arch-x86/encode-test.c
@@ -628,6 +628,37 @@ void test_encoding_membase_xmm_high(void)
 #endif
 }
 
+void test_encoding_memlocal_xmm_high(void)
+{
+#ifdef CONFIG_X86_64
+	struct stack_frame frame = {
+		.nr_args	= 0,
+	};
+	struct stack_slot slot = {
+		.parent		= &frame,
+		.index		= 0,
+	};
+	uint8_t encoding[] = { 0xf3, 0x44, 0x0f, 0x10, 0x85, 0xf8, 0xff, 0xff, 0xff };
+	struct insn insn = { };
+
+	setup();
+
+	/* movss  -0x8(%rbp),%xmm8 */
+	insn.type			= INSN_MOVSS_MEMLOCAL_XMM;
+	insn.src.slot			= &slot;
+	insn.src.type			= OPERAND_MEMLOCAL;
+	insn.dest.reg.interval		= &reg_xmm8;
+	insn.dest.type			= OPERAND_REG;
+
+	insn_encode(&insn, buffer, NULL);
+
+	assert_int_equals(ARRAY_SIZE(encoding), buffer_offset(buffer));
+	assert_mem_equals(encoding, buffer_ptr(buffer), ARRAY_SIZE(encoding));
+
+	teardown();
+#endif
+}
+
 void test_encoding_membase_xmm_high_disp8(void)
 {
 #ifdef CONFIG_X86_64