main函数校验flag长度是否为16位，若是则依次对flag加密后，而后若flag正确则输出常量"Bingo!"(此处有多解)

if ( v6 > 0x10 )                              // flag.len == 16
  {
    puts("input is too long!");
  }
  else if ( v6 == 16 )
  {
    v7 = sub_55A527C5BC00((unsigned __int64)&flag, 16, &ptr);// v7 == 8，ptr为flag 变换后的八个字符
    if ( v7
      && (v8 = sub_55A527C5C180(ptr, v7, (__int64)&::a3, 16, &dest), (v9 = (char *)v8) != 0LL)
      && dest > 0
      && (unsigned __int16)sub_55A527C5C3D0((__int64)v8, dest) == 0x69E2 )
    {
      for ( i = 0LL; dest > (signed int)i; ++i )
        v9[i] ^= 0x17u;
      puts(v9);
      if ( ptr )
        free(ptr);
      free(v9);
    }
    else
    {
      puts("input flag is wrong!");
    }
  }
  else
  {
    puts("input is too short!");
  }

第一个函数很简单，主要是对flag通过置换表进行置换，如 ’ab' 则 置换为 0xab
由于小端序，所以如果我们输入是'1234567890123456'（128bits）那么经过置换后变成一个64位的整数 0x5634129078563412（64bits）（由于置换表的缘故此处也存在多解，譬如 'f' 'F' '~' 均被置换为0xF）
第二个函数是对flag置换后形成的64位整数进行32轮的轮换加密形成一个新的64位整数X ,然后校验高8位是否小于4，并将BYTE(BYTE7(X))(X)置0
轮换加密函数很长，但是由于a2 < -1且 v27 <= 2 所以只需分析那一小部分即可

if ( a2 < -1 )
  {
    v24 = -a2;
    v25 = 0x9E3779B9 * (52 / v24 + 6);          // v24 == 2
    if ( v25 )
    {
      srch = &src[v24 - 1];
      v27 = ~v4;
      v44 = &src[~v4];
      v28 = ~v4 - 2 - ((~v4 - 3) & 0xFFFFFFFE);
      do
      {
        v29 = v25 >> 2;
        if ( v27 <= 2 )                         // true
        {
          v31 = v27;
        }
        else                                    // false
        {
              略
        }
        srcl_ = &src[v31];
        do
        {
          srcl = *(srcl_ - 1);
          --srcl_;
          srch_next = srcl_[1]
                    - (((srch_next ^ v25)
                      + (srcl ^ *(_DWORD *)(tables + 4LL * (((unsigned __int8)v29 ^ (unsigned __int8)v31) & 3)))) ^ (((srch_next >> 3) ^ 16 * srcl) + ((srcl >> 5) ^ 4 * srch_next)));
          srcl_[1] = srch_next;                 // 后八位0x56341290替换为 0x45d10f13 小端
          --v31;
        }
        while ( v31 );
        srcl_next = *src
                  - (((((unsigned int)*srch >> 5) ^ 4 * srch_next) + (16 * *srch ^ (srch_next >> 3))) ^ ((*(_DWORD *)(tables + 4LL * (v29 & 3)) ^ *srch) + (v25 ^ srch_next)));
        v42 = v25 == 0x9E3779B9;
        v25 += 0x61C88647;
        srch_next = srcl_next;
        *src = srcl_next;                       // 前四位0x78654321替换为  0x35158711 小端
      }
      while ( !v42 );
    }
    return 0LL;
  }
  return result;
}

第三个函数是一些位操作，并且验证返回值是否为 0x69E2，如果是则将X按字节从低到高异或0x17输出
题目提示输出'Bingo!'，所以将其异或回去得到一个长度为6的byte数组，（由于小端序所以是64位整数的低48位）合理猜测高8位为0x02，所以只需爆破次高8位数据即可（由此构造一个64位数经前两轮解密而后动态调试验证第三个函数返回了0x69E2且输出'Bingo!'）
题目提示 md5(rctf{your answer}) == 5f8243a662cf71bf31d2b2602638dc1d
由此可以爆破flag了，算了一下大概需要 0xff*(2^n+km)次运算(n取决于flag经过第一轮置换字母的数量，km取决了flag经过第一轮置换f的数量，k是常量)
权衡了一下先爆破第二轮加密过程的多解，第一轮默认为小写,且0xf均视为'f'
幸运的是爆破到第二个flag就出来了
rctf{05e8a376e4e0446e}
加解密脚本如下

#include<iostream>
#include<string>
#include <openssl/md5.h>
#include<cstdio>
#include<cstring>
using namespace std;
// -lcrypto 
string MD5(const string& src )
{
    MD5_CTX ctx;

    string md5_string;
    unsigned char md[16] = { 0 };
    char tmp[33] = { 0 };

    MD5_Init( &ctx );
    MD5_Update( &ctx, src.c_str(), src.size() );
    MD5_Final( md, &ctx );

    for( int i = 0; i < 16; ++i )
    {   
        memset( tmp, 0x00, sizeof( tmp ) );
        sprintf( tmp, "%02X", md[i] );
        md5_string += tmp;
    }   
    return md5_string;
}
// encrypt01
unsigned long long encrypt01 (string str)
{
    if (str.length() != 16) return 0;
    unsigned char byte_5626AE9EB620[55] = {
        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 
        0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 
        0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 
        0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F
    };
    unsigned long long res = 0;
    unsigned long long ptr = 0;
    for (int i = 0; i < str.length(); i+=2)
    {
        unsigned char a = byte_5626AE9EB620[str[i] - 0x30];
        unsigned char b = byte_5626AE9EB620[str[i+1] - 0x30];
        ptr = (a << 4) + b;
        ptr <<= 4*i;
        res += ptr;
    }
    return res;
}

string decrypt01 (unsigned long long src)
{
    unsigned char byte_5626AE9EB620[55] = {
        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 
        0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 
        0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 
        0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F
    };
    char chars [16]={0};
    unsigned char dest = 0xff;
    for (int i = 0; i < 16; i+=2)
    {
        unsigned char ch = src & dest ;
        src >>= 8;
        if (((ch & 0xf0) >> 4) <= 9)
            chars[i] = ((ch & 0xf0) >> 4) + 0x30;
        else
            chars[i] = ((ch & 0xf0) >> 4) + 0x57;
        if ((ch & 0x0f) <= 9)
            chars[i+1] = (ch & 0x0f) + 0x30;
        else
            chars[i+1] = (ch & 0x0f) + 0x57;
    }
    return chars;
}

// encrypt02
unsigned long long  encrypt02 (unsigned long long src){
    unsigned int v25 = (0x9E3779B9 * (52 / 2 + 6) & 0xffffffff);
    unsigned int l = src & 0xffffffff;
    unsigned int r = (src >> 32) & 0xffffffff;
    int i = 0;
    bool flag = 0;
    do{
        int v31 = 1;
        unsigned int v29 = v25 >> 2;
        int table[] = {0xE0C7E0C7, 0xC6F1D3D7, 0xC6D3C6D3, 0xC4D0D2CE};
        r = r - (((l^v25) + (l^table[(v29^v31)&3])) ^ (((l>>3)^16*l) + ((l>>5)^4*l)));
        l = l - (((v25^r) + (r^table[(v29&3)])) ^ (((r>>5)^4*r) + (16*r^(r>>3))));
        flag = v25 == 0x9E3779B9;
        v25 += 0x61C88647;
    }while (!flag);
    unsigned long long res = ((unsigned long long)r << 32) + l;
    return res;
}


//decrypt02
unsigned long long decrypt02(unsigned long long dest){
    unsigned int v25 = 0x9E3779B9;
    unsigned int l = dest & 0xffffffff;
    unsigned int r = (dest >> 32) & 0xffffffff;
    bool flag = 0;
    do{ 
        int v31 = 1;
        unsigned int v29 = v25 >> 2;
        int table[] = {0xE0C7E0C7, 0xC6F1D3D7, 0xC6D3C6D3, 0xC4D0D2CE};
        l = l + (((v25^r) + (r^table[(v29&3)])) ^ (((r>>5)^4*r) + (16*r^(r>>3))));
        r = r + (((l^v25) + (l^table[(v29^v31)&3])) ^ (((l>>3)^16*l) + ((l>>5)^4*l)));
        flag = v25 ==(0x9E3779B9 * (52 / 2 + 6) & 0xffffffff);
        v25 -= 0x61C88647;
    }while(!flag);
    unsigned long long res = ((unsigned long long)r << 32) + l;
    return res;
}

int main(){
    //BYTE7(crack) == 02
    unsigned long long crack = (unsigned long long)0x02 << 56;
    string str = "Bingo!";
    int i = str.length();
    while(i--) crack += ((unsigned long long)(str[i]^0x17) << (i * 8));

    string flagmd5 = "5F8243A662CF71BF31D2B2602638DC1D";

    for (unsigned long long  i =  0x0; i <= 0xff; i+=0x1)
    {                          
        crack += (unsigned long long)0x1 << 48;
        cout << hex << crack << endl;
        string flag = "";
        flag += "rctf{";
        flag += decrypt01(decrypt02(crack));
        flag += "}";
        if (MD5(flag) == flagmd5)
        {
            cout << "success!" << endl;
            cout << flag << endl;
            return 0;
        }
    }

    //encrypt03
    // int v11 = r >> 24;
    // if (v11 <= 4)//true
    // {
    //  int v13 = 8 - v11;//v13 == 6
    //  //dest = v13
    //  //将src[v13]置0
    // }

    //encrypt04
    //i=0
    //v13 = 8 - 2
    //依次将v[i++]二进制逆序，循环进行下列level1算法直到v[v13-1],算出最终结果为0x4796

    //encrytp05
    //移位操作0x4796 -> 0x69e2

    return 0;
}