需要额外加上:

proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";

例如：

location / {
        proxy_pass http://localhost;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "Upgrade";
    }

实际使用：

http {
    upstream wsupstream  {
        server 127.0.0.1:7397;
    }
    server {
        location ^~/websocket {
            location ^~/websocket {
            proxy_pass http://wsupstream ;
            proxy_connect_timeout 4s;
            proxy_read_timeout 3600s; 
            proxy_send_timeout 12s; 
            proxy_http_version 1.1;
            proxy_redirect    off;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header Host $host;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";
        }
        }
    }
}

亲测可用，有问题可以讨论。