365 lines
13 KiB
Java
365 lines
13 KiB
Java
/* Copyright (C) 2022 Dinglan Peng
|
|
*
|
|
* This is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This software is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this software; if not, write to the Free Software
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
|
|
* USA.
|
|
*/
|
|
|
|
package com.tigervnc.rfb;
|
|
|
|
import java.io.UnsupportedEncodingException;
|
|
import java.math.BigInteger;
|
|
import java.nio.ByteBuffer;
|
|
import java.security.InvalidKeyException;
|
|
import java.security.KeyFactory;
|
|
import java.security.KeyPair;
|
|
import java.security.KeyPairGenerator;
|
|
import java.security.MessageDigest;
|
|
import java.security.NoSuchAlgorithmException;
|
|
import java.security.PrivateKey;
|
|
import java.security.PublicKey;
|
|
import java.security.SecureRandom;
|
|
import java.security.interfaces.RSAPublicKey;
|
|
import java.security.spec.InvalidKeySpecException;
|
|
import java.security.spec.RSAPublicKeySpec;
|
|
import java.util.Arrays;
|
|
|
|
import javax.crypto.BadPaddingException;
|
|
import javax.crypto.Cipher;
|
|
import javax.crypto.IllegalBlockSizeException;
|
|
import javax.crypto.NoSuchPaddingException;
|
|
import javax.swing.JOptionPane;
|
|
|
|
import com.tigervnc.rdr.*;
|
|
import com.tigervnc.vncviewer.*;
|
|
|
|
public class CSecurityRSAAES extends CSecurity {
|
|
|
|
private static final int MinKeyLength = 1024;
|
|
private static final int MaxKeyLength = 8192;
|
|
|
|
private static final byte[] bigIntToBytes(BigInteger n, int bytes) {
|
|
int bits = n.bitCount();
|
|
byte[] arr = n.toByteArray();
|
|
//int len = bits % 8 == 0 ? arr.length - 1 : arr.length;
|
|
//System.out.printf("%d %d %d\n", bits, len, bytes);
|
|
//if (len > bytes)
|
|
// return null;
|
|
int len = arr.length < bytes ? arr.length : bytes;
|
|
byte[] res = new byte[bytes];
|
|
System.arraycopy(arr, arr.length - len, res, bytes - len, len);
|
|
return res;
|
|
}
|
|
|
|
public CSecurityRSAAES(int secType, int keySize, boolean isAllEncrypted) {
|
|
this.secType = secType;
|
|
this.keySize = keySize;
|
|
this.isAllEncrypted = isAllEncrypted;
|
|
}
|
|
|
|
public boolean processMsg(CConnection cc) {
|
|
readPubclicKey(cc);
|
|
verifyServer();
|
|
writePublicKey(cc);
|
|
writeRandom(cc);
|
|
readRandom(cc);
|
|
setCipher(cc);
|
|
writeHash(cc);
|
|
readHash();
|
|
readSubtype();
|
|
writeCredentials();
|
|
return true;
|
|
}
|
|
|
|
private void readPubclicKey(CConnection cc) {
|
|
InStream is = cc.getInStream();
|
|
serverKeyLength = is.readU32();
|
|
if (serverKeyLength < MinKeyLength)
|
|
throw new AuthFailureException("server key is too short");
|
|
if (serverKeyLength > MaxKeyLength)
|
|
throw new AuthFailureException("server key is too long");
|
|
int size = (serverKeyLength + 7) / 8;
|
|
serverKeyN = new byte[size];
|
|
serverKeyE = new byte[size];
|
|
is.readBytes(ByteBuffer.wrap(serverKeyN), size);
|
|
is.readBytes(ByteBuffer.wrap(serverKeyE), size);
|
|
BigInteger modulus = new BigInteger(1, serverKeyN);
|
|
BigInteger publicExponent = new BigInteger(1, serverKeyE);
|
|
RSAPublicKeySpec spec = new RSAPublicKeySpec(modulus, publicExponent);
|
|
try {
|
|
KeyFactory factory = KeyFactory.getInstance("RSA");
|
|
serverKey = factory.generatePublic(spec);
|
|
} catch (NoSuchAlgorithmException e) {
|
|
throw new AuthFailureException("RSA algorithm is not supported");
|
|
} catch (InvalidKeySpecException e) {
|
|
throw new AuthFailureException("server key is invalid");
|
|
}
|
|
}
|
|
|
|
private void verifyServer() {
|
|
MessageDigest digest;
|
|
try {
|
|
digest = MessageDigest.getInstance("SHA-1");
|
|
} catch (NoSuchAlgorithmException e) {
|
|
throw new AuthFailureException("SHA-1 algorithm is not supported");
|
|
}
|
|
byte[] length = new byte[4];
|
|
length[0] = (byte) ((serverKeyLength & 0xff000000) >> 24);
|
|
length[1] = (byte) ((serverKeyLength & 0xff0000) >> 16);
|
|
length[2] = (byte) ((serverKeyLength & 0xff00) >> 8);
|
|
length[3] = (byte) (serverKeyLength & 0xff);
|
|
digest.update(length);
|
|
digest.update(serverKeyN);
|
|
digest.update(serverKeyE);
|
|
byte[] f = digest.digest();
|
|
String title = "Server key fingerprint";
|
|
String text = String.format(
|
|
"The server has provided the following identifying information:\n" +
|
|
"Fingerprint: %02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x\n" +
|
|
"Please verify that the information is correct and press \"Yes\". " +
|
|
"Otherwise press \"No\"", f[0], f[1], f[2], f[3], f[4], f[5], f[6], f[7]);
|
|
if (!msg.showMsgBox(JOptionPane.YES_NO_OPTION, title, text))
|
|
throw new AuthFailureException("server key mismatch");
|
|
}
|
|
|
|
private void writePublicKey(CConnection cc) {
|
|
OutStream os = cc.getOutStream();
|
|
clientKeyLength = serverKeyLength;
|
|
KeyPairGenerator kpg;
|
|
try {
|
|
kpg = KeyPairGenerator.getInstance("RSA");
|
|
} catch (NoSuchAlgorithmException e) {
|
|
throw new AuthFailureException("RSA algorithm is not supported");
|
|
}
|
|
kpg.initialize(clientKeyLength);
|
|
KeyPair kp = kpg.generateKeyPair();
|
|
clientKey = kp.getPrivate();
|
|
clientPublicKey = kp.getPublic();
|
|
RSAPublicKey rsaKey = (RSAPublicKey) clientPublicKey;
|
|
BigInteger modulus = rsaKey.getModulus();
|
|
BigInteger publicExponent = rsaKey.getPublicExponent();
|
|
|
|
clientKeyN = bigIntToBytes(modulus, (clientKeyLength + 7) / 8);
|
|
clientKeyE = bigIntToBytes(publicExponent, (clientKeyLength + 7) / 8);
|
|
if (clientKeyN == null || clientKeyN == null) {
|
|
throw new AuthFailureException("failed to generate RSA keys");
|
|
}
|
|
os.writeU32(clientKeyLength);
|
|
os.writeBytes(clientKeyN, 0, clientKeyN.length);
|
|
os.writeBytes(clientKeyE, 0, clientKeyE.length);
|
|
os.flush();
|
|
}
|
|
|
|
private void writeRandom(CConnection cc) {
|
|
OutStream os = cc.getOutStream();
|
|
SecureRandom sr = new SecureRandom();
|
|
clientRandom = new byte[keySize / 8];
|
|
sr.nextBytes(clientRandom);
|
|
byte[] encrypted;
|
|
try {
|
|
Cipher cipher = Cipher.getInstance("RSA/ECB/PKCS1Padding");
|
|
cipher.init(Cipher.ENCRYPT_MODE, serverKey);
|
|
encrypted = cipher.doFinal(clientRandom);
|
|
} catch (NoSuchAlgorithmException | NoSuchPaddingException e) {
|
|
throw new AuthFailureException("RSA algorithm is not supported");
|
|
} catch (InvalidKeyException | IllegalBlockSizeException |
|
|
BadPaddingException e) {
|
|
throw new AuthFailureException("failed to encrypt random");
|
|
}
|
|
os.writeU16(encrypted.length);
|
|
os.writeBytes(encrypted, 0, encrypted.length);
|
|
os.flush();
|
|
}
|
|
|
|
private void readRandom(CConnection cc) {
|
|
InStream is = cc.getInStream();
|
|
int size = is.readU16();
|
|
if (size != clientKeyN.length)
|
|
throw new AuthFailureException("client key length doesn't match");
|
|
byte[] buffer = new byte[size];
|
|
is.readBytes(ByteBuffer.wrap(buffer), size);
|
|
try {
|
|
Cipher cipher = Cipher.getInstance("RSA/ECB/PKCS1Padding");
|
|
cipher.init(Cipher.DECRYPT_MODE, clientKey);
|
|
serverRandom = cipher.doFinal(buffer);
|
|
} catch (NoSuchAlgorithmException | NoSuchPaddingException e) {
|
|
throw new AuthFailureException("RSA algorithm is not supported");
|
|
} catch (InvalidKeyException | IllegalBlockSizeException |
|
|
BadPaddingException e) {
|
|
System.out.println(e.getMessage());
|
|
throw new AuthFailureException("failed to decrypt server random");
|
|
}
|
|
if (serverRandom.length != keySize / 8)
|
|
throw new AuthFailureException("server random length doesn't match");
|
|
}
|
|
|
|
private void setCipher(CConnection cc) {
|
|
rawis = cc.getInStream();
|
|
rawos = cc.getOutStream();
|
|
MessageDigest digest;
|
|
try {
|
|
digest = MessageDigest.getInstance(keySize == 128 ? "SHA-1" : "SHA-256");
|
|
} catch (NoSuchAlgorithmException e) {
|
|
throw new AuthFailureException("hash algorithm is not supported");
|
|
}
|
|
digest.update(clientRandom);
|
|
digest.update(serverRandom);
|
|
byte[] key = Arrays.copyOfRange(digest.digest(), 0, keySize / 8);
|
|
rais = new AESInStream(rawis, key);
|
|
digest.reset();
|
|
digest.update(serverRandom);
|
|
digest.update(clientRandom);
|
|
key = Arrays.copyOfRange(digest.digest(), 0, keySize / 8);
|
|
raos = new AESOutStream(rawos, key);
|
|
if (isAllEncrypted)
|
|
cc.setStreams(rais, raos);
|
|
}
|
|
|
|
private void writeHash(CConnection cc) {
|
|
MessageDigest digest;
|
|
try {
|
|
digest = MessageDigest.getInstance(keySize == 128 ? "SHA-1" : "SHA-256");
|
|
} catch (NoSuchAlgorithmException e) {
|
|
throw new AuthFailureException("hash algorithm is not supported");
|
|
}
|
|
int len = serverKeyLength;
|
|
byte[] lenServerKey = new byte[] {
|
|
(byte) ((len & 0xff000000) >> 24),
|
|
(byte) ((len & 0xff0000) >> 16),
|
|
(byte) ((len & 0xff00) >> 8),
|
|
(byte) (len & 0xff)
|
|
};
|
|
len = clientKeyLength;
|
|
byte[] lenClientKey = new byte[] {
|
|
(byte) ((len & 0xff000000) >> 24),
|
|
(byte) ((len & 0xff0000) >> 16),
|
|
(byte) ((len & 0xff00) >> 8),
|
|
(byte) (len & 0xff)
|
|
};
|
|
digest.update(lenClientKey);
|
|
digest.update(clientKeyN);
|
|
digest.update(clientKeyE);
|
|
digest.update(lenServerKey);
|
|
digest.update(serverKeyN);
|
|
digest.update(serverKeyE);
|
|
byte[] hash = digest.digest();
|
|
raos.writeBytes(hash, 0, hash.length);
|
|
raos.flush();
|
|
}
|
|
|
|
void readHash() {
|
|
MessageDigest digest;
|
|
try {
|
|
digest = MessageDigest.getInstance(keySize == 128 ? "SHA-1" : "SHA-256");
|
|
} catch (NoSuchAlgorithmException e) {
|
|
throw new AuthFailureException("hash algorithm is not supported");
|
|
}
|
|
int len = serverKeyLength;
|
|
byte[] lenServerKey = new byte[] {
|
|
(byte)((len & 0xff000000) >> 24),
|
|
(byte)((len & 0xff0000) >> 16),
|
|
(byte)((len & 0xff00) >> 8),
|
|
(byte)(len & 0xff)
|
|
};
|
|
len = clientKeyLength;
|
|
byte[] lenClientKey = new byte[] {
|
|
(byte)((len & 0xff000000) >> 24),
|
|
(byte)((len & 0xff0000) >> 16),
|
|
(byte)((len & 0xff00) >> 8),
|
|
(byte)(len & 0xff)
|
|
};
|
|
digest.update(lenServerKey);
|
|
digest.update(serverKeyN);
|
|
digest.update(serverKeyE);
|
|
digest.update(lenClientKey);
|
|
digest.update(clientKeyN);
|
|
digest.update(clientKeyE);
|
|
byte[] realHash = digest.digest();
|
|
ByteBuffer hash = ByteBuffer.allocate(realHash.length);
|
|
rais.readBytes(hash, realHash.length);
|
|
if (!Arrays.equals(hash.array(), realHash)) {
|
|
throw new AuthFailureException("hash doesn't match");
|
|
}
|
|
}
|
|
|
|
private void readSubtype() {
|
|
subtype = rais.readU8();
|
|
if (subtype != Security.secTypeRA2UserPass &&
|
|
subtype != Security.secTypeRA2Pass)
|
|
throw new AuthFailureException("unknown RSA-AES subtype");
|
|
}
|
|
|
|
private void writeCredentials() {
|
|
StringBuffer username = new StringBuffer();
|
|
StringBuffer password = new StringBuffer();
|
|
CSecurity.upg.getUserPasswd(secType == Security.secTypeRA256,
|
|
subtype == Security.secTypeRA2UserPass ?
|
|
username : null,
|
|
password);
|
|
if (username.length() > 255)
|
|
throw new AuthFailureException("username is too long");
|
|
byte[] usernameBytes;
|
|
try {
|
|
usernameBytes = username.toString().getBytes("UTF-8");
|
|
} catch (UnsupportedEncodingException e) {
|
|
throw new AuthFailureException("UTF-8 is not supported");
|
|
}
|
|
raos.writeU8(usernameBytes.length);
|
|
if (usernameBytes.length != 0)
|
|
raos.writeBytes(usernameBytes, 0, usernameBytes.length);
|
|
if (password.length() > 255)
|
|
throw new AuthFailureException("password is too long");
|
|
byte[] passwordBytes;
|
|
try {
|
|
passwordBytes = password.toString().getBytes("UTF-8");
|
|
} catch (UnsupportedEncodingException e) {
|
|
throw new AuthFailureException("UTF-8 is not supported");
|
|
}
|
|
raos.writeU8(passwordBytes.length);
|
|
if (passwordBytes.length != 0)
|
|
raos.writeBytes(passwordBytes, 0, passwordBytes.length);
|
|
raos.flush();
|
|
}
|
|
|
|
public int getType() {
|
|
return secType;
|
|
}
|
|
|
|
public String description() {
|
|
return "RSA-ASE security types";
|
|
}
|
|
|
|
private int secType;
|
|
private int subtype;
|
|
private int keySize;
|
|
private boolean isAllEncrypted;
|
|
|
|
private PrivateKey clientKey;
|
|
private PublicKey clientPublicKey;
|
|
private PublicKey serverKey;
|
|
private int serverKeyLength;
|
|
private byte[] serverKeyN;
|
|
private byte[] serverKeyE;
|
|
private int clientKeyLength;
|
|
private byte[] clientKeyN;
|
|
private byte[] clientKeyE;
|
|
private byte[] serverRandom;
|
|
private byte[] clientRandom;
|
|
|
|
private AESInStream rais;
|
|
private AESOutStream raos;
|
|
private InStream rawis;
|
|
private OutStream rawos;
|
|
}
|