refactor: secureUtils fingerprint method

Signed-off-by: sithlord48 <sithlord48@gmail.com>

src/lib/gui/CMakeLists.txt

@@ -113,5 +113,6 @@ if(WIN32)
     ${target}
     base
     arch
+    net
   )
 endif()

src/lib/net/CMakeLists.txt

@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: MIT
 
 add_library(net STATIC
+  FingerprintTypes.h
   IDataSocket.cpp
   IDataSocket.h
   IListenSocket.h
@@ -53,5 +54,8 @@ target_link_libraries(
   PRIVATE mt io)
 
 if(WIN32)
-  target_link_libraries(net PRIVATE Crypt32 ws2_32)
+  target_link_libraries(
+    net
+    PUBLIC OpenSSL::applink
+    PRIVATE Crypt32 ws2_32 OpenSSL::applink)
 endif()

src/lib/net/FingerprintTypes.h

@@ -0,0 +1,18 @@
+/*
+ * Deskflow -- mouse and keyboard sharing utility
+ * SPDX-FileCopyrightText: (C) 2025 Deskflow Developers
+ * SPDX-License-Identifier: GPL-2.0-only WITH LicenseRef-OpenSSL-Exception
+ */
+
+#pragma once
+
+namespace deskflow {
+
+enum FingerprintType
+{
+  Invalid,
+  SHA1,
+  SHA256
+};
+
+}

src/lib/net/SecureSocket.cpp

@@ -1,5 +1,6 @@
 /*
  * Deskflow -- mouse and keyboard sharing utility
+ * SPDX-FileCopyrightText: (C) 2025 Deskflow Developers
  * SPDX-FileCopyrightText: (C) 2015 - 2016 Symless Ltd.
  * SPDX-License-Identifier: GPL-2.0-only WITH LicenseRef-OpenSSL-Exception
  */
@@ -614,23 +615,15 @@ void SecureSocket::disconnect()
 bool SecureSocket::verifyCertFingerprint()
 {
   // calculate received certificate fingerprint
-  using AutoX509 = std::unique_ptr<X509, decltype(&X509_free)>;
-  AutoX509 cert(SSL_get_peer_certificate(m_ssl->m_ssl), &X509_free);
-
-  unsigned char tempFingerprint[EVP_MAX_MD_SIZE];
-  unsigned int tempFingerprintLen;
-  int digestResult = X509_digest(cert.get(), EVP_sha256(), tempFingerprint, &tempFingerprintLen);
-
-  if (digestResult <= 0) {
-    LOG((CLOG_ERR "failed to calculate fingerprint, digest result: %d", digestResult));
+  std::vector<std::uint8_t> fingerprint_raw;
+  try {
+    fingerprint_raw =
+        deskflow::SSLCertFingerprint(SSL_get_peer_certificate(m_ssl->m_ssl), deskflow::FingerprintType::SHA1);
+  } catch (const std::exception &e) {
+    LOG((CLOG_ERR "%s", e.what()));
     return false;
   }
 
-  // format fingerprint into hexdecimal format with colon separator
-  std::vector<uint8_t> fingerprint_raw;
-  fingerprint_raw.assign(
-      reinterpret_cast<uint8_t *>(tempFingerprint), reinterpret_cast<uint8_t *>(tempFingerprint) + tempFingerprintLen
-  );
   auto fingerprint = deskflow::formatSSLFingerprint(fingerprint_raw);
   LOG((CLOG_NOTE "server fingerprint: %s", fingerprint.c_str()));
 

src/lib/net/SecureUtils.cpp

@@ -8,8 +8,30 @@
 #include "SecureUtils.h"
 #include "base/String.h"
 
+#include <openssl/pem.h>
+#include <openssl/x509.h>
+#include <openssl/x509v3.h>
+#include <stdexcept>
+
 namespace deskflow {
 
+namespace {
+
+const EVP_MD *digestForType(FingerprintType type)
+{
+  switch (type) {
+  case FingerprintType::SHA1:
+    return EVP_sha1();
+  case FingerprintType::SHA256:
+    return EVP_sha256();
+  default:
+    break;
+  }
+  throw std::runtime_error("Unknown fingerprint type " + std::to_string(static_cast<int>(type)));
+}
+
+} // namespace
+
 std::string formatSSLFingerprint(const std::vector<uint8_t> &fingerprint, bool enableSeparators)
 {
   std::string result = deskflow::string::toHex(fingerprint, 2);
@@ -25,4 +47,23 @@ std::string formatSSLFingerprint(const std::vector<uint8_t> &fingerprint, bool e
   return result;
 }
 
+std::vector<uint8_t> SSLCertFingerprint(X509 *cert, FingerprintType type)
+{
+  if (!cert) {
+    throw std::runtime_error("certificate is null");
+  }
+
+  unsigned char digest[EVP_MAX_MD_SIZE];
+  unsigned int digestLength = 0;
+  int result = X509_digest(cert, digestForType(type), digest, &digestLength);
+
+  if (result <= 0) {
+    throw std::runtime_error("failed to calculate fingerprint, digest result: " + std::to_string(result));
+  }
+
+  std::vector<std::uint8_t> digestVec;
+  digestVec.assign(reinterpret_cast<std::uint8_t *>(digest), reinterpret_cast<std::uint8_t *>(digest) + digestLength);
+  return digestVec;
+}
+
 } // namespace deskflow

src/lib/net/SecureUtils.h

@@ -7,7 +7,10 @@
 
 #pragma once
 
-#include <stdint.h>
+#include "FingerprintTypes.h"
+
+#include <cstdint>
+#include <openssl/ossl_typ.h>
 #include <string>
 #include <vector>
 
@@ -21,4 +24,5 @@ namespace deskflow {
  */
 std::string formatSSLFingerprint(const std::vector<uint8_t> &fingerprint, bool enableSeparators = true);
 
+std::vector<std::uint8_t> SSLCertFingerprint(X509 *cert, FingerprintType type);
 } // namespace deskflow