Commit 095f4e57 authored by Honfika's avatar Honfika

- Detect HTTPS ClientHello

- Using upstream proxy when host is excluded, too (earlier upstream proxy was ignored in this case which is wrong)
parent 70c107a4
......@@ -16,8 +16,7 @@ namespace Titanium.Web.Proxy.Examples.Basic
//share requestBody outside handlers
//Using a dictionary is not a good idea since it can cause memory overflow
//ideally the data should be moved out of memory
//private readonly IDictionary<Guid, string> requestBodyHistory
// = new ConcurrentDictionary<Guid, string>();
//private readonly IDictionary<Guid, string> requestBodyHistory = new ConcurrentDictionary<Guid, string>();
public ProxyTestController()
{
......@@ -57,11 +56,14 @@ namespace Titanium.Web.Proxy.Examples.Basic
ExcludedHttpsHostNameRegex = new List<string>
{
"dropbox.com"
}
},
//Include Https addresses you want to proxy (others will be excluded)
//for example github.com
//IncludedHttpsHostNameRegex = new List<string> { "github.com" }
//IncludedHttpsHostNameRegex = new List<string>
//{
// "github.com"
//},
//You can set only one of the ExcludedHttpsHostNameRegex and IncludedHttpsHostNameRegex properties, otherwise ArgumentException will be thrown
......
......@@ -306,6 +306,21 @@ namespace Titanium.Web.Proxy.Helpers
return streamBuffer[bufferPos++];
}
public async Task<int> PeekByteAsync(int index)
{
if (Available <= index)
{
await FillBufferAsync();
}
if (Available <= index)
{
return -1;
}
return streamBuffer[bufferPos + index];
}
public byte ReadByteFromBuffer()
{
if (bufferLength == 0)
......@@ -396,6 +411,8 @@ namespace Titanium.Web.Proxy.Helpers
public bool DataAvailable => bufferLength > 0;
public int Available => bufferLength;
/// <summary>
/// When overridden in a derived class, gets or sets the position within the current stream.
/// </summary>
......@@ -428,14 +445,22 @@ namespace Titanium.Web.Proxy.Helpers
/// </summary>
public bool FillBuffer()
{
bufferLength = baseStream.Read(streamBuffer, 0, streamBuffer.Length);
bufferPos = 0;
if (bufferLength > 0)
{
OnDataReceived(streamBuffer, 0, bufferLength);
//normally we fill the buffer only when it is empty, but sometimes we need more data
//move the remanining data to the beginning of the buffer
Buffer.BlockCopy(streamBuffer, bufferPos, streamBuffer, 0, bufferLength);
}
bufferPos = 0;
int readBytes = baseStream.Read(streamBuffer, bufferLength, streamBuffer.Length - bufferLength);
if (readBytes > 0)
{
OnDataReceived(streamBuffer, bufferLength, readBytes);
bufferLength += readBytes;
}
return bufferLength > 0;
return readBytes > 0;
}
/// <summary>
......@@ -454,14 +479,22 @@ namespace Titanium.Web.Proxy.Helpers
/// <returns></returns>
public async Task<bool> FillBufferAsync(CancellationToken cancellationToken)
{
bufferLength = await baseStream.ReadAsync(streamBuffer, 0, streamBuffer.Length, cancellationToken);
bufferPos = 0;
if (bufferLength > 0)
{
OnDataReceived(streamBuffer, 0, bufferLength);
//normally we fill the buffer only when it is empty, but sometimes we need more data
//move the remanining data to the beginning of the buffer
Buffer.BlockCopy(streamBuffer, bufferPos, streamBuffer, 0, bufferLength);
}
bufferPos = 0;
int readBytes = await baseStream.ReadAsync(streamBuffer, bufferLength, streamBuffer.Length - bufferLength, cancellationToken);
if (readBytes > 0)
{
OnDataReceived(streamBuffer, bufferLength, readBytes);
bufferLength += readBytes;
}
return bufferLength > 0;
return readBytes > 0;
}
private class ReadAsyncResult : IAsyncResult
......
......@@ -170,20 +170,12 @@ namespace Titanium.Web.Proxy.Helpers
/// relays the input clientStream to the server at the specified host name and port with the given httpCmd and headers as prefix
/// Usefull for websocket requests
/// </summary>
/// <param name="server"></param>
/// <param name="remoteHostName"></param>
/// <param name="remotePort"></param>
/// <param name="httpCmd"></param>
/// <param name="httpVersion"></param>
/// <param name="requestHeaders"></param>
/// <param name="isHttps"></param>
/// <param name="clientStream"></param>
/// <param name="tcpConnectionFactory"></param>
/// <param name="connection"></param>
/// <returns></returns>
internal static async Task SendRaw(ProxyServer server, string remoteHostName, int remotePort, string httpCmd, Version httpVersion,
Dictionary<string, HttpHeader> requestHeaders, bool isHttps, Stream clientStream, TcpConnectionFactory tcpConnectionFactory,
TcpConnection connection = null)
internal static async Task SendRaw(string httpCmd, IEnumerable<HttpHeader> requestHeaders, Stream clientStream, TcpConnection connection)
{
//prepare the prefix content
StringBuilder sb = null;
......@@ -199,7 +191,7 @@ namespace Titanium.Web.Proxy.Helpers
if (requestHeaders != null)
{
foreach (string header in requestHeaders.Select(t => t.Value.ToString()))
foreach (string header in requestHeaders.Select(t => t.ToString()))
{
sb.Append(header);
sb.Append(ProxyConstants.NewLine);
......@@ -209,43 +201,14 @@ namespace Titanium.Web.Proxy.Helpers
sb.Append(ProxyConstants.NewLine);
}
bool connectionCreated = false;
TcpConnection tcpConnection;
var tunnelStream = connection.Stream;
//create new connection if connection is null
if (connection == null)
{
tcpConnection = await tcpConnectionFactory.CreateClient(server, remoteHostName, remotePort, httpVersion, isHttps, null, null);
connectionCreated = true;
}
else
{
tcpConnection = connection;
}
try
{
var tunnelStream = tcpConnection.Stream;
//Now async relay all server=>client & client=>server data
var sendRelay = clientStream.CopyToAsync(sb?.ToString() ?? string.Empty, tunnelStream);
//Now async relay all server=>client & client=>server data
var sendRelay = clientStream.CopyToAsync(sb?.ToString() ?? string.Empty, tunnelStream);
var receiveRelay = tunnelStream.CopyToAsync(string.Empty, clientStream);
var receiveRelay = tunnelStream.CopyToAsync(string.Empty, clientStream);
await Task.WhenAll(sendRelay, receiveRelay);
}
finally
{
//if connection was null
//then a new connection was created
//so dispose the new connection
if (connectionCreated)
{
tcpConnection.Dispose();
Interlocked.Decrement(ref server.serverConnectionCount);
}
}
await Task.WhenAll(sendRelay, receiveRelay);
}
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Titanium.Web.Proxy.Helpers;
namespace Titanium.Web.Proxy.Http
{
class HttpsTools
{
public static async Task<bool> IsClientHello(CustomBufferedStream clientStream)
{
//detects the HTTPS ClientHello message as it is described in the following url:
//https://stackoverflow.com/questions/3897883/how-to-detect-an-incoming-ssl-https-handshake-ssl-wire-format
int recordType = await clientStream.PeekByteAsync(0);
if (recordType == 0x80)
{
//SSL 2
int length = await clientStream.PeekByteAsync(1);
if (length < 9)
{
// Message body too short.
return false;
}
if (await clientStream.PeekByteAsync(2) != 0x01)
{
// should be ClientHello
return false;
}
int majorVersion = await clientStream.PeekByteAsync(3);
int minorVersion = await clientStream.PeekByteAsync(4);
return true;
}
else if (recordType == 0x16)
{
//SSL 3.0 or TLS 1.0, 1.1 and 1.2
int majorVersion = await clientStream.PeekByteAsync(1);
int minorVersion = await clientStream.PeekByteAsync(2);
int length1 = await clientStream.PeekByteAsync(3);
int length2 = await clientStream.PeekByteAsync(4);
int length = (length1 << 8) + length2;
if (await clientStream.PeekByteAsync(5) != 0x01)
{
// should be ClientHello
return false;
}
return true;
}
return false;
}
}
}
......@@ -53,6 +53,7 @@ namespace Titanium.Web.Proxy.Models
public bool IpV6Enabled => Equals(IpAddress, IPAddress.IPv6Any)
|| Equals(IpAddress, IPAddress.IPv6Loopback)
|| Equals(IpAddress, IPAddress.IPv6None);
}
/// <summary>
......@@ -68,12 +69,6 @@ namespace Titanium.Web.Proxy.Models
internal bool IsSystemHttpsProxy { get; set; }
/// <summary>
/// Remote HTTPS ports we are allowed to communicate with
/// CONNECT request to ports other than these will not be decrypted
/// </summary>
public List<int> RemoteHttpsPorts { get; set; }
/// <summary>
/// List of host names to exclude using Regular Expressions.
/// </summary>
......@@ -121,12 +116,6 @@ namespace Titanium.Web.Proxy.Models
/// <param name="enableSsl"></param>
public ExplicitProxyEndPoint(IPAddress ipAddress, int port, bool enableSsl) : base(ipAddress, port, enableSsl)
{
//init to well known HTTPS ports
RemoteHttpsPorts = new List<int>
{
443,
8443
};
}
}
......
......@@ -15,12 +15,16 @@ namespace Titanium.Web.Proxy.Network.Tcp
internal ExternalProxy UpStreamHttpsProxy { get; set; }
internal ExternalProxy UpStreamProxy => UseProxy ? IsHttps ? UpStreamHttpsProxy : UpStreamHttpProxy : null;
internal string HostName { get; set; }
internal int Port { get; set; }
internal bool IsHttps { get; set; }
internal bool UseProxy { get; set; }
/// <summary>
/// Http version
/// </summary>
......
......@@ -32,29 +32,18 @@ namespace Titanium.Web.Proxy.Network.Tcp
internal async Task<TcpConnection> CreateClient(ProxyServer server, string remoteHostName, int remotePort, Version httpVersion, bool isHttps,
ExternalProxy externalHttpProxy, ExternalProxy externalHttpsProxy)
{
bool useHttpProxy = false;
//check if external proxy is set for HTTP
if (!isHttps && externalHttpProxy != null && !(externalHttpProxy.HostName == remoteHostName && externalHttpProxy.Port == remotePort))
{
useHttpProxy = true;
//check if we need to ByPass
if (externalHttpProxy.BypassLocalhost && NetworkHelper.IsLocalIpAddress(remoteHostName))
{
useHttpProxy = false;
}
}
bool useProxy = false;
var externalProxy = isHttps ? externalHttpsProxy : externalHttpProxy;
bool useHttpsProxy = false;
//check if external proxy is set for HTTPS
if (isHttps && externalHttpsProxy != null && !(externalHttpsProxy.HostName == remoteHostName && externalHttpsProxy.Port == remotePort))
//check if external proxy is set for HTTP/HTTPS
if (externalProxy != null && !(externalProxy.HostName == remoteHostName && externalProxy.Port == remotePort))
{
useHttpsProxy = true;
useProxy = true;
//check if we need to ByPass
if (externalHttpsProxy.BypassLocalhost && NetworkHelper.IsLocalIpAddress(remoteHostName))
if (externalProxy.BypassLocalhost && NetworkHelper.IsLocalIpAddress(remoteHostName))
{
useHttpsProxy = false;
useProxy = false;
}
}
......@@ -63,75 +52,60 @@ namespace Titanium.Web.Proxy.Network.Tcp
try
{
if (isHttps)
//If this proxy uses another external proxy then create a tunnel request for HTTP/HTTPS connections
if (useProxy)
{
//If this proxy uses another external proxy then create a tunnel request for HTTPS connections
if (useHttpsProxy)
client = new TcpClient(server.UpStreamEndPoint);
await client.ConnectAsync(externalProxy.HostName, externalProxy.Port);
stream = new CustomBufferedStream(client.GetStream(), server.BufferSize);
using (var writer = new StreamWriter(stream, Encoding.ASCII, server.BufferSize, true)
{
client = new TcpClient(server.UpStreamEndPoint);
await client.ConnectAsync(externalHttpsProxy.HostName, externalHttpsProxy.Port);
stream = new CustomBufferedStream(client.GetStream(), server.BufferSize);
NewLine = ProxyConstants.NewLine
})
{
await writer.WriteLineAsync($"CONNECT {remoteHostName}:{remotePort} HTTP/{httpVersion}");
await writer.WriteLineAsync($"Host: {remoteHostName}:{remotePort}");
await writer.WriteLineAsync("Connection: Keep-Alive");
using (var writer = new StreamWriter(stream, Encoding.ASCII, server.BufferSize, true)
{
NewLine = ProxyConstants.NewLine
})
if (!string.IsNullOrEmpty(externalProxy.UserName) && externalProxy.Password != null)
{
await writer.WriteLineAsync($"CONNECT {remoteHostName}:{remotePort} HTTP/{httpVersion}");
await writer.WriteLineAsync($"Host: {remoteHostName}:{remotePort}");
await writer.WriteLineAsync("Connection: Keep-Alive");
if (!string.IsNullOrEmpty(externalHttpsProxy.UserName) && externalHttpsProxy.Password != null)
{
await writer.WriteLineAsync("Proxy-Connection: keep-alive");
await writer.WriteLineAsync("Proxy-Authorization" + ": Basic " +
Convert.ToBase64String(Encoding.UTF8.GetBytes(
externalHttpsProxy.UserName + ":" + externalHttpsProxy.Password)));
}
await writer.WriteLineAsync();
await writer.FlushAsync();
writer.Close();
await writer.WriteLineAsync("Proxy-Connection: keep-alive");
await writer.WriteLineAsync("Proxy-Authorization" + ": Basic " +
Convert.ToBase64String(Encoding.UTF8.GetBytes(
externalProxy.UserName + ":" + externalProxy.Password)));
}
await writer.WriteLineAsync();
await writer.FlushAsync();
writer.Close();
}
using (var reader = new CustomBinaryReader(stream, server.BufferSize))
{
string result = await reader.ReadLineAsync();
if (!new[] { "200 OK", "connection established" }.Any(s => result.ContainsIgnoreCase(s)))
{
throw new Exception("Upstream proxy failed to create a secure tunnel");
}
using (var reader = new CustomBinaryReader(stream, server.BufferSize))
{
string result = await reader.ReadLineAsync();
await reader.ReadAndIgnoreAllLinesAsync();
if (!new[] { "200 OK", "connection established" }.Any(s => result.ContainsIgnoreCase(s)))
{
throw new Exception("Upstream proxy failed to create a secure tunnel");
}
await reader.ReadAndIgnoreAllLinesAsync();
}
else
{
client = new TcpClient(server.UpStreamEndPoint);
await client.ConnectAsync(remoteHostName, remotePort);
stream = new CustomBufferedStream(client.GetStream(), server.BufferSize);
}
}
else
{
client = new TcpClient(server.UpStreamEndPoint);
await client.ConnectAsync(remoteHostName, remotePort);
stream = new CustomBufferedStream(client.GetStream(), server.BufferSize);
}
if (isHttps)
{
var sslStream = new SslStream(stream, false, server.ValidateServerCertificate, server.SelectClientCertificate);
stream = new CustomBufferedStream(sslStream, server.BufferSize);
await sslStream.AuthenticateAsClientAsync(remoteHostName, null, server.SupportedSslProtocols, server.CheckCertificateRevocation);
}
else
{
if (useHttpProxy)
{
client = new TcpClient(server.UpStreamEndPoint);
await client.ConnectAsync(externalHttpProxy.HostName, externalHttpProxy.Port);
stream = new CustomBufferedStream(client.GetStream(), server.BufferSize);
}
else
{
client = new TcpClient(server.UpStreamEndPoint);
await client.ConnectAsync(remoteHostName, remotePort);
stream = new CustomBufferedStream(client.GetStream(), server.BufferSize);
}
}
client.ReceiveTimeout = server.ConnectionTimeOutSeconds * 1000;
client.SendTimeout = server.ConnectionTimeOutSeconds * 1000;
......@@ -152,6 +126,7 @@ namespace Titanium.Web.Proxy.Network.Tcp
HostName = remoteHostName,
Port = remotePort,
IsHttps = isHttps,
UseProxy = useProxy,
TcpClient = client,
StreamReader = new CustomBinaryReader(stream, server.BufferSize),
Stream = stream,
......
......@@ -90,10 +90,9 @@ namespace Titanium.Web.Proxy
List<HttpHeader> connectRequestHeaders = null;
//Client wants to create a secure tcp tunnel (its a HTTPS request)
if (httpVerb == "CONNECT" && !excluded && endPoint.RemoteHttpsPorts.Contains(httpRemoteUri.Port))
//Client wants to create a secure tcp tunnel (probably its a HTTPS or Websocket request)
if (httpVerb == "CONNECT")
{
httpRemoteUri = new Uri("https://" + httpCmdSplit[1]);
connectRequestHeaders = new List<HttpHeader>();
string tmpLine;
while (!string.IsNullOrEmpty(tmpLine = await clientStreamReader.ReadLineAsync()))
......@@ -104,61 +103,90 @@ namespace Titanium.Web.Proxy
connectRequestHeaders.Add(newHeader);
}
if (await CheckAuthorization(clientStreamWriter, connectRequestHeaders) == false)
if (httpRemoteUri.Port == 80)
{
return;
// why is this needed? HTTPS not allowed on port 80?
excluded = true;
}
if (!excluded)
{
if (await CheckAuthorization(clientStreamWriter, connectRequestHeaders) == false)
{
return;
}
httpRemoteUri = new Uri("https://" + httpCmdSplit[1]);
}
//write back successfull CONNECT response
await WriteConnectResponse(clientStreamWriter, version);
SslStream sslStream = null;
if (!excluded && !await HttpsTools.IsClientHello(clientStream))
{
excluded = true;
}
try
if (!excluded)
{
sslStream = new SslStream(clientStream);
SslStream sslStream = null;
string certName = HttpHelper.GetWildCardDomainName(httpRemoteUri.Host);
try
{
sslStream = new SslStream(clientStream);
string certName = HttpHelper.GetWildCardDomainName(httpRemoteUri.Host);
var certificate = endPoint.GenericCertificate ?? CertificateManager.CreateCertificate(certName, false);
var certificate = endPoint.GenericCertificate ?? CertificateManager.CreateCertificate(certName, false);
//Successfully managed to authenticate the client using the fake certificate
await sslStream.AuthenticateAsServerAsync(certificate, false, SupportedSslProtocols, false);
//Successfully managed to authenticate the client using the fake certificate
await sslStream.AuthenticateAsServerAsync(certificate, false, SupportedSslProtocols, false);
//HTTPS server created - we can now decrypt the client's traffic
clientStream = new CustomBufferedStream(sslStream, BufferSize);
//HTTPS server created - we can now decrypt the client's traffic
clientStream = new CustomBufferedStream(sslStream, BufferSize);
clientStreamReader.Dispose();
clientStreamReader = new CustomBinaryReader(clientStream, BufferSize);
clientStreamWriter = new StreamWriter(clientStream)
clientStreamReader.Dispose();
clientStreamReader = new CustomBinaryReader(clientStream, BufferSize);
clientStreamWriter = new StreamWriter(clientStream)
{
NewLine = ProxyConstants.NewLine
};
}
catch
{
NewLine = ProxyConstants.NewLine
};
sslStream?.Dispose();
return;
}
//Now read the actual HTTPS request line
httpCmd = await clientStreamReader.ReadLineAsync();
}
catch
//Sorry cannot do a HTTPS request decrypt to port 80 at this time
else
{
sslStream?.Dispose();
var args = new SessionEventArgs(BufferSize, HandleHttpSessionResponse);
args.WebSession.Request.RequestUri = httpRemoteUri;
args.WebSession.Request.HttpVersion = version;
args.WebSession.Request.Method = httpVerb;
args.ProxyClient.ClientStream = clientStream;
args.ProxyClient.ClientStreamReader = clientStreamReader;
args.ProxyClient.ClientStreamWriter = clientStreamWriter;
//create new connection
using (var connection = await GetServerConnection(args))
{
if (connection.UseProxy)
{
await TcpHelper.SendRaw(null, null, clientStream, connection);
}
else
{
await TcpHelper.SendRaw(null, null, clientStream, connection);
}
}
return;
}
//Now read the actual HTTPS request line
httpCmd = await clientStreamReader.ReadLineAsync();
}
//Sorry cannot do a HTTPS request decrypt to port 80 at this time
else if (httpVerb == "CONNECT")
{
//Siphon out CONNECT request headers
await clientStreamReader.ReadAndIgnoreAllLinesAsync();
//write back successfull CONNECT response
await WriteConnectResponse(clientStreamWriter, version);
await TcpHelper.SendRaw(this,
httpRemoteUri.Host, httpRemoteUri.Port,
null, version, null,
false,
clientStream, tcpConnectionFactory);
return;
}
//Now create the request
......@@ -287,7 +315,7 @@ namespace Titanium.Web.Proxy
//break up the line into three components (method, remote URL & Http Version)
var httpCmdSplit = httpCmd.Split(ProxyConstants.SpaceSplit, 3);
string httpMethod = httpCmdSplit[0];
string httpMethod = httpCmdSplit[0].ToUpper();
//find the request HTTP version
var httpVersion = HttpHeader.Version11;
......@@ -310,7 +338,7 @@ namespace Titanium.Web.Proxy
args.WebSession.Request.RequestUri = httpRemoteUri;
args.WebSession.Request.Method = httpMethod.Trim().ToUpper();
args.WebSession.Request.Method = httpMethod;
args.WebSession.Request.HttpVersion = httpVersion;
args.ProxyClient.ClientStream = clientStream;
args.ProxyClient.ClientStreamReader = clientStreamReader;
......@@ -346,16 +374,6 @@ namespace Titanium.Web.Proxy
break;
}
//if upgrading to websocket then relay the requet without reading the contents
if (args.WebSession.Request.UpgradeToWebSocket)
{
await TcpHelper.SendRaw(this, httpRemoteUri.Host, httpRemoteUri.Port, httpCmd, httpVersion, args.WebSession.Request.RequestHeaders,
args.IsHttps, clientStream, tcpConnectionFactory, connection);
args.Dispose();
break;
}
if (connection == null)
{
connection = await GetServerConnection(args);
......@@ -368,6 +386,15 @@ namespace Titanium.Web.Proxy
connection = await GetServerConnection(args);
}
//if upgrading to websocket then relay the requet without reading the contents
if (args.WebSession.Request.UpgradeToWebSocket)
{
await TcpHelper.SendRaw(httpCmd, args.WebSession.Request.RequestHeaders.Values, clientStream, connection);
args.Dispose();
break;
}
//construct the web request that we are going to issue on behalf of the client.
disposed = await HandleHttpSessionRequestInternal(connection, args, false);
......
......@@ -86,6 +86,7 @@
<Compile Include="Helpers\WinHttp\WinHttpHandle.cs" />
<Compile Include="Helpers\WinHttp\WinHttpWebProxyFinder.cs" />
<Compile Include="Http\HeaderParser.cs" />
<Compile Include="Http\HttpsTools.cs" />
<Compile Include="Http\Responses\GenericResponse.cs" />
<Compile Include="Network\CachedCertificate.cs" />
<Compile Include="Network\Certificate\WinCertificateMaker.cs" />
......@@ -163,4 +164,4 @@
<Target Name="AfterBuild">
</Target>
-->
</Project>
</Project>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment