Unverified Commit 931d59c1 authored by honfika's avatar honfika Committed by GitHub

Merge pull request #688 from justcoding121/master

stable
parents 886752da dd0a19ad
......@@ -42,6 +42,10 @@ namespace Titanium.Web.Proxy.Examples.Basic
proxyServer.ForwardToUpstreamGateway = true;
proxyServer.CertificateManager.SaveFakeCertificates = true;
// this is just to show the functionality, provided implementations use junk value
//proxyServer.GetCustomUpStreamProxyFunc = onGetCustomUpStreamProxyFunc;
//proxyServer.CustomUpStreamProxyFailureFunc = onCustomUpStreamProxyFailureFunc;
// optionally set the Certificate Engine
// Under Mono or Non-Windows runtimes only BouncyCastle will be supported
//proxyServer.CertificateManager.CertificateEngine = Network.CertificateEngine.BouncyCastle;
......@@ -116,6 +120,18 @@ namespace Titanium.Web.Proxy.Examples.Basic
//proxyServer.CertificateManager.RemoveTrustedRootCertificates();
}
private async Task<IExternalProxy> onGetCustomUpStreamProxyFunc(SessionEventArgsBase arg)
{
// this is just to show the functionality, provided values are junk
return new ExternalProxy() { BypassLocalhost = false, HostName = "127.0.0.9", Port = 9090, Password = "fake", UserName = "fake", UseDefaultCredentials = false };
}
private async Task<IExternalProxy> onCustomUpStreamProxyFailureFunc(SessionEventArgsBase arg)
{
// this is just to show the functionality, provided values are junk
return new ExternalProxy() { BypassLocalhost = false, HostName = "127.0.0.10", Port = 9191, Password = "fake2", UserName = "fake2", UseDefaultCredentials = false };
}
private async Task onBeforeTunnelConnectRequest(object sender, TunnelConnectSessionEventArgs e)
{
string hostname = e.HttpClient.Request.RequestUri.Host;
......@@ -135,7 +151,7 @@ namespace Titanium.Web.Proxy.Examples.Basic
return Task.FromResult(false);
}
// intecept & cancel redirect or update requests
// intercept & cancel redirect or update requests
private async Task onRequest(object sender, SessionEventArgs e)
{
await writeToConsole("Active Client Connections:" + ((ProxyServer)sender).ClientConnectionCount);
......
......@@ -32,6 +32,8 @@
<GridViewColumn Header="SentBytes" DisplayMemberBinding="{Binding SentDataCount}" />
<GridViewColumn Header="ReceivedBytes" DisplayMemberBinding="{Binding ReceivedDataCount}" />
<GridViewColumn Header="ClientEndPoint" DisplayMemberBinding="{Binding ClientEndPoint}" />
<GridViewColumn Header="ClientConnectionId" DisplayMemberBinding="{Binding ClientConnectionId}" />
<GridViewColumn Header="ServerConnectionId" DisplayMemberBinding="{Binding ServerConnectionId}" />
</GridView>
</ListView.View>
</ListView>
......
......@@ -146,13 +146,15 @@ namespace Titanium.Web.Proxy.Examples.Wpf
{
if (sessionDictionary.TryGetValue(e.HttpClient, out var item))
{
item.Update();
item.Update(e);
}
});
}
private async Task ProxyServer_BeforeRequest(object sender, SessionEventArgs e)
{
//if (e.HttpClient.Request.HttpVersion.Major != 2) return;
SessionListItem item = null;
await Dispatcher.InvokeAsync(() => { item = addSession(e); });
......@@ -175,7 +177,7 @@ namespace Titanium.Web.Proxy.Examples.Wpf
{
if (sessionDictionary.TryGetValue(e.HttpClient, out item))
{
item.Update();
item.Update(e);
}
});
......@@ -190,7 +192,7 @@ namespace Titanium.Web.Proxy.Examples.Wpf
e.HttpClient.Response.KeepBody = true;
await e.GetResponseBody();
await Dispatcher.InvokeAsync(() => { item.Update(); });
await Dispatcher.InvokeAsync(() => { item.Update(e); });
if (item == SelectedSession)
{
await Dispatcher.InvokeAsync(selectedSessionChanged);
......@@ -225,6 +227,8 @@ namespace Titanium.Web.Proxy.Examples.Wpf
var item = new SessionListItem
{
Number = lastSessionNumber,
ClientConnectionId = e.ClientConnectionId,
ServerConnectionId = e.ServerConnectionId,
HttpClient = e.HttpClient,
ClientEndPoint = e.ClientEndPoint,
IsTunnelConnect = isTunnelConnect
......@@ -236,7 +240,8 @@ namespace Titanium.Web.Proxy.Examples.Wpf
var session = (SessionEventArgsBase)sender;
if (sessionDictionary.TryGetValue(session.HttpClient, out var li))
{
var tunnelType = session.HttpClient.ConnectRequest?.TunnelType ?? TunnelType.Unknown;
var connectRequest = session.HttpClient.ConnectRequest;
var tunnelType = connectRequest?.TunnelType ?? TunnelType.Unknown;
if (tunnelType != TunnelType.Unknown)
{
li.Protocol = TunnelTypeToString(tunnelType);
......@@ -244,6 +249,7 @@ namespace Titanium.Web.Proxy.Examples.Wpf
li.ReceivedDataCount += args.Count;
//if (tunnelType == TunnelType.Http2)
AppendTransferLog(session.GetHashCode() + (isTunnelConnect ? "_tunnel" : "") + "_received",
args.Buffer, args.Offset, args.Count);
}
......@@ -254,7 +260,8 @@ namespace Titanium.Web.Proxy.Examples.Wpf
var session = (SessionEventArgsBase)sender;
if (sessionDictionary.TryGetValue(session.HttpClient, out var li))
{
var tunnelType = session.HttpClient.ConnectRequest?.TunnelType ?? TunnelType.Unknown;
var connectRequest = session.HttpClient.ConnectRequest;
var tunnelType = connectRequest?.TunnelType ?? TunnelType.Unknown;
if (tunnelType != TunnelType.Unknown)
{
li.Protocol = TunnelTypeToString(tunnelType);
......@@ -262,7 +269,8 @@ namespace Titanium.Web.Proxy.Examples.Wpf
li.SentDataCount += args.Count;
AppendTransferLog(session.GetHashCode() + (isTunnelConnect ? "_tunnel" : "") + "_sent",
//if (tunnelType == TunnelType.Http2)
AppendTransferLog( session.GetHashCode() + (isTunnelConnect ? "_tunnel" : "") + "_sent",
args.Buffer, args.Offset, args.Count);
}
};
......@@ -272,6 +280,8 @@ namespace Titanium.Web.Proxy.Examples.Wpf
te.DecryptedDataReceived += (sender, args) =>
{
var session = (SessionEventArgsBase)sender;
//var tunnelType = session.HttpClient.ConnectRequest?.TunnelType ?? TunnelType.Unknown;
//if (tunnelType == TunnelType.Http2)
AppendTransferLog(session.GetHashCode() + "_decrypted_received", args.Buffer, args.Offset,
args.Count);
};
......@@ -279,11 +289,13 @@ namespace Titanium.Web.Proxy.Examples.Wpf
te.DecryptedDataSent += (sender, args) =>
{
var session = (SessionEventArgsBase)sender;
//var tunnelType = session.HttpClient.ConnectRequest?.TunnelType ?? TunnelType.Unknown;
//if (tunnelType == TunnelType.Http2)
AppendTransferLog(session.GetHashCode() + "_decrypted_sent", args.Buffer, args.Offset, args.Count);
};
}
item.Update();
item.Update(e);
return item;
}
......@@ -362,9 +374,15 @@ namespace Titanium.Web.Proxy.Examples.Wpf
//string hexStr = string.Join(" ", data.Select(x => x.ToString("X2")));
var sb = new StringBuilder();
sb.AppendLine("URI: " + request.RequestUri);
sb.Append(request.HeaderText);
sb.Append(request.Encoding.GetString(data));
sb.Append(truncated ? Environment.NewLine + $"Data is truncated after {truncateLimit} bytes" : null);
if (truncated)
{
sb.AppendLine();
sb.Append($"Data is truncated after {truncateLimit} bytes");
}
sb.Append((request as ConnectRequest)?.ClientHelloInfo);
TextBoxRequest.Text = sb.ToString();
......@@ -381,7 +399,12 @@ namespace Titanium.Web.Proxy.Examples.Wpf
sb = new StringBuilder();
sb.Append(response.HeaderText);
sb.Append(response.Encoding.GetString(data));
sb.Append(truncated ? Environment.NewLine + $"Data is truncated after {truncateLimit} bytes" : null);
if (truncated)
{
sb.AppendLine();
sb.Append($"Data is truncated after {truncateLimit} bytes");
}
sb.Append((response as ConnectResponse)?.ServerHelloInfo);
if (SelectedSession.Exception != null)
{
......
......@@ -2,6 +2,7 @@
using System.ComponentModel;
using System.Net;
using System.Runtime.CompilerServices;
using Titanium.Web.Proxy.EventArguments;
using Titanium.Web.Proxy.Examples.Wpf.Annotations;
using Titanium.Web.Proxy.Http;
......@@ -18,9 +19,23 @@ namespace Titanium.Web.Proxy.Examples.Wpf
private long sentDataCount;
private string statusCode;
private string url;
private Guid clientConnectionId;
private Guid serverConnectionId;
public int Number { get; set; }
public Guid ClientConnectionId
{
get => clientConnectionId;
set => SetField(ref clientConnectionId, value);
}
public Guid ServerConnectionId
{
get => serverConnectionId;
set => SetField(ref serverConnectionId, value);
}
public HttpWebClient HttpClient { get; set; }
public IPEndPoint ClientEndPoint { get; set; }
......@@ -123,13 +138,15 @@ namespace Titanium.Web.Proxy.Examples.Wpf
PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName));
}
public void Update()
public void Update(SessionEventArgsBase args)
{
var request = HttpClient.Request;
var response = HttpClient.Response;
int statusCode = response?.StatusCode ?? 0;
StatusCode = statusCode == 0 ? "-" : statusCode.ToString();
Protocol = request.RequestUri.Scheme;
ClientConnectionId = args.ClientConnectionId;
ServerConnectionId = args.ServerConnectionId;
if (IsTunnelConnect)
{
......
using System;
using System.Globalization;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Titanium.Web.Proxy.Exceptions;
using Titanium.Web.Proxy.Helpers;
using Titanium.Web.Proxy.StreamExtended.BufferPool;
using Titanium.Web.Proxy.StreamExtended.Network;
......@@ -51,11 +51,22 @@ namespace Titanium.Web.Proxy.EventArguments
{
// read the chunk trail of the previous chunk
string? s = baseReader.ReadLineAsync().Result;
if (s == null)
{
bytesRemaining = -1;
return;
}
}
readChunkTrail = true;
string? chunkHead = baseReader.ReadLineAsync().Result!;
string? chunkHead = baseReader.ReadLineAsync().Result;
if (chunkHead == null)
{
bytesRemaining = -1;
return;
}
int idx = chunkHead.IndexOf(";", StringComparison.Ordinal);
if (idx >= 0)
{
......@@ -80,6 +91,50 @@ namespace Titanium.Web.Proxy.EventArguments
}
}
private async Task getNextChunkAsync()
{
if (readChunkTrail)
{
// read the chunk trail of the previous chunk
string? s = await baseReader.ReadLineAsync();
if (s == null)
{
bytesRemaining = -1;
return;
}
}
readChunkTrail = true;
string? chunkHead = await baseReader.ReadLineAsync();
if (chunkHead == null)
{
bytesRemaining = -1;
return;
}
int idx = chunkHead.IndexOf(";", StringComparison.Ordinal);
if (idx >= 0)
{
chunkHead = chunkHead.Substring(0, idx);
}
if (!int.TryParse(chunkHead, NumberStyles.HexNumber, null, out int chunkSize))
{
throw new ProxyHttpException($"Invalid chunk length: '{chunkHead}'", null, null);
}
bytesRemaining = chunkSize;
if (chunkSize == 0)
{
bytesRemaining = -1;
// chunk trail
await baseReader.ReadLineAsync();
}
}
public override void Flush()
{
throw new NotSupportedException();
......@@ -131,6 +186,42 @@ namespace Titanium.Web.Proxy.EventArguments
return res;
}
public override async Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
if (bytesRemaining == -1)
{
return 0;
}
if (bytesRemaining == 0)
{
if (isChunked)
{
await getNextChunkAsync();
}
else
{
bytesRemaining = -1;
}
}
if (bytesRemaining == -1)
{
return 0;
}
int toRead = (int)Math.Min(count, bytesRemaining);
int res = await baseReader.ReadAsync(buffer, offset, toRead, cancellationToken);
bytesRemaining -= res;
if (res == 0)
{
bytesRemaining = -1;
}
return res;
}
public async Task Finish()
{
if (bytesRemaining != -1)
......
......@@ -4,12 +4,10 @@ using System.IO;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using Titanium.Web.Proxy.Compression;
using Titanium.Web.Proxy.Helpers;
using Titanium.Web.Proxy.Http;
using Titanium.Web.Proxy.Http.Responses;
using Titanium.Web.Proxy.Models;
using Titanium.Web.Proxy.Network;
using Titanium.Web.Proxy.Network.Tcp;
using Titanium.Web.Proxy.StreamExtended.Network;
......@@ -65,11 +63,6 @@ namespace Titanium.Web.Proxy.EventArguments
/// </summary>
public event EventHandler<MultipartRequestPartSentEventArgs>? MultipartRequestPartSent;
private HttpStream getStream(bool isRequest)
{
return isRequest ? (HttpStream)ClientStream : HttpClient.Connection.Stream;
}
/// <summary>
/// Read request body content as bytes[] for current session
/// </summary>
......@@ -194,15 +187,15 @@ namespace Titanium.Web.Proxy.EventArguments
private async Task<byte[]> readBodyAsync(bool isRequest, CancellationToken cancellationToken)
{
using var bodyStream = new MemoryStream();
using var http = new HttpStream(bodyStream, BufferPool);
using var writer = new HttpStream(bodyStream, BufferPool);
if (isRequest)
{
await CopyRequestBodyAsync(http, TransformationMode.Uncompress, cancellationToken);
await CopyRequestBodyAsync(writer, TransformationMode.Uncompress, cancellationToken);
}
else
{
await CopyResponseBodyAsync(http, TransformationMode.Uncompress, cancellationToken);
await copyResponseBodyAsync(writer, TransformationMode.Uncompress, cancellationToken);
}
return bodyStream.ToArray();
......@@ -223,9 +216,9 @@ namespace Titanium.Web.Proxy.EventArguments
return;
}
using var bodyStream = new MemoryStream();
using var http = new HttpStream(bodyStream, BufferPool);
await copyBodyAsync(isRequest, true, http, TransformationMode.None, null, cancellationToken);
var reader = isRequest ? (HttpStream)ClientStream : HttpClient.Connection.Stream;
await reader.CopyBodyAsync(requestResponse, true, new NullWriter(), TransformationMode.None, null, cancellationToken);
}
/// <summary>
......@@ -235,13 +228,13 @@ namespace Titanium.Web.Proxy.EventArguments
internal async Task CopyRequestBodyAsync(IHttpStreamWriter writer, TransformationMode transformation, CancellationToken cancellationToken)
{
var request = HttpClient.Request;
var reader = ClientStream;
long contentLength = request.ContentLength;
// send the request body bytes to server
if (contentLength > 0 && hasMulipartEventSubscribers && request.IsMultipartFormData)
{
var reader = getStream(true);
var boundary = HttpHelper.GetBoundaryFromContentType(request.ContentType);
using (var copyStream = new CopyStream(reader, writer, BufferPool))
......@@ -267,54 +260,13 @@ namespace Titanium.Web.Proxy.EventArguments
}
else
{
await copyBodyAsync(true, false, writer, transformation, OnDataSent, cancellationToken);
await reader.CopyBodyAsync(request, false, writer, transformation, OnDataSent, cancellationToken);
}
}
internal async Task CopyResponseBodyAsync(IHttpStreamWriter writer, TransformationMode transformation, CancellationToken cancellationToken)
{
await copyBodyAsync(false, false, writer, transformation, OnDataReceived, cancellationToken);
}
private async Task copyBodyAsync(bool isRequest, bool useOriginalHeaderValues, IHttpStreamWriter writer, TransformationMode transformation, Action<byte[], int, int>? onCopy, CancellationToken cancellationToken)
private async Task copyResponseBodyAsync(IHttpStreamWriter writer, TransformationMode transformation, CancellationToken cancellationToken)
{
var stream = getStream(isRequest);
var requestResponse = isRequest ? (RequestResponseBase)HttpClient.Request : HttpClient.Response;
bool isChunked = useOriginalHeaderValues? requestResponse.OriginalIsChunked : requestResponse.IsChunked;
long contentLength = useOriginalHeaderValues ? requestResponse.OriginalContentLength : requestResponse.ContentLength;
if (transformation == TransformationMode.None)
{
await writer.CopyBodyAsync(stream, isChunked, contentLength, onCopy, cancellationToken);
return;
}
LimitedStream limitedStream;
Stream? decompressStream = null;
string? contentEncoding = useOriginalHeaderValues ? requestResponse.OriginalContentEncoding : requestResponse.ContentEncoding;
Stream s = limitedStream = new LimitedStream(stream, BufferPool, isChunked, contentLength);
if (transformation == TransformationMode.Uncompress && contentEncoding != null)
{
s = decompressStream = DecompressionFactory.Create(CompressionUtil.CompressionNameToEnum(contentEncoding), s);
}
try
{
var http = new HttpStream(s, BufferPool, true);
await writer.CopyBodyAsync(http, false, -1, onCopy, cancellationToken);
}
finally
{
decompressStream?.Dispose();
await limitedStream.Finish();
limitedStream.Dispose();
}
await HttpClient.Connection.Stream.CopyBodyAsync(HttpClient.Response, false, writer, transformation, OnDataReceived, cancellationToken);
}
/// <summary>
......@@ -638,7 +590,6 @@ namespace Titanium.Web.Proxy.EventArguments
HttpClient.Response = response;
HttpClient.Response.Locked = true;
}
}
/// <summary>
......
......@@ -33,6 +33,10 @@ namespace Titanium.Web.Proxy.EventArguments
internal HttpClientStream ClientStream { get; }
public Guid ClientConnectionId => ClientConnection.Id;
public Guid ServerConnectionId => HttpClient.HasConnection ? ServerConnection.Id : Guid.Empty;
protected readonly IBufferPool BufferPool;
protected readonly ExceptionHandler ExceptionFunc;
private bool enableWinAuth;
......
......@@ -52,18 +52,16 @@ namespace Titanium.Web.Proxy
if (await HttpHelper.IsConnectMethod(clientStream, BufferPool, cancellationToken) == 1)
{
// read the first line HTTP command
string? httpCmd = await clientStream.ReadLineAsync(cancellationToken);
if (string.IsNullOrEmpty(httpCmd))
var requestLine = await clientStream.ReadRequestLine(cancellationToken);
if (requestLine.IsEmpty())
{
return;
}
Request.ParseRequestLine(httpCmd!, out string _, out string httpUrl, out var version);
var connectRequest = new ConnectRequest(httpUrl)
var connectRequest = new ConnectRequest(requestLine.RequestUri)
{
OriginalUrlData = HttpHeader.Encoding.GetBytes(httpUrl),
HttpVersion = version
RequestUriString8 = requestLine.RequestUri,
HttpVersion = requestLine.Version
};
await HeaderParser.ReadHeaders(clientStream, connectRequest.Headers, cancellationToken);
......@@ -91,8 +89,7 @@ namespace Titanium.Web.Proxy
}
// send the response
await clientStream.WriteResponseAsync(connectArgs.HttpClient.Response,
cancellationToken: cancellationToken);
await clientStream.WriteResponseAsync(connectArgs.HttpClient.Response, cancellationToken);
return;
}
......@@ -101,20 +98,19 @@ namespace Titanium.Web.Proxy
await endPoint.InvokeBeforeTunnelConnectResponse(this, connectArgs, ExceptionFunc);
// send the response
await clientStream.WriteResponseAsync(connectArgs.HttpClient.Response,
cancellationToken: cancellationToken);
await clientStream.WriteResponseAsync(connectArgs.HttpClient.Response, cancellationToken);
return;
}
// write back successful CONNECT response
var response = ConnectResponse.CreateSuccessfulConnectResponse(version);
var response = ConnectResponse.CreateSuccessfulConnectResponse(requestLine.Version);
// Set ContentLength explicitly to properly handle HTTP 1.0
response.ContentLength = 0;
response.Headers.FixProxyHeaders();
connectArgs.HttpClient.Response = response;
await clientStream.WriteResponseAsync(response, cancellationToken: cancellationToken);
await clientStream.WriteResponseAsync(response, cancellationToken);
var clientHelloInfo = await SslTools.PeekClientHello(clientStream, BufferPool, cancellationToken);
......@@ -142,8 +138,8 @@ namespace Titanium.Web.Proxy
{
// todo: this is a hack, because Titanium does not support HTTP protocol changing currently
var connection = await tcpConnectionFactory.GetServerConnection(this, connectArgs,
isConnect: true, applicationProtocols: SslExtensions.Http2ProtocolAsList,
noCache: true, cancellationToken: cancellationToken);
true, SslExtensions.Http2ProtocolAsList,
true, cancellationToken);
http2Supported = connection.NegotiatedApplicationProtocol ==
SslApplicationProtocol.Http2;
......@@ -172,12 +168,12 @@ namespace Titanium.Web.Proxy
// don't pass cancellation token here
// it could cause floating server connections when client exits
prefetchConnectionTask = tcpConnectionFactory.GetServerConnection(this, connectArgs,
isConnect: true, applicationProtocols: null, noCache: false,
cancellationToken: CancellationToken.None);
true, null, false,
CancellationToken.None);
}
}
string connectHostname = httpUrl;
string connectHostname = requestLine.RequestUri.GetString();
int idx = connectHostname.IndexOf(":");
if (idx >= 0)
{
......@@ -216,6 +212,8 @@ namespace Titanium.Web.Proxy
// HTTPS server created - we can now decrypt the client's traffic
clientStream = new HttpClientStream(sslStream, BufferPool);
sslStream = null; // clientStream was created, no need to keep SSL stream reference
clientStream.DataRead += (o, args) => connectArgs.OnDecryptedDataSent(args.Buffer, args.Offset, args.Count);
clientStream.DataWrite += (o, args) => connectArgs.OnDecryptedDataReceived(args.Buffer, args.Offset, args.Count);
}
......@@ -255,8 +253,8 @@ namespace Titanium.Web.Proxy
// If we detected that client tunnel CONNECTs without SSL by checking for empty client hello then
// this connection should not be HTTPS.
var connection = await tcpConnectionFactory.GetServerConnection(this, connectArgs,
isConnect: true, applicationProtocols: SslExtensions.Http2ProtocolAsList,
noCache: true, cancellationToken: cancellationToken);
true, SslExtensions.Http2ProtocolAsList,
true, cancellationToken);
try
{
......@@ -271,7 +269,12 @@ namespace Titanium.Web.Proxy
try
{
// clientStream.Available should be at most BufferSize because it is using the same buffer size
await clientStream.ReadAsync(data, 0, available, cancellationToken);
int read = await clientStream.ReadAsync(data, 0, available, cancellationToken);
if (read != available)
{
throw new Exception("Internal error.");
}
await connection.Stream.WriteAsync(data, 0, available, true, cancellationToken);
}
finally
......@@ -327,8 +330,8 @@ namespace Titanium.Web.Proxy
}
var connection = await tcpConnectionFactory.GetServerConnection(this, connectArgs,
isConnect: true, applicationProtocols: SslExtensions.Http2ProtocolAsList,
noCache: true, cancellationToken: cancellationToken);
true, SslExtensions.Http2ProtocolAsList,
true, cancellationToken);
try
{
#if NETSTANDARD2_1
......
......@@ -20,11 +20,28 @@ namespace Titanium.Web.Proxy.Helpers
/// <param name="response">The response object.</param>
/// <param name="cancellationToken">Optional cancellation token for this async task.</param>
/// <returns>The Task.</returns>
internal async Task WriteResponseAsync(Response response, CancellationToken cancellationToken = default)
internal async ValueTask WriteResponseAsync(Response response, CancellationToken cancellationToken = default)
{
var headerBuilder = new HeaderBuilder();
// Write back response status to client
headerBuilder.WriteResponseLine(response.HttpVersion, response.StatusCode, response.StatusDescription);
await WriteAsync(response, headerBuilder, cancellationToken);
}
internal async ValueTask<RequestStatusInfo> ReadRequestLine(CancellationToken cancellationToken = default)
{
// read the first line HTTP command
string? httpCmd = await ReadLineAsync(cancellationToken);
if (string.IsNullOrEmpty(httpCmd))
{
return default;
}
Request.ParseRequestLine(httpCmd!, out string method, out var requestUri, out var version);
return new RequestStatusInfo { Method = method, RequestUri = requestUri, Version = version };
}
}
}
......@@ -170,7 +170,7 @@ namespace Titanium.Web.Proxy.Helpers
/// Determines whether is connect method.
/// </summary>
/// <returns>1: when CONNECT, 0: when valid HTTP method, -1: otherwise</returns>
internal static Task<int> IsConnectMethod(IPeekStream httpReader, IBufferPool bufferPool, CancellationToken cancellationToken = default)
internal static ValueTask<int> IsConnectMethod(IPeekStream httpReader, IBufferPool bufferPool, CancellationToken cancellationToken = default)
{
return startsWith(httpReader, bufferPool, "CONNECT", cancellationToken);
}
......@@ -179,7 +179,7 @@ namespace Titanium.Web.Proxy.Helpers
/// Determines whether is pri method (HTTP/2).
/// </summary>
/// <returns>1: when PRI, 0: when valid HTTP method, -1: otherwise</returns>
internal static Task<int> IsPriMethod(IPeekStream httpReader, IBufferPool bufferPool, CancellationToken cancellationToken = default)
internal static ValueTask<int> IsPriMethod(IPeekStream httpReader, IBufferPool bufferPool, CancellationToken cancellationToken = default)
{
return startsWith(httpReader, bufferPool, "PRI", cancellationToken);
}
......@@ -190,7 +190,7 @@ namespace Titanium.Web.Proxy.Helpers
/// <returns>
/// 1: when starts with the given string, 0: when valid HTTP method, -1: otherwise
/// </returns>
private static async Task<int> startsWith(IPeekStream httpReader, IBufferPool bufferPool, string expectedStart, CancellationToken cancellationToken = default)
private static async ValueTask<int> startsWith(IPeekStream httpReader, IBufferPool bufferPool, string expectedStart, CancellationToken cancellationToken = default)
{
const int lengthToCheck = 10;
if (bufferPool.BufferSize < lengthToCheck)
......
using System.IO;
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Titanium.Web.Proxy.Exceptions;
using Titanium.Web.Proxy.Http;
using Titanium.Web.Proxy.StreamExtended.BufferPool;
......@@ -8,7 +10,7 @@ namespace Titanium.Web.Proxy.Helpers
{
internal sealed class HttpServerStream : HttpStream
{
internal HttpServerStream(Stream stream, IBufferPool bufferPool)
internal HttpServerStream(Stream stream, IBufferPool bufferPool)
: base(stream, bufferPool)
{
}
......@@ -19,11 +21,34 @@ namespace Titanium.Web.Proxy.Helpers
/// <param name="request">The request object.</param>
/// <param name="cancellationToken">Optional cancellation token for this async task.</param>
/// <returns></returns>
internal async Task WriteRequestAsync(Request request, CancellationToken cancellationToken = default)
internal async ValueTask WriteRequestAsync(Request request, CancellationToken cancellationToken = default)
{
var headerBuilder = new HeaderBuilder();
headerBuilder.WriteRequestLine(request.Method, request.Url, request.HttpVersion);
headerBuilder.WriteRequestLine(request.Method, request.RequestUriString, request.HttpVersion);
await WriteAsync(request, headerBuilder, cancellationToken);
}
internal async ValueTask<ResponseStatusInfo> ReadResponseStatus(CancellationToken cancellationToken = default)
{
try
{
string httpStatus = await ReadLineAsync(cancellationToken) ??
throw new ServerConnectionException("Server connection was closed.");
if (httpStatus == string.Empty)
{
// is this really possible?
httpStatus = await ReadLineAsync(cancellationToken) ??
throw new ServerConnectionException("Server connection was closed.");
}
Response.ParseResponseLine(httpStatus, out var version, out int statusCode, out string description);
return new ResponseStatusInfo { Version = version, StatusCode = statusCode, Description = description };
}
catch (Exception e) when (!(e is ServerConnectionException))
{
throw new ServerConnectionException("Server connection was closed.");
}
}
}
}
using System.Threading;
using System.Threading.Tasks;
using Titanium.Web.Proxy.StreamExtended.Network;
internal class NullWriter : IHttpStreamWriter
{
public static NullWriter Instance { get; } = new NullWriter();
public void Write(byte[] buffer, int offset, int count)
{
}
#if NET45
public async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
}
#else
public Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
#endif
public ValueTask WriteLineAsync(CancellationToken cancellationToken = default)
{
throw new System.NotImplementedException();
}
public ValueTask WriteLineAsync(string value, CancellationToken cancellationToken = default)
{
throw new System.NotImplementedException();
}
}
using System;
using Titanium.Web.Proxy.Models;
namespace Titanium.Web.Proxy.Helpers
{
struct RequestStatusInfo
{
public string Method { get; set; }
public ByteString RequestUri { get; set; }
public Version Version { get; set; }
public bool IsEmpty()
{
return Method == null && RequestUri.Length == 0 && Version == null;
}
}
}
using System;
namespace Titanium.Web.Proxy.Helpers
{
struct ResponseStatusInfo
{
public Version Version { get; set; }
public int StatusCode { get; set; }
public string Description { get; set; }
}
}
......@@ -100,12 +100,10 @@ namespace Titanium.Web.Proxy.Helpers
/// <param name="onDataSend"></param>
/// <param name="onDataReceive"></param>
/// <param name="cancellationTokenSource"></param>
/// <param name="exceptionFunc"></param>
/// <returns></returns>
private static async Task sendRawTap(Stream clientStream, Stream serverStream, IBufferPool bufferPool,
Action<byte[], int, int>? onDataSend, Action<byte[], int, int>? onDataReceive,
CancellationTokenSource cancellationTokenSource,
ExceptionHandler exceptionFunc)
CancellationTokenSource cancellationTokenSource)
{
// Now async relay all server=>client & client=>server data
var sendRelay =
......@@ -139,8 +137,7 @@ namespace Titanium.Web.Proxy.Helpers
{
// todo: fix APM mode
return sendRawTap(clientStream, serverStream, bufferPool, onDataSend, onDataReceive,
cancellationTokenSource,
exceptionFunc);
cancellationTokenSource);
}
}
}
using System;
using Titanium.Web.Proxy.Extensions;
using Titanium.Web.Proxy.Models;
using Titanium.Web.Proxy.StreamExtended;
namespace Titanium.Web.Proxy.Http
......@@ -9,7 +10,7 @@ namespace Titanium.Web.Proxy.Http
/// </summary>
public class ConnectRequest : Request
{
public ConnectRequest(string authority)
internal ConnectRequest(ByteString authority)
{
Method = "CONNECT";
Authority = authority;
......
......@@ -41,11 +41,21 @@ namespace Titanium.Web.Proxy.Http
WriteLine();
}
public void WriteHeaders(HeaderCollection headers)
public void WriteHeaders(HeaderCollection headers, bool sendProxyAuthorization = true,
string? upstreamProxyUserName = null, string? upstreamProxyPassword = null)
{
if (upstreamProxyUserName != null && upstreamProxyPassword != null)
{
WriteHeader(HttpHeader.ProxyConnectionKeepAlive);
WriteHeader(HttpHeader.GetProxyAuthorizationHeader(upstreamProxyUserName, upstreamProxyPassword));
}
foreach (var header in headers)
{
WriteHeader(header);
if (sendProxyAuthorization || !KnownHeaders.ProxyAuthorization.Equals(header.Name))
{
WriteHeader(header);
}
}
WriteLine();
......
......@@ -7,7 +7,7 @@ namespace Titanium.Web.Proxy.Http
{
internal static class HeaderParser
{
internal static async Task ReadHeaders(ILineStream reader, HeaderCollection headerCollection,
internal static async ValueTask ReadHeaders(ILineStream reader, HeaderCollection headerCollection,
CancellationToken cancellationToken)
{
string? tmpLine;
......@@ -24,17 +24,5 @@ namespace Titanium.Web.Proxy.Http
headerCollection.AddHeader(headerName, headerValue);
}
}
/// <summary>
/// Increase size of buffer and copy existing content to new buffer
/// </summary>
/// <param name="buffer"></param>
/// <param name="size"></param>
private static void resizeBuffer(ref byte[] buffer, long size)
{
var newBuffer = new byte[size];
Buffer.BlockCopy(buffer, 0, newBuffer, 0, buffer.Length);
buffer = newBuffer;
}
}
}
......@@ -103,9 +103,8 @@ namespace Titanium.Web.Proxy.Http
/// <summary>
/// Prepare and send the http(s) request
/// </summary>
/// <returns></returns>
internal async Task SendRequest(bool enable100ContinueBehaviour, bool isTransparent,
CancellationToken cancellationToken)
/// <returns></returns>
internal async Task SendRequest(bool enable100ContinueBehaviour, bool isTransparent, CancellationToken cancellationToken)
{
var upstreamProxy = Connection.UpStreamProxy;
......@@ -114,23 +113,17 @@ namespace Titanium.Web.Proxy.Http
var serverStream = Connection.Stream;
string url;
if (useUpstreamProxy || isTransparent)
if (!useUpstreamProxy || isTransparent)
{
url = Request.Url;
url = Request.RequestUriString;
}
else
{
url = Request.RequestUri.GetOriginalPathAndQuery();
if (url == string.Empty)
{
url = "/";
}
url = Request.RequestUri.ToString();
}
var headerBuilder = new HeaderBuilder();
// prepare the request & headers
headerBuilder.WriteRequestLine(Request.Method, url, Request.HttpVersion);
string? upstreamProxyUserName = null;
string? upstreamProxyPassword = null;
// Send Authentication to Upstream proxy if needed
if (!isTransparent && upstreamProxy != null
......@@ -138,21 +131,16 @@ namespace Titanium.Web.Proxy.Http
&& !string.IsNullOrEmpty(upstreamProxy.UserName)
&& upstreamProxy.Password != null)
{
headerBuilder.WriteHeader(HttpHeader.ProxyConnectionKeepAlive);
headerBuilder.WriteHeader(HttpHeader.GetProxyAuthorizationHeader(upstreamProxy.UserName, upstreamProxy.Password));
}
// write request headers
foreach (var header in Request.Headers)
{
if (isTransparent || header.Name != KnownHeaders.ProxyAuthorization.String)
{
headerBuilder.WriteHeader(header);
}
upstreamProxyUserName = upstreamProxy.UserName;
upstreamProxyPassword = upstreamProxy.Password;
}
headerBuilder.WriteLine();
// prepare the request & headers
var headerBuilder = new HeaderBuilder();
headerBuilder.WriteRequestLine(Request.Method, url, Request.HttpVersion);
headerBuilder.WriteHeaders(Request.Headers, !isTransparent, upstreamProxyUserName, upstreamProxyPassword);
// write request headers
await serverStream.WriteHeadersAsync(headerBuilder, cancellationToken);
if (enable100ContinueBehaviour && Request.ExpectContinue)
......@@ -183,28 +171,10 @@ namespace Titanium.Web.Proxy.Http
return;
}
string httpStatus;
try
{
httpStatus = await Connection.Stream.ReadLineAsync(cancellationToken) ??
throw new ServerConnectionException("Server connection was closed.");
}
catch (Exception e) when (!(e is ServerConnectionException))
{
throw new ServerConnectionException("Server connection was closed.");
}
if (httpStatus == string.Empty)
{
httpStatus = await Connection.Stream.ReadLineAsync(cancellationToken) ??
throw new ServerConnectionException("Server connection was closed.");
}
Response.ParseResponseLine(httpStatus, out var version, out int statusCode, out string statusDescription);
Response.HttpVersion = version;
Response.StatusCode = statusCode;
Response.StatusDescription = statusDescription;
var httpStatus = await Connection.Stream.ReadResponseStatus(cancellationToken);
Response.HttpVersion = httpStatus.Version;
Response.StatusCode = httpStatus.StatusCode;
Response.StatusDescription = httpStatus.Description;
// Read the response headers in to unique and non-unique header collections
await HeaderParser.ReadHeaders(Connection.Stream, Response.Headers, cancellationToken);
......
......@@ -24,26 +24,15 @@ namespace Titanium.Web.Proxy.Http
/// </summary>
public bool IsHttps { get; internal set; }
private ByteString originalUrlData;
private ByteString urlData;
private ByteString requestUriString8;
internal ByteString OriginalUrlData
internal ByteString RequestUriString8
{
get => originalUrlData;
get => requestUriString8;
set
{
originalUrlData = value;
UrlData = value;
}
}
private protected ByteString UrlData
{
get => urlData;
set
{
urlData = value;
var scheme = getUriScheme(UrlData);
requestUriString8 = value;
var scheme = getUriScheme(value);
if (scheme.Length > 0)
{
IsHttps = scheme.Equals(ProxyServer.UriSchemeHttps8);
......@@ -51,24 +40,42 @@ namespace Titanium.Web.Proxy.Http
}
}
internal string? Authority { get; set; }
internal ByteString Authority { get; set; }
/// <summary>
/// The original request Url.
/// Request HTTP Uri.
/// </summary>
public string OriginalUrl => originalUrlData.GetString();
public Uri RequestUri
{
get
{
string url = Url;
try
{
return new Uri(url);
}
catch (Exception ex)
{
throw new Exception($"Invalid URI: '{url}'", ex);
}
}
set
{
Url = value.ToString();
}
}
/// <summary>
/// Request HTTP Uri.
/// The request url as it is in the HTTP header
/// </summary>
public Uri RequestUri
public string Url
{
get
{
string url = UrlData.GetString();
if (getUriScheme(UrlData).Length == 0)
string url = RequestUriString8.GetString();
if (getUriScheme(RequestUriString8).Length == 0)
{
string? hostAndPath = Host ?? Authority;
string? hostAndPath = Host ?? Authority.GetString();
if (url.StartsWith("/"))
{
......@@ -82,28 +89,26 @@ namespace Titanium.Web.Proxy.Http
url = string.Concat(IsHttps ? "https://" : "http://", hostAndPath);
}
try
{
return new Uri(url);
}
catch (Exception ex)
{
throw new Exception($"Invalid URI: '{url}'", ex);
}
return url;
}
set
{
RequestUriString = value;
}
}
/// <summary>
/// The request url as it is in the HTTP header
/// The request uri as it is in the HTTP header
/// </summary>
public string Url
public string RequestUriString
{
get => UrlData.GetString();
get => RequestUriString8.GetString();
set
{
UrlData = value.GetByteString();
RequestUriString8 = (ByteString)value;
if (Host != null)
var scheme = getUriScheme(RequestUriString8);
if (scheme.Length > 0 && Host != null)
{
var uri = new Uri(value);
Host = uri.Authority;
......@@ -111,13 +116,6 @@ namespace Titanium.Web.Proxy.Http
}
}
[Obsolete("This property is obsolete. Use Url property instead")]
public string RequestUriString
{
get => Url;
set => Url = value;
}
/// <summary>
/// Has request body?
/// </summary>
......@@ -219,7 +217,7 @@ namespace Titanium.Web.Proxy.Http
get
{
var headerBuilder = new HeaderBuilder();
headerBuilder.WriteRequestLine(Method, Url, HttpVersion);
headerBuilder.WriteRequestLine(Method, RequestUriString, HttpVersion);
headerBuilder.WriteHeaders(Headers);
return headerBuilder.GetString(HttpHeader.Encoding);
}
......@@ -256,7 +254,7 @@ namespace Titanium.Web.Proxy.Http
}
}
internal static void ParseRequestLine(string httpCmd, out string httpMethod, out string httpUrl,
internal static void ParseRequestLine(string httpCmd, out string method, out ByteString requestUri,
out Version version)
{
int firstSpace = httpCmd.IndexOf(' ');
......@@ -271,21 +269,21 @@ namespace Titanium.Web.Proxy.Http
// break up the line into three components (method, remote URL & Http Version)
// Find the request Verb
httpMethod = httpCmd.Substring(0, firstSpace);
if (!isAllUpper(httpMethod))
method = httpCmd.Substring(0, firstSpace);
if (!isAllUpper(method))
{
httpMethod = httpMethod.ToUpper();
method = method.ToUpper();
}
version = HttpHeader.Version11;
if (firstSpace == lastSpace)
{
httpUrl = httpCmd.AsSpan(firstSpace + 1).ToString();
requestUri = (ByteString)httpCmd.AsSpan(firstSpace + 1).ToString();
}
else
{
httpUrl = httpCmd.AsSpan(firstSpace + 1, lastSpace - firstSpace - 1).ToString();
requestUri = (ByteString)httpCmd.AsSpan(firstSpace + 1, lastSpace - firstSpace - 1).ToString();
// parse the HTTP version
var httpVersion = httpCmd.AsSpan(lastSpace + 1);
......
......@@ -196,7 +196,7 @@ namespace Titanium.Web.Proxy.Http
/// Use the encoding specified to decode the byte[] data to string
/// </summary>
[Browsable(false)]
public string BodyString => bodyString ?? (bodyString = Encoding.GetString(Body));
public string BodyString => bodyString ??= Encoding.GetString(Body);
/// <summary>
/// Was the body read by user?
......@@ -210,6 +210,8 @@ namespace Titanium.Web.Proxy.Http
internal bool Locked { get; set; }
internal bool BodyAvailable => BodyInternal != null;
internal bool IsBodySent { get; set; }
internal abstract void EnsureBodyAvailable(bool throwWhenNotReadYet = true);
......
......@@ -264,8 +264,8 @@ namespace Titanium.Web.Proxy.Http2
request.HttpVersion = HttpVersion.Version20;
request.Method = method.GetString();
request.IsHttps = headerListener.Scheme == ProxyServer.UriSchemeHttps;
request.Authority = headerListener.Authority.GetString();
request.OriginalUrlData = path;
request.Authority = headerListener.Authority;
request.RequestUriString8 = path;
//request.RequestUri = headerListener.GetUri();
}
......@@ -475,7 +475,7 @@ namespace Titanium.Web.Proxy.Http2
encoder.EncodeHeader(writer, StaticTable.KnownHeaderMethod, request.Method.GetByteString());
encoder.EncodeHeader(writer, StaticTable.KnownHeaderAuhtority, uri.Authority.GetByteString());
encoder.EncodeHeader(writer, StaticTable.KnownHeaderScheme, uri.Scheme.GetByteString());
encoder.EncodeHeader(writer, StaticTable.KnownHeaderPath, request.Url.GetByteString(), false,
encoder.EncodeHeader(writer, StaticTable.KnownHeaderPath, request.RequestUriString8, false,
HpackUtil.IndexType.None, false);
}
else
......
using System;
using System.Text;
using Titanium.Web.Proxy.Extensions;
namespace Titanium.Web.Proxy.Models
{
......@@ -28,11 +29,31 @@ namespace Titanium.Web.Proxy.Models
return Data.Span.SequenceEqual(other.Data.Span);
}
public int IndexOf(byte value)
{
return Span.IndexOf(value);
}
public ByteString Slice(int start)
{
return Data.Slice(start);
}
public ByteString Slice(int start, int length)
{
return Data.Slice(start, length);
}
public override int GetHashCode()
{
return Data.GetHashCode();
}
public override string ToString()
{
return this.GetString();
}
public static explicit operator ByteString(string str) => new ByteString(Encoding.ASCII.GetBytes(str));
public static implicit operator ByteString(byte[] data) => new ByteString(data);
......
using System;
using System.IO;
using System.Security.Cryptography.X509Certificates;
using System.Threading;
using Org.BouncyCastle.Asn1;
using Org.BouncyCastle.Asn1.Pkcs;
using Org.BouncyCastle.Asn1.X509;
......@@ -219,11 +218,9 @@ namespace Titanium.Web.Proxy.Network.Certificate
/// <param name="subject">The s subject cn.</param>
/// <param name="switchToMtaIfNeeded">if set to <c>true</c> [switch to MTA if needed].</param>
/// <param name="signingCert">The signing cert.</param>
/// <param name="cancellationToken">Task cancellation token</param>
/// <returns>X509Certificate2.</returns>
private X509Certificate2 makeCertificateInternal(string subject,
bool switchToMtaIfNeeded, X509Certificate2? signingCert = null,
CancellationToken cancellationToken = default)
bool switchToMtaIfNeeded, X509Certificate2? signingCert = null)
{
return makeCertificateInternal(subject, $"CN={subject}",
DateTime.UtcNow.AddDays(-certificateGraceDays), DateTime.UtcNow.AddDays(certificateValidDays),
......
......@@ -38,8 +38,8 @@ namespace Titanium.Web.Proxy.Network
try
{
// setup connection
currentConnection = currentConnection as TcpServerConnection ??
await generator();
currentConnection ??= await generator();
// try
@continue = await action(currentConnection);
......
......@@ -180,16 +180,41 @@ namespace Titanium.Web.Proxy.Network.Tcp
session.CustomUpStreamProxyUsed = customUpStreamProxy;
var uri = session.HttpClient.Request.RequestUri;
var request = session.HttpClient.Request;
string host;
int port;
if (request.Authority.Length > 0)
{
var authority = request.Authority;
int idx = authority.IndexOf((byte)':');
if (idx == -1)
{
host = authority.GetString();
port = 80;
}
else
{
host = authority.Slice(0, idx).GetString();
port = int.Parse(authority.Slice(idx + 1).GetString());
}
}
else
{
var uri = request.RequestUri;
host = uri.Host;
port = uri.Port;
}
return await GetServerConnection(
uri.Host,
uri.Port,
host,
port,
session.HttpClient.Request.HttpVersion,
isHttps, applicationProtocols, isConnect,
server, session, session.HttpClient.UpStreamEndPoint ?? server.UpStreamEndPoint,
customUpStreamProxy ?? (isHttps ? server.UpStreamHttpsProxy : server.UpStreamHttpProxy),
noCache, cancellationToken);
}
/// <summary>
/// Gets a TCP connection to server from connection pool.
/// </summary>
......@@ -281,21 +306,25 @@ namespace Titanium.Web.Proxy.Network.Tcp
}
}
bool useUpstreamProxy = false;
bool useUpstreamProxy1 = false;
// check if external proxy is set for HTTP/HTTPS
if (externalProxy != null &&
!(externalProxy.HostName == remoteHostName && externalProxy.Port == remotePort))
if (externalProxy != null && !(externalProxy.HostName == remoteHostName && externalProxy.Port == remotePort))
{
useUpstreamProxy = true;
useUpstreamProxy1 = true;
// check if we need to ByPass
if (externalProxy.BypassLocalhost && NetworkHelper.IsLocalIpAddress(remoteHostName))
{
useUpstreamProxy = false;
useUpstreamProxy1 = false;
}
}
if (!useUpstreamProxy1)
{
externalProxy = null;
}
TcpClient? tcpClient = null;
HttpServerStream? stream = null;
......@@ -307,8 +336,8 @@ namespace Titanium.Web.Proxy.Network.Tcp
retry:
try
{
string hostname = useUpstreamProxy ? externalProxy!.HostName : remoteHostName;
int port = useUpstreamProxy ? externalProxy!.Port : remotePort;
string hostname = externalProxy != null ? externalProxy.HostName : remoteHostName;
int port = externalProxy?.Port ?? remotePort;
var ipAddresses = await Dns.GetHostAddressesAsync(hostname);
if (ipAddresses == null || ipAddresses.Length == 0)
......@@ -348,7 +377,37 @@ retry:
tcpClient.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
}
await tcpClient.ConnectAsync(ipAddress, port);
var connectTask = tcpClient.ConnectAsync(ipAddress, port);
await Task.WhenAny(connectTask, Task.Delay(proxyServer.ConnectTimeOutSeconds * 1000));
if (!connectTask.IsCompleted || !tcpClient.Connected)
{
// here we can just do some cleanup and let the loop continue since
// we will either get a connection or wind up with a null tcpClient
// which will throw
try
{
connectTask.Dispose();
}
catch
{
// ignore
}
try
{
#if NET45
tcpClient?.Close();
#else
tcpClient?.Dispose();
#endif
tcpClient = null;
}
catch
{
// ignore
}
continue;
}
break;
}
catch (Exception e)
......@@ -366,6 +425,17 @@ retry:
if (tcpClient == null)
{
if (session != null && proxyServer.CustomUpStreamProxyFailureFunc != null)
{
var newUpstreamProxy = await proxyServer.CustomUpStreamProxyFailureFunc(session);
if (newUpstreamProxy != null)
{
session.CustomUpStreamProxyUsed = newUpstreamProxy;
session.TimeLine["Retrying Upstream Proxy Connection"] = DateTime.Now;
return await createServerConnection(remoteHostName, remotePort, httpVersion, isHttps, sslProtocol, applicationProtocols, isConnect, proxyServer, session, upStreamEndPoint, externalProxy, cacheKey, cancellationToken);
}
}
throw new Exception($"Could not establish connection to {hostname}", lastException);
}
......@@ -378,34 +448,30 @@ retry:
stream = new HttpServerStream(tcpClient.GetStream(), proxyServer.BufferPool);
if (useUpstreamProxy && (isConnect || isHttps))
if (externalProxy != null && (isConnect || isHttps))
{
string authority = $"{remoteHostName}:{remotePort}";
var authority = $"{remoteHostName}:{remotePort}".GetByteString();
var connectRequest = new ConnectRequest(authority)
{
IsHttps = isHttps,
OriginalUrlData = HttpHeader.Encoding.GetBytes(authority),
RequestUriString8 = authority,
HttpVersion = httpVersion
};
connectRequest.Headers.AddHeader(KnownHeaders.Connection, KnownHeaders.ConnectionKeepAlive);
if (!string.IsNullOrEmpty(externalProxy!.UserName) && externalProxy.Password != null)
if (!string.IsNullOrEmpty(externalProxy.UserName) && externalProxy.Password != null)
{
connectRequest.Headers.AddHeader(HttpHeader.ProxyConnectionKeepAlive);
connectRequest.Headers.AddHeader(
HttpHeader.GetProxyAuthorizationHeader(externalProxy.UserName, externalProxy.Password));
connectRequest.Headers.AddHeader(HttpHeader.GetProxyAuthorizationHeader(externalProxy.UserName, externalProxy.Password));
}
await stream.WriteRequestAsync(connectRequest, cancellationToken: cancellationToken);
await stream.WriteRequestAsync(connectRequest, cancellationToken);
string httpStatus = await stream.ReadLineAsync(cancellationToken)
?? throw new ServerConnectionException("Server connection was closed.");
var httpStatus = await stream.ReadResponseStatus(cancellationToken);
Response.ParseResponseLine(httpStatus, out _, out int statusCode, out string statusDescription);
if (statusCode != 200 && !statusDescription.EqualsIgnoreCase("OK")
&& !statusDescription.EqualsIgnoreCase("Connection Established"))
if (httpStatus.StatusCode != 200 && !httpStatus.Description.EqualsIgnoreCase("OK")
&& !httpStatus.Description.EqualsIgnoreCase("Connection Established"))
{
throw new Exception("Upstream proxy failed to create a secure tunnel");
}
......@@ -455,7 +521,7 @@ retry:
}
return new TcpServerConnection(proxyServer, tcpClient, stream, remoteHostName, remotePort, isHttps,
negotiatedApplicationProtocol, httpVersion, useUpstreamProxy, externalProxy, upStreamEndPoint, cacheKey);
negotiatedApplicationProtocol, httpVersion, externalProxy, upStreamEndPoint, cacheKey);
}
......@@ -628,4 +694,3 @@ retry:
}
}
}
......@@ -15,9 +15,11 @@ namespace Titanium.Web.Proxy.Network.Tcp
/// </summary>
internal class TcpServerConnection : IDisposable
{
public Guid Id { get; } = Guid.NewGuid();
internal TcpServerConnection(ProxyServer proxyServer, TcpClient tcpClient, HttpServerStream stream,
string hostName, int port, bool isHttps, SslApplicationProtocol negotiatedApplicationProtocol,
Version version, bool useUpstreamProxy, IExternalProxy? upStreamProxy, IPEndPoint? upStreamEndPoint, string cacheKey)
Version version, IExternalProxy? upStreamProxy, IPEndPoint? upStreamEndPoint, string cacheKey)
{
TcpClient = tcpClient;
LastAccess = DateTime.Now;
......@@ -29,7 +31,6 @@ namespace Titanium.Web.Proxy.Network.Tcp
IsHttps = isHttps;
NegotiatedApplicationProtocol = negotiatedApplicationProtocol;
Version = version;
UseUpstreamProxy = useUpstreamProxy;
UpStreamProxy = upStreamProxy;
UpStreamEndPoint = upStreamEndPoint;
......@@ -50,8 +51,6 @@ namespace Titanium.Web.Proxy.Network.Tcp
internal SslApplicationProtocol NegotiatedApplicationProtocol { get; set; }
internal bool UseUpstreamProxy { get; set; }
/// <summary>
/// Local NIC via connection is made
/// </summary>
......
......@@ -196,6 +196,12 @@ namespace Titanium.Web.Proxy
/// </summary>
public int ConnectionTimeOutSeconds { get; set; } = 60;
/// <summary>
/// Seconds server connection are to wait for connection to be established.
/// Default value is 20 seconds.
/// </summary>
public int ConnectTimeOutSeconds { get; set; } = 20;
/// <summary>
/// Maximum number of concurrent connections per remote host in cache.
/// Only valid when connection pooling is enabled.
......@@ -277,6 +283,12 @@ namespace Titanium.Web.Proxy
/// </summary>
public Func<SessionEventArgsBase, Task<IExternalProxy?>>? GetCustomUpStreamProxyFunc { get; set; }
/// <summary>
/// A callback to provide a chance for an upstream proxy failure to be handled by a new upstream proxy.
/// User should return the ExternalProxy object with valid credentials or null.
/// </summary>
public Func<SessionEventArgsBase, Task<IExternalProxy?>>? CustomUpStreamProxyFailureFunc { get; set; }
/// <summary>
/// Callback for error events in this proxy instance.
/// </summary>
......
......@@ -58,8 +58,8 @@ namespace Titanium.Web.Proxy
}
// read the request line
string? httpCmd = await clientStream.ReadLineAsync(cancellationToken);
if (string.IsNullOrEmpty(httpCmd))
var requestLine = await clientStream.ReadRequestLine(cancellationToken);
if (requestLine.IsEmpty())
{
return;
}
......@@ -73,8 +73,6 @@ namespace Titanium.Web.Proxy
{
try
{
Request.ParseRequestLine(httpCmd!, out string httpMethod, out string httpUrl, out var version);
// Read the request headers in to unique and non-unique header collections
await HeaderParser.ReadHeaders(clientStream, args.HttpClient.Request.Headers,
cancellationToken);
......@@ -86,10 +84,10 @@ namespace Titanium.Web.Proxy
request.Authority = connectRequest.Authority;
}
request.OriginalUrlData = HttpHeader.Encoding.GetBytes(httpUrl);
request.RequestUriString8 = requestLine.RequestUri;
request.Method = httpMethod;
request.HttpVersion = version;
request.Method = requestLine.Method;
request.HttpVersion = requestLine.Version;
if (!args.IsTransparent)
{
......@@ -99,8 +97,7 @@ namespace Titanium.Web.Proxy
await onBeforeResponse(args);
// send the response
await clientStream.WriteResponseAsync(args.HttpClient.Response,
cancellationToken: cancellationToken);
await clientStream.WriteResponseAsync(args.HttpClient.Response, cancellationToken);
return;
}
......@@ -173,7 +170,7 @@ namespace Titanium.Web.Proxy
connection = null;
}
var result = await handleHttpSessionRequest(httpMethod, httpUrl, version, args, connection,
var result = await handleHttpSessionRequest(args, connection,
clientConnection.NegotiatedApplicationProtocol,
cancellationToken, cancellationTokenSource);
......@@ -253,19 +250,35 @@ namespace Titanium.Web.Proxy
}
}
private async Task<RetryResult> handleHttpSessionRequest(string requestHttpMethod, string requestHttpUrl, Version requestVersion, SessionEventArgs args,
private async Task<RetryResult> handleHttpSessionRequest(SessionEventArgs args,
TcpServerConnection? serverConnection, SslApplicationProtocol sslApplicationProtocol,
CancellationToken cancellationToken, CancellationTokenSource cancellationTokenSource)
{
args.HttpClient.Request.Locked = true;
// do not cache server connections for WebSockets
bool noCache = args.HttpClient.Request.UpgradeToWebSocket;
if (noCache)
{
serverConnection = null;
}
// a connection generator task with captured parameters via closure.
Func<Task<TcpServerConnection>> generator = () =>
tcpConnectionFactory.GetServerConnection(this, args, isConnect: false,
applicationProtocol: sslApplicationProtocol,
noCache: false, cancellationToken: cancellationToken);
tcpConnectionFactory.GetServerConnection(this,
args,
false,
sslApplicationProtocol,
noCache,
cancellationToken);
// for connection pool, retry fails until cache is exhausted.
return await retryPolicy<ServerConnectionException>().ExecuteAsync(async (connection) =>
{
// set the connection and send request headers
args.HttpClient.SetConnection(connection);
args.TimeLine["Connection Ready"] = DateTime.Now;
if (args.HttpClient.Request.UpgradeToWebSocket)
......@@ -273,29 +286,24 @@ namespace Titanium.Web.Proxy
args.HttpClient.ConnectRequest!.TunnelType = TunnelType.Websocket;
// if upgrading to websocket then relay the request without reading the contents
await handleWebSocketUpgrade(requestHttpMethod, requestHttpUrl, requestVersion, args, args.HttpClient.Request,
args.HttpClient.Response, args.ClientStream,
connection, cancellationTokenSource, cancellationToken);
await handleWebSocketUpgrade(args, args.ClientStream, connection, cancellationTokenSource, cancellationToken);
return false;
}
// construct the web request that we are going to issue on behalf of the client.
await handleHttpSessionRequest(connection, args);
await handleHttpSessionRequest(args);
return true;
}, generator, serverConnection);
}
private async Task handleHttpSessionRequest(TcpServerConnection connection, SessionEventArgs args)
private async Task handleHttpSessionRequest(SessionEventArgs args)
{
var cancellationToken = args.CancellationTokenSource.Token;
var request = args.HttpClient.Request;
request.Locked = true;
var body = request.CompressBodyAndUpdateContentLength();
// set the connection and send request headers
args.HttpClient.SetConnection(connection);
await args.HttpClient.SendRequest(Enable100ContinueBehaviour, args.IsTransparent,
cancellationToken);
......
......@@ -70,7 +70,7 @@ namespace Titanium.Web.Proxy
if (response.Locked)
{
// write custom user response with body and return.
await clientStream.WriteResponseAsync(response, cancellationToken: cancellationToken);
await clientStream.WriteResponseAsync(response, cancellationToken);
if (args.HttpClient.HasConnection && !args.HttpClient.CloseServerConnection)
{
......@@ -93,8 +93,7 @@ namespace Titanium.Web.Proxy
// clear current response
await args.ClearResponse(cancellationToken);
await handleHttpSessionRequest(args.HttpClient.Request.Method, args.HttpClient.Request.Url, args.HttpClient.Request.HttpVersion,
args, null, args.ClientConnection.NegotiatedApplicationProtocol,
await handleHttpSessionRequest(args, null, args.ClientConnection.NegotiatedApplicationProtocol,
cancellationToken, args.CancellationTokenSource);
return;
}
......@@ -106,26 +105,25 @@ namespace Titanium.Web.Proxy
response.Headers.FixProxyHeaders();
}
if (response.IsBodyRead)
{
await clientStream.WriteResponseAsync(response, cancellationToken: cancellationToken);
}
else
await clientStream.WriteResponseAsync(response, cancellationToken);
if (response.OriginalHasBody)
{
// Write back response status to client
var headerBuilder = new HeaderBuilder();
headerBuilder.WriteResponseLine(response.HttpVersion, response.StatusCode, response.StatusDescription);
headerBuilder.WriteHeaders(response.Headers);
await clientStream.WriteHeadersAsync(headerBuilder, cancellationToken);
// Write body if exists
if (response.HasBody)
if (response.IsBodySent)
{
// syphon out body
await args.SyphonOutBodyAsync(false, cancellationToken);
}
else
{
await args.CopyResponseBodyAsync(clientStream, TransformationMode.None,
cancellationToken);
// Copy body if exists
var serverStream = args.HttpClient.Connection.Stream;
await serverStream.CopyBodyAsync(response, false, clientStream, TransformationMode.None,
args.OnDataReceived, cancellationToken);
}
}
args.TimeLine["Response Sent"] = DateTime.Now;
}
......
using System.Threading;
using System;
using System.Threading;
using System.Threading.Tasks;
namespace Titanium.Web.Proxy.StreamExtended.Network
......@@ -8,5 +9,8 @@ namespace Titanium.Web.Proxy.StreamExtended.Network
int Read(byte[] buffer, int offset, int count);
Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken);
Task CopyBodyAsync(IHttpStreamWriter writer, bool isChunked, long contentLength,
Action<byte[], int, int>? onCopy, CancellationToken cancellationToken);
}
}
......@@ -7,13 +7,14 @@ namespace Titanium.Web.Proxy.StreamExtended.Network
/// <summary>
/// A concrete implementation of this interface is required when calling CopyStream.
/// </summary>
internal interface IHttpStreamWriter
public interface IHttpStreamWriter
{
void Write(byte[] buffer, int i, int bufferLength);
void Write(byte[] buffer, int offset, int count);
Task WriteAsync(byte[] buffer, int i, int bufferLength, CancellationToken cancellationToken);
Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken);
Task CopyBodyAsync(IHttpStreamReader streamReader, bool isChunked, long contentLength,
Action<byte[], int, int>? onCopy, CancellationToken cancellationToken);
ValueTask WriteLineAsync(CancellationToken cancellationToken = default);
ValueTask WriteLineAsync(string value, CancellationToken cancellationToken = default);
}
}
......@@ -154,7 +154,7 @@ namespace Titanium.Web.Proxy.StreamExtended
if(extensionsStartPosition < recordLength + 5)
{
extensions = await ReadExtensions(majorVersion, minorVersion, peekStream, bufferPool, cancellationToken);
extensions = await ReadExtensions(majorVersion, minorVersion, peekStream, cancellationToken);
}
var clientHelloInfo = new ClientHelloInfo(3, majorVersion, minorVersion, random, sessionId, ciphers, peekStream.Position)
......@@ -292,7 +292,7 @@ namespace Titanium.Web.Proxy.StreamExtended
if (extensionsStartPosition < recordLength + 5)
{
extensions = await ReadExtensions(majorVersion, minorVersion, peekStream, bufferPool, cancellationToken);
extensions = await ReadExtensions(majorVersion, minorVersion, peekStream, cancellationToken);
}
var serverHelloInfo = new ServerHelloInfo(3, majorVersion, minorVersion, random, sessionId, cipherSuite, peekStream.Position)
......@@ -308,7 +308,7 @@ namespace Titanium.Web.Proxy.StreamExtended
return null;
}
private static async Task<Dictionary<string, SslExtension>?> ReadExtensions(int majorVersion, int minorVersion, PeekStreamReader peekStreamReader, IBufferPool bufferPool, CancellationToken cancellationToken)
private static async Task<Dictionary<string, SslExtension>?> ReadExtensions(int majorVersion, int minorVersion, PeekStreamReader peekStreamReader, CancellationToken cancellationToken)
{
Dictionary<string, SslExtension>? extensions = null;
if (majorVersion > 3 || majorVersion == 3 && minorVersion >= 1)
......
......@@ -75,23 +75,24 @@ namespace Titanium.Web.Proxy
// HTTPS server created - we can now decrypt the client's traffic
clientStream = new HttpClientStream(sslStream, BufferPool);
sslStream = null; // clientStream was created, no need to keep SSL stream reference
}
catch (Exception e)
{
var certname = certificate?.GetNameInfo(X509NameType.SimpleName, false);
var certName = certificate?.GetNameInfo(X509NameType.SimpleName, false);
var session = new SessionEventArgs(this, endPoint, clientConnection, clientStream, null,
cancellationTokenSource);
throw new ProxyConnectException(
$"Couldn't authenticate host '{httpsHostName}' with certificate '{certname}'.", e, session);
$"Couldn't authenticate host '{httpsHostName}' with certificate '{certName}'.", e, session);
}
}
else
{
var connection = await tcpConnectionFactory.GetServerConnection(httpsHostName, endPoint.Port,
httpVersion: HttpHeader.VersionUnknown, isHttps: false, applicationProtocols: null,
isConnect: true, proxyServer: this, session:null, upStreamEndPoint: UpStreamEndPoint,
externalProxy: UpStreamHttpsProxy, noCache: true, cancellationToken: cancellationToken);
HttpHeader.VersionUnknown, false, null,
true, this, null, UpStreamEndPoint,
UpStreamHttpsProxy, true, cancellationToken);
try
{
......
......@@ -16,42 +16,25 @@ namespace Titanium.Web.Proxy
/// <summary>
/// Handle upgrade to websocket
/// </summary>
private async Task handleWebSocketUpgrade(string requestHttpMethod, string requestHttpUrl, Version requestVersion,
SessionEventArgs args, Request request, Response response,
private async Task handleWebSocketUpgrade(SessionEventArgs args,
HttpClientStream clientStream, TcpServerConnection serverConnection,
CancellationTokenSource cancellationTokenSource, CancellationToken cancellationToken)
{
// prepare the prefix content
var headerBuilder = new HeaderBuilder();
headerBuilder.WriteRequestLine(requestHttpMethod, requestHttpUrl, requestVersion);
headerBuilder.WriteHeaders(request.Headers);
await serverConnection.Stream.WriteHeadersAsync(headerBuilder, cancellationToken);
await serverConnection.Stream.WriteRequestAsync(args.HttpClient.Request, cancellationToken);
string httpStatus;
try
{
httpStatus = await serverConnection.Stream.ReadLineAsync(cancellationToken)
?? throw new ServerConnectionException("Server connection was closed.");
}
catch (Exception e) when (!(e is ServerConnectionException))
{
throw new ServerConnectionException("Server connection was closed.", e);
}
Response.ParseResponseLine(httpStatus, out var responseVersion,
out int responseStatusCode,
out string responseStatusDescription);
response.HttpVersion = responseVersion;
response.StatusCode = responseStatusCode;
response.StatusDescription = responseStatusDescription;
var httpStatus = await serverConnection.Stream.ReadResponseStatus(cancellationToken);
var response = args.HttpClient.Response;
response.HttpVersion = httpStatus.Version;
response.StatusCode = httpStatus.StatusCode;
response.StatusDescription = httpStatus.Description;
await HeaderParser.ReadHeaders(serverConnection.Stream, response.Headers,
cancellationToken);
if (!args.IsTransparent)
{
await clientStream.WriteResponseAsync(response,
cancellationToken: cancellationToken);
await clientStream.WriteResponseAsync(response, cancellationToken);
}
// If user requested call back then do it
......
......@@ -22,7 +22,7 @@ namespace Titanium.Web.Proxy.IntegrationTests.Helpers
var request = new Request
{
Method = "POST",
Url = "/",
RequestUriString = "/",
HttpVersion = new Version(1, 1)
};
request.Headers.AddHeader(KnownHeaders.Host, server);
......
......@@ -28,7 +28,7 @@ namespace Titanium.Web.Proxy.IntegrationTests.Helpers
Request.ParseRequestLine(line, out var method, out var url, out var version);
var request = new Request
{
Method = method, Url = url, HttpVersion = version
Method = method, RequestUriString8 = url, HttpVersion = version
};
while (!string.IsNullOrEmpty(line = reader.ReadLine()))
{
......
......@@ -15,9 +15,12 @@ namespace Titanium.Web.Proxy.IntegrationTests
{
var testSuite = new TestSuite();
bool serverCalled = false;
var server = testSuite.GetServer();
server.HandleRequest((context) =>
{
serverCalled = true;
return context.Response.WriteAsync("I am server. I received your greetings.");
});
......@@ -37,11 +40,11 @@ namespace Titanium.Web.Proxy.IntegrationTests
var response = await client.GetAsync(new Uri(server.ListeningHttpUrl));
Assert.IsFalse(serverCalled, "Server should not be called.");
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
var body = await response.Content.ReadAsStringAsync();
Assert.IsTrue(body.Contains("TitaniumWebProxy-Stopped!!"));
}
[TestMethod]
......@@ -76,7 +79,6 @@ namespace Titanium.Web.Proxy.IntegrationTests
var body = await response.Content.ReadAsStringAsync();
Assert.IsTrue(body.Contains("TitaniumWebProxy-Stopped!!"));
}
[TestMethod]
......@@ -111,7 +113,6 @@ namespace Titanium.Web.Proxy.IntegrationTests
var body = await response.Content.ReadAsStringAsync();
Assert.IsTrue(body.Contains("TitaniumWebProxy-Stopped!!"));
}
......@@ -147,7 +148,6 @@ namespace Titanium.Web.Proxy.IntegrationTests
var body = await response.Content.ReadAsStringAsync();
Assert.IsTrue(body.Contains("TitaniumWebProxy-Stopped!!"));
}
[TestMethod]
......@@ -181,7 +181,6 @@ namespace Titanium.Web.Proxy.IntegrationTests
var body = await response.Content.ReadAsStringAsync();
Assert.IsTrue(body.Contains("TitaniumWebProxy-Stopped!!"));
}
}
}
\ No newline at end of file
}
......@@ -16,7 +16,7 @@
<FrameworkReference Include="Microsoft.AspNetCore.App" />
<PackageReference Include="Microsoft.AspNetCore.Server.Kestrel" Version="2.2.0" />
<PackageReference Include="Microsoft.AspNetCore.Server.Kestrel.Https" Version="2.2.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="3.0.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="3.0.1" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.4.0" />
<PackageReference Include="MSTest.TestAdapter" Version="2.0.0" />
<PackageReference Include="MSTest.TestFramework" Version="2.0.0" />
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment