Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -362,3 +362,22 @@ MigrationBackup/
# Fody - auto-generated XML schema
FodyWeavers.xsd
/src/TlsClient.Core/runtimes/win-x64/native/tls-client-latest.dll

# TlsClient.NET — generated artifacts that must never be committed:
# src/native/TlsClient.Native.{os}-{arch}/ — platform package directories
# rendered from src/native/template/
# by build/native-builder/prepare.js
# build/temp/ — staging directory for native DLLs
# downloaded by the prepare-libraries
# pipeline before they are placed into
# the rendered platform packages
# local-nuget/ — output of local `dotnet pack` runs
# used to consume the wrapper from
# another solution as a local feed
# /runtimes/ — ad-hoc DLL staging at the repo root
# (e.g. for NativeLoader.GetLibraryPath()
# default lookup during local debugging)
/src/native/TlsClient.Native.*/
/build/temp/
/local-nuget/
/runtimes/
5 changes: 5 additions & 0 deletions build/native-builder/prepare.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ const fsPromises = fs.promises;
// Define paths based on environment variables
const basePath = process.env.GITHUB_WORKSPACE;
const tlsVersion = process.env.TLS_CLIENT_VERSION?.replace('v', '') || '';
// PACKAGE_VERSION is the NuGet package version stamped into the platform-specific
// project file. Defaults to TLS_CLIENT_VERSION when not set so existing CI runs
// (which only set TLS_CLIENT_VERSION) keep working unchanged.
const packageVersion = process.env.PACKAGE_VERSION || tlsVersion;
const tlsClientRegex = new RegExp(`tls-client-(.*)-(.*)-${tlsVersion}\\.(.*)`);

// Set up paths
Expand Down Expand Up @@ -111,6 +115,7 @@ async function processLibraries() {
os: lib.os,
arch: lib.arch,
version: tlsVersion,
packageVersion: packageVersion,
ext: lib.ext,
runtimeIdentifier: lib.runtimeIdentifier,
};
Expand Down
214 changes: 214 additions & 0 deletions docs/streaming.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,214 @@
# 🔀 Streaming Responses (SSE / chunked)

`TlsClient.Native` can consume long-lived streaming responses — Server-Sent
Events (`text/event-stream`), NDJSON firehoses, or any chunked HTTP/1.1 / HTTP/2
body — without buffering the entire body in memory first.

This is **native-only**. `TlsClient.Api` doesn't expose it because the
upstream `tls-client` HTTP service doesn't forward chunked bodies in a
streaming fashion.

---

## 🧱 The streaming API

A streaming response is consumed through four short, synchronous calls on
`NativeTlsClient`. Each maps 1:1 to a native entry point in the underlying
`tls-client` library:

| `NativeTlsClient` method | Native export | What it does |
| ------------------------------------------------- | --------------- | --------------------------------------------------------------------------- |
| `RequestStream(Request)` | `requestStream` | Issues the request and returns once **headers** are available; body stays open and is owned by the native side. |
| `ReadStream(Guid streamId, int timeoutMs)` | `readStream` | Returns the next chunk (base64-encoded bytes), or one of `EOF` / `Timeout` / `Error`. |
| `ReadStreamAll(Guid streamId)` | `readStreamAll` | Drains the rest of the body in one call. Returns the same `Response` shape as the non-streaming `Request()`. |
| `CancelStream(Guid streamId)` | `cancelStream` | Closes the underlying connection and releases native resources. Idempotent — safe to call after EOF, error, or on an unknown id. |

`async` variants (`RequestStreamAsync`, `ReadStreamAsync`, `ReadStreamAllAsync`,
`CancelStreamAsync`) follow the same contract.

The static `NativeTlsClient.IsStreamingSupported` reports whether the loaded
native library exposes the four entry points. Older `tls-client` builds may
not, in which case the streaming methods throw `EntryPointNotFoundException`
on first use:

```csharp
if (!NativeTlsClient.IsStreamingSupported)
throw new NotSupportedException("Loaded native library does not expose the streaming entry points.");
```

---

## 🚀 Quick Start

### Server-Sent Events

```csharp
using TlsClient.Native;
using TlsClient.Core.Models.Entities;
using TlsClient.Core.Models.Requests;
using System.Text;

NativeTlsClient.Initialize("{PATH_TO_NATIVE_LIBRARY}");

using var client = new NativeTlsClient(new TlsClientOptions(
TlsClientIdentifier.Chrome133,
"Mozilla/5.0 ... Chrome/133")
{
// ⚠️ A negative duration disables the native http.Client.Timeout entirely.
// Required for long-lived SSE streams. TimeSpan.Zero does NOT work — it
// is treated as "use the 30 s default" by the native side.
Timeout = System.Threading.Timeout.InfiniteTimeSpan,
});

var start = client.RequestStream(new Request
{
RequestUrl = "https://example.com/sse",
RequestMethod = HttpMethod.Get,
Headers = new() { ["Accept"] = "text/event-stream" },
});

try
{
var sb = new StringBuilder();
while (true)
{
var chunk = client.ReadStream(start.StreamId, timeoutMs: 1000);

if (!string.IsNullOrEmpty(chunk.Error))
throw new IOException($"stream error: {chunk.Error}");

if (chunk.Timeout)
{
// No data for 1s. Check your CancellationToken here, then continue.
ct.ThrowIfCancellationRequested();
continue;
}

sb.Append(Encoding.UTF8.GetString(chunk.GetChunkBytes()));

// Pull complete SSE events (`\n\n`-separated) out of the buffer:
int sep;
while ((sep = sb.ToString().IndexOf("\n\n", StringComparison.Ordinal)) >= 0)
{
string ev = sb.ToString(0, sep);
sb.Remove(0, sep + 2);
ProcessEvent(ev);
}

if (chunk.EOF) break;
}
}
finally
{
client.CancelStream(start.StreamId); // idempotent
}
```

### "I don't know upfront if it's streaming"

When the same code path may receive either an SSE response *or* a regular JSON
body, branch on `Content-Type` from the headers `RequestStream` returns:

```csharp
var start = client.RequestStream(req);

string ct = start.Headers?.GetValueOrDefault("Content-Type")?.FirstOrDefault() ?? "";
if (ct.Contains("text/event-stream") || ct.Contains("application/x-ndjson"))
{
// Streaming — loop ReadStream as in the SSE example above.
}
else
{
// Not streaming — drain the body in one extra FFI call.
Response full = client.ReadStreamAll(start.StreamId);
Console.WriteLine(full.Body);
}
```

The non-streaming branch costs **one extra FFI call** vs. plain `Request()` —
negligible compared to the network round-trip and avoids the "predict ahead of
time" problem.

---

## 🔁 The `timeoutMs` parameter on `ReadStream`

| Value | Behaviour |
| --------- | ------------------------------------------------------------------------------------------ |
| `< 0` | Block until the next chunk, EOF, or error. |
| `= 0` | Non-blocking poll — returns `Timeout=true` immediately if no chunk is buffered. |
| `> 0` | Block up to `timeoutMs` ms, then return `Timeout=true` if no chunk arrived (heartbeat). |

`Timeout=true` is **not** an error — it just means "no data yet." Use it to
periodically check a `CancellationToken` from your read loop without blocking
the underlying P/Invoke thread indefinitely.

---

## 🧩 Single-thread tunneling

A common pattern with `TlsClient.Native` is to serialize every native call
through a single dedicated thread (to keep the cgo boundary simple). The
streaming API fits that model:

* `RequestStream`, `ReadStream`, `ReadStreamAll`, and `CancelStream` are all
short-lived calls bounded by `timeoutMs`.
* The Go side keeps a per-stream goroutine that pumps body bytes into a
buffered channel — `ReadStream` just pops the next chunk.
* You can safely interleave streaming reads with normal `Request()` calls on
the same dispatcher.

---

## ⚙️ Tuning

| Knob | Where | Effect |
| ---- | ----- | ------ |
| `Request.StreamOutputBlockSize` | per request | Per-Read buffer size on the Go side. Defaults to **4096 bytes**. SSE rarely benefits from making it smaller; large binary streams may benefit from `64 * 1024`. |
| `ReadStream(streamId, timeoutMs)` | per call | How often the read loop wakes up to check cancellation. Typical: **250–1000 ms**. |
| `Request.TimeoutMilliseconds` / `TlsClientOptions.Timeout` | per client/request | ⚠️ **Set to `Timeout.InfiniteTimeSpan` (or any negative duration) for long-lived streams.** Go's `http.Client.Timeout` covers the entire request *including* body reads, so a 30 s timeout will tear down a long-lived SSE stream. `TimeSpan.Zero` / `TimeoutMilliseconds = 0` is treated as "use the 30 s default" — only a negative value disables the deadline. |

---

## 📖 Reference (returned types)

### `StreamStartResponse`
Extends `Response` with one extra field: `Guid StreamId`. `Body` is always empty.

### `StreamChunkResponse`
| Field | Meaning |
| ------------ | ------------------------------------------------------------------------- |
| `StreamId` | Echo of the input id. |
| `Chunk` | Base64-encoded raw bytes (decompressed). Empty on EOF / Timeout / Error. |
| `EOF` | `true` once. Stream is closed; `StreamId` is invalid after this. |
| `Timeout` | `true` when no data arrived within `timeoutMs`. Stream still live. |
| `Error` | Non-empty on a fatal read error. Stream is closed. |
| `IsTerminal` | Convenience: `EOF || !string.IsNullOrEmpty(Error)`. |
| `GetChunkBytes()` | `Convert.FromBase64String(Chunk)` with a null-safe fallback. |

`CancelStream` returns the existing `DestroyResponse` (`{ Id, Body, Success }`).

---

## 🚧 Limitations

* **No callbacks across the FFI boundary** — the model is poll-based, by
design. Reverse P/Invoke into managed code from arbitrary Go goroutine
threads is fragile and would break the single-thread tunneling pattern.
* **Backpressure timeout** — if you stop calling `ReadStream` for >60 s, the
Go-side pump goroutine cancels itself to avoid leaking goroutines. Subsequent
`ReadStream` calls will return `EOF=true`.
* **One reader per stream** — concurrent `ReadStream` calls on the same
`StreamId` are serialized internally but produce undefined chunk ordering.
Don't do it.

---

## 🧯 Troubleshooting

| Symptom | Likely cause |
| -------------------------------------------- | --------------------------------------------------------------------------------------------- |
| `EntryPointNotFoundException: requestStream` | The loaded native library predates the streaming exports. Use a v1.11.2-stream or later DLL. |
| Stream ends after exactly 30 s | The native side enforces its default 30 s deadline. Set `TlsClientOptions.Timeout = Timeout.InfiniteTimeSpan` (or `Request.TimeoutMilliseconds = -1`) to disable it. Note: `TimeSpan.Zero` / `0` is interpreted as "use the default", not "no timeout". |
| Endless `Timeout=true` heartbeats | The remote side hasn't flushed any bytes yet. Verify the endpoint actually streams. |
| `Error: unknown streamId` | You called `ReadStream` after `EOF` / `Error` / `CancelStream`. Stop the loop on those. |
11 changes: 11 additions & 0 deletions readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,17 @@ Console.WriteLine(res.Status);

---

## 🔀 Streaming Responses

`TlsClient.Native` supports consuming streaming responses (Server-Sent Events,
NDJSON, chunked bodies) without buffering the whole body — through a
`RequestStream` / `ReadStream` / `CancelStream` API on `NativeTlsClient`.

See [docs/streaming.md](./docs/streaming.md) for the full guide, including SSE
examples, `Content-Type`-based branching, tuning knobs, and limitations.

---

## 🧯 Support & Issues

* Wrapper/packaging issues → open an issue **here**.
Expand Down
2 changes: 1 addition & 1 deletion src/Providers/TlsClient.Provider.HttpClient/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ TlsClient.Initialize("{LIBRARY_PATH}");

// create a TlsClient instance
var tlsClient = new TlsClientBuilder()
.WithIdentifier(TlsClientIdentifier.Chrome132)
.WithIdentifier(TlsClientIdentifier.Chrome133)
.WithUserAgent("TestClient 1.0")
.Build();

Expand Down
2 changes: 1 addition & 1 deletion src/TlsClient.Api/ApiTlsClient.cs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public ApiTlsClient(ApiTlsClientOptions options) : base(options) {
HttpClient.DefaultRequestHeaders.Add("x-api-key", options.ApiKey);
}

public ApiTlsClient(Uri apiBaseUri, string apiKey) : this(new ApiTlsClientOptions(TlsClientIdentifier.Chrome133, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 OPR/117.0.0.0", apiBaseUri, apiKey)){ }
public ApiTlsClient(Uri apiBaseUri, string apiKey) : this(new ApiTlsClientOptions(TlsClientIdentifier.Chrome133, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 OPR/118.0.0.0", apiBaseUri, apiKey)){ }

#region Sync Methods
public override Response Request(Request request) => AsyncHelpers.RunSync(() => RequestAsync(request, CancellationToken.None));
Expand Down
2 changes: 1 addition & 1 deletion src/TlsClient.Api/Models/Entities/ApiTlsClientOptions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ public class ApiTlsClientOptions : TlsClientOptions
public Uri ApiBaseUri { get; internal set; }
public string ApiKey { get; internal set; }

public ApiTlsClientOptions(Uri apiBaseUri, string apiKey) : base(TlsClientIdentifier.Chrome133, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 OPR/117.0.0.0")
public ApiTlsClientOptions(Uri apiBaseUri, string apiKey) : base(TlsClientIdentifier.Chrome133, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 OPR/118.0.0.0")
{
ApiBaseUri = ValidateBaseUri(apiBaseUri, nameof(apiBaseUri));
ApiKey = ValidateApiKey(apiKey, nameof(apiKey));
Expand Down
2 changes: 1 addition & 1 deletion src/TlsClient.Api/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ using TlsClient.Core.Models.Requests;

var options = new ApiTlsClientOptions(
TlsClientIdentifier.Chrome133,
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 OPR/117.0.0.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 OPR/118.0.0.0",
new Uri("http://127.0.0.1:8080"),
"my-auth-key-1"
);
Expand Down
2 changes: 1 addition & 1 deletion src/TlsClient.Core/BaseTlsClient.cs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ protected BaseTlsClient(TlsClientOptions options)
Options = options ?? throw new ArgumentNullException(nameof(options));
}

protected BaseTlsClient(): this(new TlsClientOptions(TlsClientIdentifier.Chrome132, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 OPR/117.0.0.0")){}
protected BaseTlsClient(): this(new TlsClientOptions(TlsClientIdentifier.Chrome133, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 OPR/118.0.0.0")){}

public abstract Response Request(Request request);
public abstract GetCookiesFromSessionResponse GetCookies(string url);
Expand Down
10 changes: 9 additions & 1 deletion src/TlsClient.Core/Models/Entities/TlsClientIdentifier.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,18 @@ public sealed class TlsClientIdentifier
public static readonly TlsClientIdentifier Chrome124 = new TlsClientIdentifier("chrome_124");
public static readonly TlsClientIdentifier Chrome130Psk = new TlsClientIdentifier("chrome_130_PSK");
public static readonly TlsClientIdentifier Chrome131 = new TlsClientIdentifier("chrome_131");
public static readonly TlsClientIdentifier Chrome132 = new TlsClientIdentifier("chrome_132");
public static readonly TlsClientIdentifier Chrome131Psk = new TlsClientIdentifier("chrome_131_PSK");
public static readonly TlsClientIdentifier Chrome133 = new TlsClientIdentifier("chrome_133");
public static readonly TlsClientIdentifier Chrome133Psk = new TlsClientIdentifier("chrome_133_PSK");
public static readonly TlsClientIdentifier Chrome144 = new TlsClientIdentifier("chrome_144");
public static readonly TlsClientIdentifier Chrome144Psk = new TlsClientIdentifier("chrome_144_PSK");
public static readonly TlsClientIdentifier Chrome146 = new TlsClientIdentifier("chrome_146");
public static readonly TlsClientIdentifier Chrome146Psk = new TlsClientIdentifier("chrome_146_PSK");
#endregion

#region Brave Profiles
public static readonly TlsClientIdentifier Brave146 = new TlsClientIdentifier("brave_146");
public static readonly TlsClientIdentifier Brave146Psk = new TlsClientIdentifier("brave_146_PSK");
#endregion

#region Safari Profiles
Expand All @@ -39,6 +45,7 @@ public sealed class TlsClientIdentifier
public static readonly TlsClientIdentifier SafariIos170 = new TlsClientIdentifier("safari_ios_17_0");
public static readonly TlsClientIdentifier SafariIos180 = new TlsClientIdentifier("safari_ios_18_0");
public static readonly TlsClientIdentifier SafariIos185 = new TlsClientIdentifier("safari_ios_18_5");
public static readonly TlsClientIdentifier SafariIos260 = new TlsClientIdentifier("safari_ios_26_0");
#endregion

#region Firefox Profiles
Expand All @@ -57,6 +64,7 @@ public sealed class TlsClientIdentifier
public static readonly TlsClientIdentifier Firefox146Psk = new TlsClientIdentifier("firefox_146_PSK");
public static readonly TlsClientIdentifier Firefox147 = new TlsClientIdentifier("firefox_147");
public static readonly TlsClientIdentifier Firefox147Psk = new TlsClientIdentifier("firefox_147_PSK");
public static readonly TlsClientIdentifier Firefox148 = new TlsClientIdentifier("firefox_148");
#endregion

#region Opera Profiles
Expand Down
10 changes: 10 additions & 0 deletions src/TlsClient.Core/Models/Entities/TlsClientOptions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,16 @@ public class TlsClientOptions
public TlsClientIdentifier? TlsClientIdentifier { get; set; } = null;
public string? ProxyURL { get; set; }
public bool IsRotatingProxy { get; set; } = false;
/// <summary>
/// Per-request timeout applied to the whole HTTP exchange — including
/// body reads — by the underlying native client. Forwarded to the
/// native side as <see cref="Requests.Request.TimeoutMilliseconds"/>.
/// <list type="bullet">
/// <item><description>Default: 60 s.</description></item>
/// <item><description><see cref="TimeSpan.Zero"/> — interpreted by the native side as "use the 30 s default", NOT "no timeout".</description></item>
/// <item><description><see cref="System.Threading.Timeout.InfiniteTimeSpan"/> (or any negative duration) — disables the deadline. Required for long-lived SSE / NDJSON / streaming responses, otherwise the connection will be torn down at the deadline.</description></item>
/// </list>
/// </summary>
public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(60);
public string? ServerNameOverwrite { get; set; }
public bool FollowRedirects { get; set; } = false;
Expand Down
10 changes: 10 additions & 0 deletions src/TlsClient.Core/Models/Requests/CancelStreamRequest.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
using System;

namespace TlsClient.Core.Models.Requests
{
// Reference: https://github.com/bogdanfinn/tls-client/blob/master/cffi_src/types.go (CancelStreamInput)
public class CancelStreamRequest
{
public Guid StreamId { get; set; }
}
}
Loading