Skip to content

Commit

Permalink
Merge pull request #1766 from riganti/request-compression
Browse files Browse the repository at this point in the history
gzip compression of command and staticCommand requests
  • Loading branch information
tomasherceg committed Feb 23, 2024
2 parents 58f1854 + dc1a62e commit a07449c
Show file tree
Hide file tree
Showing 31 changed files with 1,970 additions and 2,459 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,18 @@ public class DotvvmRuntimeConfiguration
[JsonProperty("reloadMarkupFiles")]
public DotvvmGlobal3StateFeatureFlag ReloadMarkupFiles { get; } = new("Dotvvm3StateFeatureFlag.ReloadMarkupFiles");

/// <summary>
/// When enabled, command and staticCommand requests are compressed client-side and DotVVM accepts the compressed requests.
/// It is enabled by default in Production mode.
/// See <see cref="MaxPostbackSizeBytes" /> to limit the impact of potential decompression bomb. Although compression may be enabled only for specific routes, DotVVM does not check authentication before decompressing the request.
/// </summary>
[JsonProperty("compressPostbacks")]
public Dotvvm3StateFeatureFlag CompressPostbacks { get; } = new("DotvvmFeatureFlag.CompressPostbacks");

/// <summary> Maximum size of command/staticCommand request body after decompression (does not affect file upload). Default = 128MB, lower limit is a basic protection against decompression bomb attack. Set to -1 to disable the limit. </summary>
[JsonProperty("maxPostbackSizeBytes")]
public long MaxPostbackSizeBytes { get; set; } = 1024 * 1024 * 128; // 128 MB

/// <summary>
/// Initializes a new instance of the <see cref="DotvvmRuntimeConfiguration"/> class.
/// </summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,12 @@ protected override void RenderControl(IHtmlWriter writer, IDotvvmRequestContext

// init on load
var initCode = $"window.dotvvm.init({JsonConvert.ToString(CultureInfo.CurrentCulture.Name, '"', StringEscapeHandling.EscapeHtml)});";
if (context.Configuration.ExperimentalFeatures.KnockoutDeferUpdates.IsEnabledForRoute(context.Route?.RouteName))
var config = context.Configuration;
if (!config.Runtime.CompressPostbacks.IsEnabledForRoute(context.Route?.RouteName, defaultValue: !config.Debug))
{
initCode = $"dotvvm.options.compressPOST=false;\n{initCode}";
}
if (config.ExperimentalFeatures.KnockoutDeferUpdates.IsEnabledForRoute(context.Route?.RouteName))
{
initCode = $"ko.options.deferUpdates = true;\n{initCode}";
}
Expand Down
23 changes: 21 additions & 2 deletions src/Framework/Framework/Hosting/DotvvmPresenter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ public async Task ProcessRequestCore(IDotvvmRequestContext context)
{
// perform the postback
string postData;
using (var sr = new StreamReader(context.HttpContext.Request.Body))
using (var sr = new StreamReader(ReadRequestBody(context.HttpContext.Request, context.Route?.RouteName)))
{
postData = await sr.ReadToEndAsync();
}
Expand Down Expand Up @@ -344,7 +344,7 @@ public async Task ProcessStaticCommandRequest(IDotvvmRequestContext context)
try
{
JObject postData;
using (var jsonReader = new JsonTextReader(new StreamReader(context.HttpContext.Request.Body)))
using (var jsonReader = new JsonTextReader(new StreamReader(ReadRequestBody(context.HttpContext.Request, routeName: null))))
{
postData = await JObject.LoadAsync(jsonReader);
}
Expand Down Expand Up @@ -548,6 +548,25 @@ async Task ValidateSecFetchHeaders(IDotvvmRequestContext context)
}
}

Stream ReadRequestBody(IHttpRequest request, string? routeName)
{
request.Headers.TryGetValue("Content-Encoding", out var encodingValue);
var encoding = encodingValue?.FirstOrDefault();
var limitLengthHelp = "To increase the maximum request size, use the DotvvmConfiguration.Runtime.MaxPostbackSizeBytes option.";
if (encoding is null)
return LimitLengthStream.LimitLength(request.Body, configuration.Runtime.MaxPostbackSizeBytes, limitLengthHelp);
if (encoding is "gzip")
{
var enabled = routeName is null ? this.configuration.Runtime.CompressPostbacks.IsEnabledForAnyRoute(defaultValue: true) : this.configuration.Runtime.CompressPostbacks.IsEnabledForRoute(routeName, defaultValue: true);
if (!enabled)
throw new Exception($"Content-Encoding: gzip must be enabled in DotvvmConfiguration.Runtime.CompressPostbacks.");
var gzipStream = new System.IO.Compression.GZipStream(request.Body, System.IO.Compression.CompressionMode.Decompress);
return LimitLengthStream.LimitLength(gzipStream, configuration.Runtime.MaxPostbackSizeBytes, limitLengthHelp);
}
else
throw new Exception($"Unsupported Content-Encoding {encoding}");
}

[Obsolete("Use context.RequestType == DotvvmRequestType.StaticCommand")]
public static bool DetermineIsStaticCommand(IDotvvmRequestContext context) =>
context.RequestType == DotvvmRequestType.StaticCommand;
Expand Down
4 changes: 4 additions & 0 deletions src/Framework/Framework/Resources/Scripts/dotvvm-base.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ import { replaceTypeInfo } from './metadata/typeMap'

import { StateManager } from './state-manager'

export const options = {
compressPOST: true
}

type DotvvmCoreState = {
_culture: string
_viewModelCache?: any
Expand Down
3 changes: 2 additions & 1 deletion src/Framework/Framework/Resources/Scripts/dotvvm-root.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { initCore, getViewModel, getViewModelObservable, initBindings, getCulture, getState, getStateManager } from "./dotvvm-base"
import { initCore, getViewModel, getViewModelObservable, initBindings, getCulture, getState, getStateManager, options } from "./dotvvm-base"
import * as events from './events'
import * as spa from "./spa/spa"
import * as validation from './validation/validation'
Expand Down Expand Up @@ -121,6 +121,7 @@ const dotvvmExports = {
logPostBackScriptError,
level
},
options,
translations: translations as any,
StateManager,
DotvvmEvent,
Expand Down
15 changes: 14 additions & 1 deletion src/Framework/Framework/Resources/Scripts/postback/http.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { getVirtualDirectory, getViewModel, getState, getStateManager } from '../dotvvm-base';
import { getVirtualDirectory, getViewModel, getState, getStateManager, options } from '../dotvvm-base';
import { DotvvmPostbackError } from '../shared-classes';
import { logInfoVerbose, logWarning } from '../utils/logging';
import { keys } from '../utils/objects';
Expand All @@ -25,6 +25,9 @@ export async function postJSON<T>(url: string, postData: any, signal: AbortSigna
headers.append('Content-Type', 'application/json');
headers.append('X-DotVVM-PostBack', 'true');
appendAdditionalHeaders(headers, additionalHeaders);
if (postData.length > 1000 && options.compressPOST) {
postData = await compressString(postData, headers)
}

return await fetchJson<T>(url, { body: postData, headers: headers, method: "POST", signal });
}
Expand Down Expand Up @@ -103,3 +106,13 @@ function appendAdditionalHeaders(headers: Headers, additionalHeaders?: { [key: s
}
}
}

function compressString(data: string, headers: Headers) {
if (!window.CompressionStream) {
return data
}
headers.append('Content-Encoding', 'gzip')
const blob = new Blob([data], { type: 'text/plain' })
const stream = blob.stream().pipeThrough(new CompressionStream('gzip'))
return new Response(stream).blob()
}
16 changes: 4 additions & 12 deletions src/Framework/Framework/Resources/Scripts/tests/postback.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -228,8 +228,6 @@ test("Postback: don't update unrelated properties", async () => {
// })

test("Run postbacks [Queue | no failures]", async () => {
jest.setTimeout(120_000);

// watchEvents()

await fc.assert(fc.asyncProperty(
Expand Down Expand Up @@ -266,11 +264,9 @@ test("Run postbacks [Queue | no failures]", async () => {
expect(state().Property1).toBe(parallelism)
}
), { timeout: 2000 })
})
}, 120_000)

test("Run postbacks [Queue + Deny | no failures]", async () => {
jest.setTimeout(120_000);

// watchEvents()

await fc.assert(fc.asyncProperty(
Expand Down Expand Up @@ -330,11 +326,9 @@ test("Run postbacks [Queue + Deny | no failures]", async () => {
await initDenyPostback
}
), { timeout: 2000 })
})
}, 120_000)

test("Run postbacks [Queue + Default | no failures]", async () => {
jest.setTimeout(120_000);

// watchEvents()

await fc.assert(fc.asyncProperty(
Expand Down Expand Up @@ -401,7 +395,7 @@ test("Run postbacks [Queue + Default | no failures]", async () => {
expect(state().Property2).toBe(index2)
}
), { timeout: 2000 })
})
}, 120_000)

test("Postback: AbortSignal", async () => {

Expand All @@ -422,8 +416,6 @@ test("Postback: AbortSignal", async () => {
})

test("Run postbacks [Queue + Default | no failures]", async () => {
jest.setTimeout(120_000);

// watchEvents()

await fc.assert(fc.asyncProperty(
Expand Down Expand Up @@ -511,4 +503,4 @@ test("Run postbacks [Queue + Default | no failures]", async () => {
}
}
), { timeout: 2000 })
})
}, 120_000)
2 changes: 1 addition & 1 deletion src/Framework/Framework/Resources/Scripts/tests/setup.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ global.compileConstants = { isSpa: true, debug: true }
global.ko = require("../knockout-latest.debug")
global.dotvvm_Globalize = require("../Globalize/globalize")

const expect = require("expect")
const { expect } = require("expect")

expect.extend({
observable(obj) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ const s = getStateManager() as StateManager<any>
s.doUpdateNow()

test("Stress test - simple increments", async () => {
jest.setTimeout(120_000);

// watchEvents()

await fc.assert(fc.asyncProperty(
Expand Down Expand Up @@ -44,7 +42,7 @@ test("Stress test - simple increments", async () => {
})
}
), { timeout: 20000 })
})
}, 120_000)

test("Stress test - simple increments with postbacks in background", async () => {
jest.setTimeout(120_000);
Expand Down Expand Up @@ -88,4 +86,4 @@ test("Stress test - simple increments with postbacks in background", async () =>
})
}
), { timeout: 20000 })
})
}, 120_000)
96 changes: 96 additions & 0 deletions src/Framework/Framework/Utils/LimitLengthStream.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;

namespace DotVVM.Framework.Utils
{
internal class LimitLengthStream : Stream
{
private readonly Stream innerStream;
private readonly long maxLength;
private readonly string comment;
private long position;

public LimitLengthStream(Stream innerStream, long maxLength, string errorComment)
{
this.innerStream = innerStream;
this.maxLength = maxLength;
this.comment = errorComment;
}

private void MovePosition(long offset)
{
position += offset;
if (position > maxLength)
throw new InvalidOperationException($"The stream is limited to {maxLength} bytes: {comment}");
}

public long RemainingAllowedLength => maxLength - position;


public override bool CanRead => innerStream.CanRead;

public override bool CanSeek => false;

public override bool CanWrite => false;

public override long Length => innerStream.Length;

public override long Position
{
get => innerStream.Position;
set => throw new NotImplementedException();
}

public override void Flush() => innerStream.Flush();
public override int Read(byte[] buffer, int offset, int count)
{
var read = innerStream.Read(buffer, offset, (int)Math.Min(count, RemainingAllowedLength + 1));
MovePosition(read);
return read;
}

public override async Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
var read = await innerStream.ReadAsync(buffer, offset, (int)Math.Min(count, RemainingAllowedLength + 1), cancellationToken);
MovePosition(read);
return read;
}
#if DotNetCore
public override int Read(Span<byte> buffer)
{
var read = innerStream.Read(buffer.Slice(0, (int)Math.Min(buffer.Length, RemainingAllowedLength + 1)));
MovePosition(read);
return read;
}

public override async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default)
{
var read = await innerStream.ReadAsync(buffer.Slice(0, (int)Math.Min(buffer.Length, RemainingAllowedLength + 1)), cancellationToken);
MovePosition(read);
return read;
}
#endif
public override long Seek(long offset, SeekOrigin origin) => throw new System.NotImplementedException();
public override void SetLength(long value) => throw new System.NotImplementedException();
public override void Write(byte[] buffer, int offset, int count) => throw new System.NotImplementedException();

public static Stream LimitLength(Stream s, long maxLength, string errorComment)
{
if (maxLength < 0 || maxLength == long.MaxValue)
return s;

if (s.CanSeek)
{
if (s.Length > maxLength)
throw new InvalidOperationException($"The stream is limited to {maxLength} bytes: {errorComment}");
return s;
}
else
{
return new LimitLengthStream(s, maxLength, errorComment);
}
}
}
}
4 changes: 2 additions & 2 deletions src/Framework/Framework/build.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ async function build({ debug, spa, output, input = "dotvvm-root.ts" }) {
entryPoints: [`./Resources/Scripts/${input}`],
outfile: `./obj/javascript/${output}/dotvvm-root.js`,
define: {
"compileConstants.isSpa": spa,
"compileConstants.debug": debug,
"compileConstants.isSpa": String(spa),
"compileConstants.debug": String(debug),
},
target: [
'es2020'
Expand Down
6 changes: 3 additions & 3 deletions src/Framework/Framework/jest.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ module.exports = {
preset: 'ts-jest',
testEnvironment: 'jsdom',
setupFiles: [ "./Resources/Scripts/tests/setup.js", "./Resources/Scripts/knockout-latest.debug.js" ],
globals: {
'ts-jest': {
transform: {
"^.+\\.ts$": [ "ts-jest", {
'tsconfig': 'tsconfig.jest.json'
}
} ]
}
};
14 changes: 8 additions & 6 deletions src/Framework/Framework/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,20 @@
"license": "Apache-2.0",
"packageManager": "yarn@3.1.1",
"devDependencies": {
"@types/jest": "26.0.14",
"@types/jest": "29.0.0",
"@types/knockout": "^3.4.72",
"esbuild": "^0.14.39",
"@types/node": "20.11.5",
"esbuild": "^0.19.11",
"fast-check": "2.5.0",
"jest": "26.5.3",
"jest": "29.7.0",
"jest-environment-jsdom": "29.7.0",
"jest-github-actions-reporter": "^1.0.3",
"jest-junit": "^14.0.1",
"jest-junit": "^16.0.0",
"promise": "8.1.0",
"symbol-es6": "^0.1.2",
"systemjs": "6.7.1",
"ts-jest": "26.4.1",
"typescript": "4.7.4"
"ts-jest": "29.1.1",
"typescript": "5.3.3"
},
"scripts": {
"build": "node ./build.js",
Expand Down
4 changes: 2 additions & 2 deletions src/Framework/Framework/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@
"removeComments": false,
"sourceMap": true,
"inlineSources": true,
"target": "ES2018",
"target": "ES2020",
"declaration": true,
"moduleResolution": "node",
"strictNullChecks": true,
"lib": [ "dom", "es2015.promise", "es5", "es6" ],
"lib": [ "dom", "ES2020" ],
"module": "ES2015"
},
"exclude": ["obj/**", "Resources/Scripts/tests", "node_modules/**"],
Expand Down

0 comments on commit a07449c

Please sign in to comment.