Consolidate Rust builds

This consolidates Rust build process for various codecs into a single top-level image that is built once and reused.

This ensures that we use same version of tools across codecs (now controlled from a single place), simplifies build configs and commands, speeds up common builds and reduces disk space taken by Docker images by reusing same one.

Additionally, this PR renames all codecs to squoosh-* to work around the https://github.com/rustwasm/wasm-pack/issues/829 (which has been already fixed on master of wasm-pack but not released in a while), as well as adds `publish = false` to Cargo.toml to avoid accidental publishing for now.

I'm planning to do similar for Emscripten in a separate PR, although abilities to share configs there are much more limited due to lack of package manager in C++.
This commit is contained in:
Ingvar Stepanyan
2020-07-13 20:43:44 +01:00
parent 17ffa57a8b
commit c5c520a71d
36 changed files with 187 additions and 302 deletions

4
codecs/build-rust.sh Normal file
View File

@ -0,0 +1,4 @@
set -e
docker build -t squoosh-rust - < ../rust.Dockerfile
docker run --rm -v $PWD:/src squoosh-rust "$@"

View File

@ -1,7 +1,8 @@
[package]
name = "squooshhqx"
name = "squoosh-hqx"
version = "0.1.0"
authors = ["Surma <surma@surma.link>"]
publish = false
[lib]
crate-type = ["cdylib"]

View File

@ -1,25 +0,0 @@
#!/bin/bash
set -e
echo "============================================="
echo "Compiling wasm"
echo "============================================="
(
wasm-pack build
wasm-strip pkg/squooshhqx_bg.wasm
echo "Optimising Wasm so it doesn't break Chrome (this takes like 10-15mins. get a cup of tea)"
echo "Once https://crbug.com/974804 is fixed, we can remove this step"
wasm-opt -Os --no-validation -o pkg/squooshhqx_bg.wasm pkg/squooshhqx_bg.wasm
rm pkg/.gitignore
)
echo "============================================="
echo "Compiling wasm done"
echo "============================================="
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
echo "Did you update your docker image?"
echo "Run \`docker pull ubuntu\`"
echo "Run \`docker pull rust\`"
echo "Run \`docker build -t squoosh-hqx .\`"
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"

View File

@ -1,7 +1,6 @@
{
"name": "hqx",
"scripts": {
"build:image": "docker build -t squoosh-hqx .",
"build": "docker run --rm -v $(pwd):/src squoosh-hqx ./build.sh"
"build": "../build-rust.sh"
}
}

View File

@ -1,15 +1,15 @@
{
"name": "squooshhqx",
"name": "squoosh-hqx",
"collaborators": [
"Surma <surma@surma.link>"
],
"version": "0.1.0",
"files": [
"squooshhqx_bg.wasm",
"squooshhqx.js",
"squooshhqx.d.ts"
"squoosh_hqx_bg.wasm",
"squoosh_hqx.js",
"squoosh_hqx.d.ts"
],
"module": "squooshhqx.js",
"types": "squooshhqx.d.ts",
"sideEffects": "false"
"module": "squoosh_hqx.js",
"types": "squoosh_hqx.d.ts",
"sideEffects": false
}

10
codecs/hqx/pkg/squoosh_hqx.d.ts vendored Normal file
View File

@ -0,0 +1,10 @@
/* tslint:disable */
/* eslint-disable */
/**
* @param {Uint32Array} input_image
* @param {number} input_width
* @param {number} input_height
* @param {number} factor
* @returns {Uint32Array}
*/
export function resize(input_image: Uint32Array, input_width: number, input_height: number, factor: number): Uint32Array;

View File

@ -0,0 +1,2 @@
import * as wasm from "./squoosh_hqx_bg.wasm";
export * from "./squoosh_hqx_bg.js";

View File

@ -1,5 +1,6 @@
/* tslint:disable */
/* eslint-disable */
export const memory: WebAssembly.Memory;
export function resize(a: number, b: number, c: number, d: number, e: number, f: number): void;
export function __wbindgen_malloc(a: number): number;
export function __wbindgen_free(a: number, b: number): void;
export function resize(a: number, b: number, c: number, d: number, e: number, f: number): void;

View File

@ -0,0 +1,48 @@
import * as wasm from './squoosh_hqx_bg.wasm';
let cachegetUint32Memory0 = null;
function getUint32Memory0() {
if (cachegetUint32Memory0 === null || cachegetUint32Memory0.buffer !== wasm.memory.buffer) {
cachegetUint32Memory0 = new Uint32Array(wasm.memory.buffer);
}
return cachegetUint32Memory0;
}
let WASM_VECTOR_LEN = 0;
function passArray32ToWasm0(arg, malloc) {
const ptr = malloc(arg.length * 4);
getUint32Memory0().set(arg, ptr / 4);
WASM_VECTOR_LEN = arg.length;
return ptr;
}
let cachegetInt32Memory0 = null;
function getInt32Memory0() {
if (cachegetInt32Memory0 === null || cachegetInt32Memory0.buffer !== wasm.memory.buffer) {
cachegetInt32Memory0 = new Int32Array(wasm.memory.buffer);
}
return cachegetInt32Memory0;
}
function getArrayU32FromWasm0(ptr, len) {
return getUint32Memory0().subarray(ptr / 4, ptr / 4 + len);
}
/**
* @param {Uint32Array} input_image
* @param {number} input_width
* @param {number} input_height
* @param {number} factor
* @returns {Uint32Array}
*/
export function resize(input_image, input_width, input_height, factor) {
var ptr0 = passArray32ToWasm0(input_image, wasm.__wbindgen_malloc);
var len0 = WASM_VECTOR_LEN;
wasm.resize(8, ptr0, len0, input_width, input_height, factor);
var r0 = getInt32Memory0()[8 / 4 + 0];
var r1 = getInt32Memory0()[8 / 4 + 1];
var v1 = getArrayU32FromWasm0(r0, r1).slice();
wasm.__wbindgen_free(r0, r1 * 4);
return v1;
}

Binary file not shown.

View File

@ -1,9 +0,0 @@
/* tslint:disable */
/**
* @param {Uint32Array} input_image
* @param {number} input_width
* @param {number} input_height
* @param {number} factor
* @returns {Uint32Array}
*/
export function resize(input_image: Uint32Array, input_width: number, input_height: number, factor: number): Uint32Array;

View File

@ -1,46 +0,0 @@
import * as wasm from './squooshhqx_bg.wasm';
let cachegetUint32Memory = null;
function getUint32Memory() {
if (cachegetUint32Memory === null || cachegetUint32Memory.buffer !== wasm.memory.buffer) {
cachegetUint32Memory = new Uint32Array(wasm.memory.buffer);
}
return cachegetUint32Memory;
}
let WASM_VECTOR_LEN = 0;
function passArray32ToWasm(arg) {
const ptr = wasm.__wbindgen_malloc(arg.length * 4);
getUint32Memory().set(arg, ptr / 4);
WASM_VECTOR_LEN = arg.length;
return ptr;
}
let cachegetInt32Memory = null;
function getInt32Memory() {
if (cachegetInt32Memory === null || cachegetInt32Memory.buffer !== wasm.memory.buffer) {
cachegetInt32Memory = new Int32Array(wasm.memory.buffer);
}
return cachegetInt32Memory;
}
function getArrayU32FromWasm(ptr, len) {
return getUint32Memory().subarray(ptr / 4, ptr / 4 + len);
}
/**
* @param {Uint32Array} input_image
* @param {number} input_width
* @param {number} input_height
* @param {number} factor
* @returns {Uint32Array}
*/
export function resize(input_image, input_width, input_height, factor) {
const retptr = 8;
const ret = wasm.resize(retptr, passArray32ToWasm(input_image), WASM_VECTOR_LEN, input_width, input_height, factor);
const memi32 = getInt32Memory();
const v0 = getArrayU32FromWasm(memi32[retptr / 4 + 0], memi32[retptr / 4 + 1]).slice();
wasm.__wbindgen_free(memi32[retptr / 4 + 0], memi32[retptr / 4 + 1] * 4);
return v0;
}

Binary file not shown.

View File

@ -1,11 +1,5 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "adler"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
[[package]]
name = "adler32"
version = "1.1.0"
@ -221,23 +215,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9f8082297d534141b30c8d39e9b1773713ab50fdbe4ff30f750d063b3bfd701"
[[package]]
name = "libdeflate-sys"
version = "0.5.0"
name = "libdeflater"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21e39efa87b84db3e13ff4e2dfac1e57220abcbd7fe8ec44d238f7f4f787cc1f"
checksum = "66dca08b13369865b2f6dca1dd05f833985cbe6c12a676b04d55f78b85e80246"
dependencies = [
"cc",
]
[[package]]
name = "libdeflater"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4810980d791f26d470e2d7d91a3d4d22aa3a4b709fb7e9c5e43ee54f83a01f2"
dependencies = [
"libdeflate-sys",
]
[[package]]
name = "log"
version = "0.4.8"
@ -271,15 +256,6 @@ dependencies = [
"adler32",
]
[[package]]
name = "miniz_oxide"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be0f75932c1f6cfae3c04000e40114adf955636e19040f9c0a2c380702aa1c7f"
dependencies = [
"adler",
]
[[package]]
name = "num-integer"
version = "0.1.43"
@ -334,7 +310,8 @@ dependencies = [
[[package]]
name = "oxipng"
version = "3.0.0"
source = "git+https://github.com/shssoichiro/oxipng.git#3b337ac9086d64ef8df7ad0c6b7ff9e4c178b3ef"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5fd695858078338d73862ff3755f820eff0bf4f3304e4b52f22aba53463183a"
dependencies = [
"bit-vec",
"byteorder",
@ -345,9 +322,8 @@ dependencies = [
"itertools",
"libdeflater",
"log",
"miniz_oxide 0.4.0",
"miniz_oxide",
"rgb",
"rustc_version",
"zopfli",
]
@ -360,7 +336,7 @@ dependencies = [
"bitflags",
"crc32fast",
"deflate",
"miniz_oxide 0.3.7",
"miniz_oxide",
]
[[package]]
@ -415,36 +391,12 @@ dependencies = [
"bytemuck",
]
[[package]]
name = "rustc_version"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
dependencies = [
"semver",
]
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "semver"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
dependencies = [
"semver-parser",
]
[[package]]
name = "semver-parser"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
[[package]]
name = "squoosh-oxipng"
version = "0.1.0"

View File

@ -16,6 +16,3 @@ log = { version = "0.4", features = ["release_max_level_off"] }
[profile.release]
lto = true
opt-level = "s"
[patch.crates-io]
oxipng = { git = "https://github.com/shssoichiro/oxipng.git", branch = "master" }

View File

@ -1,22 +0,0 @@
#!/bin/bash
set -e
echo "============================================="
echo "Compiling wasm"
echo "============================================="
(
wasm-pack build
wasm-strip pkg/squoosh_oxipng_bg.wasm
rm pkg/.gitignore
)
echo "============================================="
echo "Compiling wasm done"
echo "============================================="
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
echo "Did you update your docker image?"
echo "Run \`docker pull ubuntu\`"
echo "Run \`docker pull rust\`"
echo "Run \`docker build -t squoosh-oxipng .\`"
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"

View File

@ -1,7 +1,6 @@
{
"name": "oxipng",
"scripts": {
"build:image": "docker build -t squoosh-oxipng .",
"build": "docker run --rm -v $(pwd):/src squoosh-oxipng ./build.sh"
"build": "../build-rust.sh"
}
}

View File

@ -1,7 +1,8 @@
[package]
name = "resize"
name = "squoosh-resize"
version = "0.1.0"
authors = ["Surma <surma@surma.link>"]
publish = false
[lib]
#crate-type = ["cdylib", "rlib"]

View File

@ -1,9 +0,0 @@
FROM rust
RUN rustup target add wasm32-unknown-unknown
RUN curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
RUN mkdir /opt/wabt && \
curl -L https://github.com/WebAssembly/wabt/releases/download/1.0.11/wabt-1.0.11-linux.tar.gz | tar -xzf - -C /opt/wabt --strip 1
ENV PATH="/opt/wabt:${PATH}"
WORKDIR /src

View File

@ -1,22 +0,0 @@
#!/bin/bash
set -e
echo "============================================="
echo "Compiling wasm"
echo "============================================="
(
wasm-pack build
wasm-strip pkg/resize_bg.wasm
rm pkg/.gitignore
)
echo "============================================="
echo "Compiling wasm done"
echo "============================================="
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
echo "Did you update your docker image?"
echo "Run \`docker pull ubuntu\`"
echo "Run \`docker pull rust\`"
echo "Run \`docker build -t squoosh-resize .\`"
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"

View File

@ -1,7 +1,6 @@
{
"name": "resize",
"scripts": {
"build:image": "docker build -t squoosh-resize .",
"build": "docker run --rm -v $(pwd):/src squoosh-resize ./build.sh"
"build": "../build-rust.sh"
}
}

View File

@ -1,15 +1,15 @@
{
"name": "resize",
"name": "squoosh-resize",
"collaborators": [
"Surma <surma@surma.link>"
],
"version": "0.1.0",
"files": [
"resize_bg.wasm",
"resize.js",
"resize.d.ts"
"squoosh_resize_bg.wasm",
"squoosh_resize.js",
"squoosh_resize.d.ts"
],
"module": "resize.js",
"types": "resize.d.ts",
"sideEffects": "false"
"module": "squoosh_resize.js",
"types": "squoosh_resize.d.ts",
"sideEffects": false
}

View File

@ -1,13 +0,0 @@
/* tslint:disable */
/**
* @param {Uint8Array} input_image
* @param {number} input_width
* @param {number} input_height
* @param {number} output_width
* @param {number} output_height
* @param {number} typ_idx
* @param {boolean} premultiply
* @param {boolean} color_space_conversion
* @returns {Uint8Array}
*/
export function resize(input_image: Uint8Array, input_width: number, input_height: number, output_width: number, output_height: number, typ_idx: number, premultiply: boolean, color_space_conversion: boolean): Uint8Array;

View File

@ -1,50 +0,0 @@
import * as wasm from './resize_bg.wasm';
let cachegetUint8Memory = null;
function getUint8Memory() {
if (cachegetUint8Memory === null || cachegetUint8Memory.buffer !== wasm.memory.buffer) {
cachegetUint8Memory = new Uint8Array(wasm.memory.buffer);
}
return cachegetUint8Memory;
}
let WASM_VECTOR_LEN = 0;
function passArray8ToWasm(arg) {
const ptr = wasm.__wbindgen_malloc(arg.length * 1);
getUint8Memory().set(arg, ptr / 1);
WASM_VECTOR_LEN = arg.length;
return ptr;
}
let cachegetInt32Memory = null;
function getInt32Memory() {
if (cachegetInt32Memory === null || cachegetInt32Memory.buffer !== wasm.memory.buffer) {
cachegetInt32Memory = new Int32Array(wasm.memory.buffer);
}
return cachegetInt32Memory;
}
function getArrayU8FromWasm(ptr, len) {
return getUint8Memory().subarray(ptr / 1, ptr / 1 + len);
}
/**
* @param {Uint8Array} input_image
* @param {number} input_width
* @param {number} input_height
* @param {number} output_width
* @param {number} output_height
* @param {number} typ_idx
* @param {boolean} premultiply
* @param {boolean} color_space_conversion
* @returns {Uint8Array}
*/
export function resize(input_image, input_width, input_height, output_width, output_height, typ_idx, premultiply, color_space_conversion) {
const retptr = 8;
const ret = wasm.resize(retptr, passArray8ToWasm(input_image), WASM_VECTOR_LEN, input_width, input_height, output_width, output_height, typ_idx, premultiply, color_space_conversion);
const memi32 = getInt32Memory();
const v0 = getArrayU8FromWasm(memi32[retptr / 4 + 0], memi32[retptr / 4 + 1]).slice();
wasm.__wbindgen_free(memi32[retptr / 4 + 0], memi32[retptr / 4 + 1] * 1);
return v0;
}

Binary file not shown.

14
codecs/resize/pkg/squoosh_resize.d.ts vendored Normal file
View File

@ -0,0 +1,14 @@
/* tslint:disable */
/* eslint-disable */
/**
* @param {Uint8Array} input_image
* @param {number} input_width
* @param {number} input_height
* @param {number} output_width
* @param {number} output_height
* @param {number} typ_idx
* @param {boolean} premultiply
* @param {boolean} color_space_conversion
* @returns {Uint8Array}
*/
export function resize(input_image: Uint8Array, input_width: number, input_height: number, output_width: number, output_height: number, typ_idx: number, premultiply: boolean, color_space_conversion: boolean): Uint8Array;

View File

@ -0,0 +1,2 @@
import * as wasm from "./squoosh_resize_bg.wasm";
export * from "./squoosh_resize_bg.js";

View File

@ -1,5 +1,6 @@
/* tslint:disable */
/* eslint-disable */
export const memory: WebAssembly.Memory;
export function resize(a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number): void;
export function __wbindgen_malloc(a: number): number;
export function __wbindgen_free(a: number, b: number): void;
export function resize(a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number): void;

View File

@ -0,0 +1,52 @@
import * as wasm from './squoosh_resize_bg.wasm';
let cachegetUint8Memory0 = null;
function getUint8Memory0() {
if (cachegetUint8Memory0 === null || cachegetUint8Memory0.buffer !== wasm.memory.buffer) {
cachegetUint8Memory0 = new Uint8Array(wasm.memory.buffer);
}
return cachegetUint8Memory0;
}
let WASM_VECTOR_LEN = 0;
function passArray8ToWasm0(arg, malloc) {
const ptr = malloc(arg.length * 1);
getUint8Memory0().set(arg, ptr / 1);
WASM_VECTOR_LEN = arg.length;
return ptr;
}
let cachegetInt32Memory0 = null;
function getInt32Memory0() {
if (cachegetInt32Memory0 === null || cachegetInt32Memory0.buffer !== wasm.memory.buffer) {
cachegetInt32Memory0 = new Int32Array(wasm.memory.buffer);
}
return cachegetInt32Memory0;
}
function getArrayU8FromWasm0(ptr, len) {
return getUint8Memory0().subarray(ptr / 1, ptr / 1 + len);
}
/**
* @param {Uint8Array} input_image
* @param {number} input_width
* @param {number} input_height
* @param {number} output_width
* @param {number} output_height
* @param {number} typ_idx
* @param {boolean} premultiply
* @param {boolean} color_space_conversion
* @returns {Uint8Array}
*/
export function resize(input_image, input_width, input_height, output_width, output_height, typ_idx, premultiply, color_space_conversion) {
var ptr0 = passArray8ToWasm0(input_image, wasm.__wbindgen_malloc);
var len0 = WASM_VECTOR_LEN;
wasm.resize(8, ptr0, len0, input_width, input_height, output_width, output_height, typ_idx, premultiply, color_space_conversion);
var r0 = getInt32Memory0()[8 / 4 + 0];
var r1 = getInt32Memory0()[8 / 4 + 1];
var v1 = getArrayU8FromWasm0(r0, r1).slice();
wasm.__wbindgen_free(r0, r1 * 1);
return v1;
}

Binary file not shown.

View File

@ -1,8 +1,9 @@
[package]
name = "rotate"
name = "squoosh-rotate"
version = "0.1.0"
authors = ["Surma <surma@google.com>"]
edition = "2018"
publish = false
[lib]
name = "rotate"

View File

@ -1,24 +1,8 @@
#!/bin/bash
#!/bin/sh
set -e
echo "============================================="
echo "Compiling wasm"
echo "============================================="
(
cargo build \
--target wasm32-unknown-unknown \
--release
cp target/wasm32-unknown-unknown/release/rotate.wasm .
wasm-strip rotate.wasm
)
echo "============================================="
echo "Compiling wasm done"
echo "============================================="
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
echo "Did you update your docker image?"
echo "Run \`docker pull ubuntu\`"
echo "Run \`docker pull rust\`"
echo "Run \`docker build -t squoosh-rotate .\`"
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
cargo build \
--target wasm32-unknown-unknown \
--release
wasm-opt -Os --strip target/wasm32-unknown-unknown/release/rotate.wasm -o rotate.wasm

View File

@ -1,8 +1,7 @@
{
"name": "rotate",
"scripts": {
"build:image": "docker build -t squoosh-rotate .",
"build": "docker run --rm -v $(pwd):/src squoosh-rotate ./build.sh",
"build": "../build-rust.sh ./build.sh",
"benchmark": "echo File size after gzip && npm run benchmark:filesize && echo Optimizing && npm run -s benchmark:optimizing",
"benchmark:baseline": "v8 --liftoff --no-wasm-tier-up --no-opt ./benchmark.js",
"benchmark:optimizing": "v8 --no-liftoff --no-wasm-tier-up ./benchmark.js",

Binary file not shown.

15
codecs/rust.Dockerfile Normal file
View File

@ -0,0 +1,15 @@
FROM emscripten/emsdk:1.39.19 AS wasm-tools
WORKDIR /opt/wasm-tools
RUN wget -qO- https://github.com/rustwasm/wasm-pack/releases/download/v0.9.1/wasm-pack-v0.9.1-x86_64-unknown-linux-musl.tar.gz | tar -xzf - --strip 1
FROM rust:1.44-stretch AS rust
RUN rustup target add wasm32-unknown-unknown
COPY --from=wasm-tools /emsdk/upstream/bin/wasm-opt /usr/local/bin/
COPY --from=wasm-tools /emsdk/upstream/lib/libbinaryen.so /usr/local/lib/
COPY --from=wasm-tools /emsdk/upstream/emscripten/system/include/libc/ /wasm32/include/
COPY --from=wasm-tools /emsdk/upstream/emscripten/system/lib/libc/musl/arch/emscripten/bits/ /wasm32/include/bits/
COPY --from=wasm-tools /opt/wasm-tools/wasm-pack /usr/local/cargo/bin/
ENV CPATH="/wasm32/include"
WORKDIR /src
CMD ["sh", "-c", "rm -rf pkg && wasm-pack build -- --verbose && rm pkg/.gitignore"]