Update dependency @webpack-cli/serve to v0.2.0

We've been running each Make command in a single thread, resulting in fairly slow builds for C++ codecs.

This change instead runs all `make` invocations with `-j` defaulting to number of cores (retrieved via `nproc`).

On my machine Docker uses a VM configured to 4 cores out of 8 available. This change brings total build time for C++ codecs down from 10m28s to 7m5s (~3.5 minutes difference).

Note (1): I've converted imagequant builds to use built-in `make` as well to leverage this parallelisation and future-proof build script.
Note (2): we don't need to do the same for Rust, since Cargo parallelises builds by default.

codecs/imagequant/build.sh

@@ -11,16 +11,9 @@ echo "============================================="
 echo "Compiling libimagequant"
 echo "============================================="
 (
-  emcc \
-    --bind \
-    ${OPTIMIZE} \
-    -s ALLOW_MEMORY_GROWTH=1 \
-    -s MODULARIZE=1 \
-    -s 'EXPORT_NAME="imagequant"' \
-    -I node_modules/libimagequant \
-    --std=c99 \
-    -c \
-    node_modules/libimagequant/{libimagequant,pam,mediancut,blur,mempool,kmeans,nearest}.c
+  cd node_modules/libimagequant
+  emconfigure ./configure --disable-sse
+  emmake make static -j`nproc`
 )
 echo "============================================="
 echo "Compiling wasm module"
@@ -29,14 +22,16 @@ echo "============================================="
   emcc \
     --bind \
     ${OPTIMIZE} \
+    --closure 1 \
     -s ALLOW_MEMORY_GROWTH=1 \
     -s MODULARIZE=1 \
     -s 'EXPORT_NAME="imagequant"' \
     -I node_modules/libimagequant \
     -o ./imagequant.js \
-    --std=c++11 *.o \
+    --std=c++11 \
     -x c++ \
-    imagequant.cpp
+    imagequant.cpp \
+    node_modules/libimagequant/libimagequant.a
 )
 echo "============================================="
 echo "Compiling wasm module done"
@@ -44,5 +39,5 @@ echo "============================================="
 
 echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
 echo "Did you update your docker image?"
-echo "Run \`docker pull trzeci/emscripten\`"
+echo "Run \`docker pull trzeci/emscripten-upstream\`"
 echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"

codecs/imagequant/imagequant.cpp

@@ -61,21 +61,21 @@ RawImage quantize(std::string rawimage, int image_width, int image_height, int n
 }
 
 const liq_color zx_colors[] = {
-  {.a = 255, .r = 0, .g = 0, .b = 0},       // regular black
-  {.a = 255, .r = 0, .g = 0, .b = 215},     // regular blue
-  {.a = 255, .r = 215, .g = 0, .b = 0},     // regular red
-  {.a = 255, .r = 215, .g = 0, .b = 215},   // regular magenta
-  {.a = 255, .r = 0, .g = 215, .b = 0},     // regular green
-  {.a = 255, .r = 0, .g = 215, .b = 215},   // regular cyan
-  {.a = 255, .r = 215, .g = 215, .b = 0},   // regular yellow
-  {.a = 255, .r = 215, .g = 215, .b = 215}, // regular white
-  {.a = 255, .r = 0, .g = 0, .b = 255},     // bright blue
-  {.a = 255, .r = 255, .g = 0, .b = 0},     // bright red
-  {.a = 255, .r = 255, .g = 0, .b = 255},   // bright magenta
-  {.a = 255, .r = 0, .g = 255, .b = 0},     // bright green
-  {.a = 255, .r = 0, .g = 255, .b = 255},   // bright cyan
-  {.a = 255, .r = 255, .g = 255, .b = 0},   // bright yellow
-  {.a = 255, .r = 255, .g = 255, .b = 255}  // bright white
+  {.r = 0, .g = 0, .b = 0, .a = 255},       // regular black
+  {.r = 0, .g = 0, .b = 215, .a = 255},     // regular blue
+  {.r = 215, .g = 0, .b = 0, .a = 255},     // regular red
+  {.r = 215, .g = 0, .b = 215, .a = 255},   // regular magenta
+  {.r = 0, .g = 215, .b = 0, .a = 255},     // regular green
+  {.r = 0, .g = 215, .b = 215, .a = 255},   // regular cyan
+  {.r = 215, .g = 215, .b = 0, .a = 255},   // regular yellow
+  {.r = 215, .g = 215, .b = 215, .a = 255}, // regular white
+  {.r = 0, .g = 0, .b = 255, .a = 255},     // bright blue
+  {.r = 255, .g = 0, .b = 0, .a = 255},     // bright red
+  {.r = 255, .g = 0, .b = 255, .a = 255},   // bright magenta
+  {.r = 0, .g = 255, .b = 0, .a = 255},     // bright green
+  {.r = 0, .g = 255, .b = 255, .a = 255},   // bright cyan
+  {.r = 255, .g = 255, .b = 0, .a = 255},   // bright yellow
+  {.r = 255, .g = 255, .b = 255, .a = 255}  // bright white
 };
 
 uint8_t block[8 * 8 * 4];

codecs/imagequant/package.json

@@ -2,7 +2,7 @@
   "name": "imagequant",
   "scripts": {
     "install": "napa",
-    "build": "docker run --rm -v $(pwd):/src trzeci/emscripten ./build.sh"
+    "build": "docker run --rm -v $(pwd):/src trzeci/emscripten-upstream ./build.sh"
   },
   "napa": {
     "libimagequant": "ImageOptim/libimagequant#2.12.1"

codecs/mozjpeg_enc/build.sh

@@ -17,7 +17,7 @@ echo "============================================="
   cd node_modules/mozjpeg
   autoreconf -fiv
   emconfigure ./configure --without-simd
-  emmake make libjpeg.la
+  emmake make libjpeg.la -j`nproc`
 )
 echo "============================================="
 echo "Compiling mozjpeg done"
@@ -30,6 +30,7 @@ echo "============================================="
   emcc \
     --bind \
     ${OPTIMIZE} \
+    --closure 1 \
     -s WASM=1 \
     -s ALLOW_MEMORY_GROWTH=1 \
     -s MODULARIZE=1 \
@@ -49,5 +50,5 @@ echo "============================================="
 
 echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
 echo "Did you update your docker image?"
-echo "Run \`docker pull trzeci/emscripten\`"
+echo "Run \`docker pull trzeci/emscripten-upstream\`"
 echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"

codecs/mozjpeg_enc/package.json

@@ -2,7 +2,7 @@
   "name": "mozjpeg_enc",
   "scripts": {
     "install": "napa",
-    "build": "docker run --rm -v $(pwd):/src trzeci/emscripten ./build.sh"
+    "build": "docker run --rm -v $(pwd):/src trzeci/emscripten-upstream ./build.sh"
   },
   "napa": {
     "mozjpeg": "mozilla/mozjpeg#v3.3.1"

codecs/oxipng/Cargo.lock

@@ -282,6 +282,7 @@ dependencies = [
 name = "oxipng"
 version = "0.1.0"
 dependencies = [
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "oxipng 2.3.0 (git+https://github.com/shssoichiro/oxipng.git)",
  "wasm-bindgen 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -289,7 +290,7 @@ dependencies = [
 [[package]]
 name = "oxipng"
 version = "2.3.0"
-source = "git+https://github.com/shssoichiro/oxipng.git#f74726915131ec7ff1df0514436aac34bc603cff"
+source = "git+https://github.com/shssoichiro/oxipng.git#ec8ecf5a800dfb41359d9cf41eed8a730062b9a8"
 dependencies = [
  "bit-vec 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -299,9 +300,9 @@ dependencies = [
  "indexmap 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libdeflater 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "miniz_oxide 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "rgb 0.8.16 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rgb 0.8.17 (registry+https://github.com/rust-lang/crates.io-index)",
  "zopfli 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -356,7 +357,7 @@ dependencies = [
 
 [[package]]
 name = "rgb"
-version = "0.8.16"
+version = "0.8.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
@@ -488,7 +489,7 @@ dependencies = [
 "checksum quote 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2bdc6c187c65bca4260c9011c9e3132efe4909da44726bad24cf7572ae338d7f"
 "checksum rayon 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db6ce3297f9c85e16621bb8cca38a06779ffc31bb8184e1be4bed2be4678a098"
 "checksum rayon-core 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "08a89b46efaf957e52b18062fb2f4660f8b8a4dde1807ca002690868ef2c85a9"
-"checksum rgb 0.8.16 (registry+https://github.com/rust-lang/crates.io-index)" = "5ec4ab2cf0b27e111e266e161cf7f9efd20125a161190da1c0945c4a4408fef3"
+"checksum rgb 0.8.17 (registry+https://github.com/rust-lang/crates.io-index)" = "a85b83fd629b0ce765f45316774fa6aaa95947fd74c8e4bbf3c6d1e349701d95"
 "checksum scopeguard 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 "checksum syn 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)" = "0df0eb663f387145cab623dea85b09c2c5b4b0aef44e945d928e682fce71bb03"
 "checksum typed-arena 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a9b2228007eba4120145f785df0f6c92ea538f5a3635a612ecf4e334c8c1446d"

codecs/oxipng/Cargo.toml

@@ -11,6 +11,7 @@ crate-type = ["cdylib"]
 [dependencies]
 oxipng = { version = "2.3.0", default-features = false }
 wasm-bindgen = "0.2.48"
+log = { version = "0.4", features = ["release_max_level_off"] }
 
 [profile.release]
 lto = true

codecs/oxipng/src/malloc_shim.rs

@@ -2,6 +2,26 @@
 //! These implementations are compatible with the standard signatures
 //! but use Rust allocator instead of including libc one as well.
 //!
+//! Rust allocator APIs requires passing size and alignment to the
+//! `dealloc` function. This is different from C API, which only
+//! expects a pointer in `free` and expects allocators to take care of
+//! storing any necessary information elsewhere.
+//!
+//! In order to simulate C API, we allocate a `size_and_data_ptr`
+//! of size `sizeof(usize) + size` where `size` is the requested number
+//! of bytes. Then, we store `size` at the beginning of the allocated
+//! chunk (within those `sizeof(usize)` bytes) and return
+//! `data_ptr = size_and_data_ptr + sizeof(usize)` to the calleer:
+//!
+//! [`size`][...actual data]
+//! -^------------------ `size_and_data_ptr`
+//! ---------^---------- `data_ptr`
+//!
+//! Then, in `free`, the caller gives us `data_ptr`. We can subtract
+//! `sizeof(usize)` back and get the original `size_and_data_ptr`.
+//! At this point we can read `size` back and call the Rust `dealloc`
+//! for the whole allocated chunk.
+//!
 //! I've raised an upstream issue to hopefully make this easier in
 //! future: https://github.com/ebiggers/libdeflate/issues/62
 
@@ -20,8 +40,8 @@ pub unsafe extern "C" fn malloc(size: usize) -> *mut u8 {
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn free(ptr: *mut u8) {
-    let size_and_data_ptr = ptr.sub(size_of::<usize>());
+pub unsafe extern "C" fn free(data_ptr: *mut u8) {
+    let size_and_data_ptr = data_ptr.sub(size_of::<usize>());
     let size = *(size_and_data_ptr as *const usize);
-    dealloc(ptr, layout_for(size))
+    dealloc(size_and_data_ptr, layout_for(size))
 }

codecs/webp_dec/build.sh

@@ -31,7 +31,7 @@ test -n "$SKIP_LIBWEBP" || (
     --disable-neon \
     --disable-sse2 \
     --disable-sse4.1
-  emmake make
+  emmake make -j`nproc`
 )
 echo "============================================="
 echo "Compiling wasm bindings"
@@ -39,6 +39,7 @@ echo "============================================="
 (
   emcc \
     ${OPTIMIZE} \
+    --closure 1 \
     --bind \
     -s ALLOW_MEMORY_GROWTH=1 \
     -s MODULARIZE=1 \
@@ -56,5 +57,5 @@ echo "============================================="
 
 echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
 echo "Did you update your docker image?"
-echo "Run \`docker pull trzeci/emscripten\`"
+echo "Run \`docker pull trzeci/emscripten-upstream\`"
 echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"

codecs/webp_dec/package.json

@@ -2,7 +2,7 @@
   "name": "webp_dec",
   "scripts": {
     "install": "napa",
-    "build": "docker run --rm -v $(pwd):/src trzeci/emscripten ./build.sh"
+    "build": "docker run --rm -v $(pwd):/src trzeci/emscripten-upstream ./build.sh"
   },
   "napa": {
     "libwebp": "webmproject/libwebp#v1.0.2"

codecs/webp_enc/build.sh

@@ -32,7 +32,7 @@ test -n "$SKIP_LIBWEBP" || (
     --disable-neon \
     --disable-sse2 \
     --disable-sse4.1
-  emmake make
+  emmake make -j`nproc`
 )
 echo "============================================="
 echo "Compiling wasm bindings"
@@ -40,6 +40,7 @@ echo "============================================="
 (
   emcc \
     ${OPTIMIZE} \
+    --closure 1 \
     --bind \
     -s ALLOW_MEMORY_GROWTH=1 \
     -s MODULARIZE=1 \
@@ -57,5 +58,5 @@ echo "============================================="
 
 echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
 echo "Did you update your docker image?"
-echo "Run \`docker pull trzeci/emscripten\`"
+echo "Run \`docker pull trzeci/emscripten-upstream\`"
 echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"

codecs/webp_enc/package.json

@@ -2,7 +2,7 @@
   "name": "webp_enc",
   "scripts": {
     "install": "napa",
-    "build": "docker run --rm -v $(pwd):/src trzeci/emscripten ./build.sh"
+    "build": "docker run --rm -v $(pwd):/src trzeci/emscripten-upstream ./build.sh"
   },
   "napa": {
     "libwebp": "webmproject/libwebp#v1.0.2"

package.json

@@ -20,7 +20,7 @@
     "@types/pretty-bytes": "5.1.0",
     "@types/webassembly-js-api": "0.0.3",
     "@webcomponents/custom-elements": "1.2.4",
-    "@webpack-cli/serve": "0.1.8",
+    "@webpack-cli/serve": "0.2.0",
     "assets-webpack-plugin": "3.9.10",
     "chalk": "2.4.2",
     "chokidar": "3.0.2",

src/codecs/oxipng/options.tsx

@@ -28,7 +28,7 @@ export default class OxiPNGEncoderOptions extends Component<Props, {}> {
           <Range
             name="level"
             min="0"
-            max="6"
+            max="3"
             step="1"
             value={options.level}
             onInput={this.onChange}

src/components/compress/index.tsx

@@ -109,11 +109,6 @@ async function preprocessImage(
     } else if (isHqx(preprocessData.resize)) {
       // Hqx can only do x2, x3 or x4.
       result = await processor.workerResize(result, preprocessData.resize);
-      // Seems like the globals from Rust from hqx and resize are conflicting.
-      // For now we can fix that by terminating the worker.
-      // TODO: Use wasm-bindgen’s new --web target to create a proper ES6 module
-      // and remove this.
-      processor.terminateWorker();
       // If the target size is not a clean x2, x3 or x4, use Catmull-Rom
       // for the remaining scaling.
       const pixelOpts = { ...preprocessData.resize, method: 'catrom' };