Merge pull request #232 from JingyuanZhang/master

chore(v2.1.0): publish core@2.1.0 webgl@1.1.0 and update models version

Author: JingyuanZhang

packages/paddlejs-backend-webgl/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@paddlejs/paddlejs-backend-webgl",
-  "version": "1.0.7",
+  "version": "1.1.0",
   "description": "",
   "main": "lib/index",
   "scripts": {

packages/paddlejs-backend-webgl/package.json

@@ -47,7 +47,7 @@
 <script lang="ts">
 import Vue from 'vue';
 import { Runner } from '@paddlejs/paddlejs-core';
-import { GLOBALS } from '@paddlejs/paddlejs-core/globals';
+import { GLOBALS } from '@paddlejs/paddlejs-core/src/globals';
 import '@paddlejs/paddlejs-backend-webgl';
 import Utils from './utils';
 
@@ -64,7 +64,6 @@ export default Vue.extend({
                     },
                     fetchShape: [1, 1000, 10, 1],
                     fill: '#fff',
-                    targetSize: { height: 224, width: 224 },
                     needPreheat: false
                 },
                 {
@@ -75,7 +74,6 @@ export default Vue.extend({
                     },
                     fetchShape: [1, 1000, 10, 1],
                     fill: '#fff',
-                    targetSize: { height: 224, width: 224 },
                     needPreheat: false
                 },
                 {
@@ -86,7 +84,6 @@ export default Vue.extend({
                     },
                     fetchShape: [1, 40, 10, 1],
                     fill: '#fff',
-                    targetSize: { height: 224, width: 224 },
                     needPreheat: false
                 },
                 {
@@ -97,7 +94,6 @@ export default Vue.extend({
                     },
                     fetchShape: [1, 1920, 10 , 1],
                     fill: '#fff',
-                    targetSize: { height: 256, width: 256 },
                     needPreheat: false
                 },
                 {
@@ -108,7 +104,6 @@ export default Vue.extend({
                     },
                     fetchShape: [1, 9, 1, 1],
                     fill: '#fff',
-                    targetSize: { height: 224, width: 224 },
                     needPreheat: false
                 },
                 {
@@ -119,7 +114,6 @@ export default Vue.extend({
                     },
                     fetchShape: [1, 2, 192, 192],
                     fill: '#fff',
-                    targetSize: { height: 224, width: 224 },
                     needPreheat: false
                 }
             ],

packages/paddlejs-benchmark/src/config.vue

@@ -5,6 +5,42 @@ As the core part of the Paddle.js ecosystem, this package hosts `@paddlejs/paddl
 which is responsible for the operation of the inference process of the entire engine, 
 and provides interfaces for backend registration and environment variable registration.
 
+
+## RunnerConfig
+
+When registering the engine you need to configure the engine, you must configure the items `modelPath`, `feedShape`, all items are configured as follows.
+
+```typescript
+
+// model struture
+enum GraphType {
+    SingleOutput = 'single',
+    MultipleOutput = 'multiple',
+    MultipleInput = 'multipleInput'
+}
+
+interface RunnerConfig {
+    modelPath: string; // model path (local or web address)
+    modelName?: string; // model name
+    feedShape: { // input feed shape
+        fc?: number; // feed channel, default is 3.
+        fw: number; // feed width
+        fh: number; // feed height
+    };
+    fill?: Color; // the color used to padding
+    mean?: number[]; // mean value
+    std?: number[]; // standard deviation
+    bgr?: boolean; // whether the image channel alignment is BGR, default is false (RGB)
+    type?: GraphType; // model structure, default is singleInput and singleOutput
+    needPreheat?: boolean; // whether to warm up the engine during initialization, default is true
+    plugins?: { // register model topology transform plugin
+        preTransforms?: Transformer[]; // transform before creating network topology
+        transforms?: Transformer[]; // transform when traversing model layers
+        postTransforms?: Transformer[]; // transform the model topology diagram after it has been created
+    };
+}
+
+```
 ## Importing
 You can install this package via npm., `@paddlejs/paddlejs-core`
 
@@ -32,3 +68,56 @@ const res = await runner.predict(mediadata, callback?);
 **Note**: If you are importing the Core package, you also need to import a backend (e.g., 
 [paddlejs-backend-webgl](/packages/paddlejs-backend-webgl), [paddlejs-backend-webgpu](/packages/paddlejs-backend-webgpu)).
 
+
+## High-level use
+
+1. `@paddlejs/paddlejs-core` provides the interface `registerOp`, through which developers can register custom operators.
+
+2. `@paddlejs/paddlejs-core` provides the global variable `env` module, through which developers can register environment variables, using the following method:
+
+    ```js
+    // set env key/flag and value
+    env.set(key, value);
+
+    // get value by key/flag
+    env.get(key);
+    ```
+
+3. transform model stucture
+
+    By registering the model transformers through `runnerConfig.plugins`, developers can make changes (add, delete, change) to the model structure, such as pruning to remove unnecessary layers to speed up inference, or adding custom layers to the end of the model and turning post-processing into layers in the model to speed up post-processing.
+
+
+4. Turn on performance flag for acceleration
+
+    Paddle.js currently provides five performance `flags`, which can be set to `true` if you want to enable inference acceleration.
+
+
+    ```js
+    env.set('webgl_pack_channel', true);
+    ```
+    Turn on `webgl_pack_channel` and the eligible conv2d will use packing shader to perform packing transformations to improve performance through vectorization calculations.
+
+
+    ```js
+    env.set('webgl_force_half_float_texture', true);
+    ```
+    Enable `webgl_force_half_float_texture`, feature map uses half float `HALF_FLOAT`.
+
+
+    ```js
+    env.set('webgl_feed_process', true);
+    ```
+    Turn on `webgl_feed_process` to convert all pre-processing parts of the model to shader processing, and keep the original image texture.
+
+
+    ```js
+    env.set('webgl_gpu_pipeline', true);
+    ```
+    Turn on `webgl_gpu_pipeline` to convert all model pre-processing parts to shader processing, and render the model results to `WebGL2RenderingContext` of webgl backend on screen. Developers can perform model post-processing on the output texture and the original image texture to achieve the `GPU_PIPELINE`: pre-processing + inference + post-processing (rendering processing) to get high performance. Take humanseg model case for reference.
+
+
+    ```js
+    env.set('webgl_pack_output', true);
+    ```
+    Enable `webgl_pack_output` to migrate the `NHWC` to `NCHW` layout transformation of the model output to the GPU, and `pack` to a four-channel layout to reduce loop processing when reading the results from the GPU

packages/paddlejs-core/README.md

@@ -4,7 +4,43 @@
 
 是 Paddle.js 推理引擎的核心部分，npm 包名是 `@paddlejs/paddlejs-core`，负责整个引擎的推理流程运行，提供计算方案注册、环境变量注册的接口。
 
-## 安装
+
+## 引擎配置 RunnerConfig
+注册引擎时需要对引擎进行配置，必须配置项为 `modelPath`、`feedShape`，所有项目配置如下：
+
+```typescript
+
+// 模型网络图结构
+enum GraphType {
+    SingleOutput = 'single', // 单输出
+    MultipleOutput = 'multiple', // 多输出
+    MultipleInput = 'multipleInput' // 多输入
+}
+
+interface RunnerConfig {
+    modelPath: string; // 模型路径（本地路径或者网络地址）
+    modelName?: string; // 模型名称
+    feedShape: { // 模型输入 feed shape
+        fc?: number; // 通道，默认为 3
+        fw: number; // 宽
+        fh: number; // 高
+    };
+    fill?: string; // 缩放后用什么颜色填充不足方形部分
+    mean?: number[]; // 平均值
+    std?: number[]; // 标准差
+    bgr?: boolean; // 图片通道排列是否是 BGR，默认为 false，为主流 RGB
+    type?: GraphType; // 模型网络图结构，默认单输入单输出
+    needPreheat?: boolean; // 是否在引擎初始化时进行预热，默认为 true
+    plugins?: { // 注册模型拓扑结构转换插件
+        preTransforms?: Transformer[]; // 在创建网络拓扑前进行转换
+        transforms?: Transformer[]; // 在遍历模型层时进行转换
+        postTransforms?: Transformer[]; // 创建网络拓扑图完成后进行转换
+    };
+}
+
+```
+
+## 安装和使用
 使用 npm 安装，`@paddlejs/paddlejs-core`
 
 ```js
@@ -30,3 +66,59 @@ const res = await runner.predict(mediadata, callback?);
 
 **Note**: 如果你引入 paddlejs-core，你仍需引入一个计算方案。(目前我们提供两种方案, webgpu 目前还是实验阶段，需要使用 chrome canary 访问，
 [paddlejs-backend-webgl](/packages/paddlejs-backend-webgl), [paddlejs-backend-webgpu](/packages/paddlejs-backend-webgpu)).
+
+
+
+## 高阶使用
+
+1. `@paddlejs/paddlejs-core` 提供接口 `registerOp`，开发者可以通过该接口，完成自定义算子注册。
+
+2. `@paddlejs/paddlejs-core` 提供全局变量 `env` 模块，开发者可以通过该模块完成环境变量注册，使用方法如下：
+
+    ```js
+    // set env key/flag and value
+    env.set(key, value);
+
+    // get value by key/flag
+    env.get(key);
+    ```
+
+3. 改变模型结构
+
+    通过 `runnerConfig.plugins` 注册模型转换器，开发者可以对模型结构进行改变（增、删、改），比如通过剪枝去除不必要的模型层来加快推理，也可以将自定义层加入模型最后，将后处理变为模型中的层来加速后处理。
+
+
+4. 开启性能 flag 实现加速
+
+    目前 Paddle.js 提供五个性能相关 `flag`，如需开启推理加速，可以将相关 `flag` 置 `true`。
+
+
+    #### 开启 `webgl_pack_channel`，合适的 conv2d 会使用 packing shader 进行排布变换，通过向量化计算来提高性能。
+
+    ```js
+    env.set('webgl_pack_channel', true);
+    ```
+
+    #### 开启 `webgl_force_half_float_texture`，feature map 使用半浮点 `HALF_FLOAT`
+
+    ```js
+    env.set('webgl_force_half_float_texture', true);
+    ```
+
+    #### 开启 `webgl_feed_process`，将模型前处理部分全部转换为 shader GPU 处理，并保留原始图片 texture
+
+    ```js
+    env.set('webgl_feed_process', true);
+    ```
+
+    #### 开启 `webgl_gpu_pipeline`，将模型前处理部分全部转换为 shader GPU 处理，且将模型推理结果上屏渲染到 webgl backend 的  `WebGL2RenderingContext` 上。开发者可对输出结果 texture 和原始图片 texture 进行模型后处理，来实现 GPU 全流程：前处理+推理+后处理（渲染处理），获得高性能，可参考 humanseg model 案例。
+
+    ```js
+    env.set('webgl_gpu_pipeline', true);
+    ```
+
+    #### 开启 `webgl_pack_output`，将模型输出结果 `NHWC` 到 `NCHW` 排布变换迁移至 GPU 进行，并 `pack` 为四通道排布，从 GPU 读取结果时，可以减少循环处理
+
+    ```js
+    env.set('webgl_pack_output', true);
+    ```

packages/paddlejs-core/README_cn.md

@@ -1,12 +1,12 @@
 import Graph from '../../src/graph';
-import { ModelConfig } from '../../src/commons/interface';
+import { RunnerConfig } from '../../src/commons/interface';
 import modelInfo from '../env/mock/model.json';
 
 describe('test graph', () => {
 
     const graphGenerator = new Graph(modelInfo, {
         type: 'single'
-    } as ModelConfig);
+    } as RunnerConfig);
     const weightMap = graphGenerator.createGraph();
 
 

packages/paddlejs-core/__tests__/spec/graph.test.ts

@@ -1,6 +1,6 @@
 {
   "name": "@paddlejs/paddlejs-core",
-  "version": "2.0.7",
+  "version": "2.1.0",
   "description": "",
   "main": "lib/index",
   "scripts": {

packages/paddlejs-core/package.json

@@ -38,18 +38,14 @@ export interface Model {
     multiOutputs?: ModelVar[]
 }
 
-export interface ModelConfig {
+export interface RunnerConfig {
     modelPath: string;
     modelName?: string;
     feedShape: {
         fc?: number;
         fw: number;
         fh: number;
     };
-    targetSize?: {
-        height: number;
-        width: number;
-    };
     fill?: string; // 缩放后用什么颜色填充不足方形部分
     mean?: number[];
     std?: number[];

packages/paddlejs-core/src/commons/interface.ts

@@ -2,7 +2,7 @@
  * @file ModelGraph，graph 生成器
  */
 
-import { ModelOp, GraphType, Model, ModelVar, ModelConfig } from './commons/interface';
+import { ModelOp, GraphType, Model, ModelVar, RunnerConfig } from './commons/interface';
 import OpExecutor from './opFactory/opExecutor';
 import transformActions from './transform';
 
@@ -28,11 +28,11 @@ export default class ModelGraph {
     weightMap: OpExecutor[] = [];
     ops: ModelOp[] = [];
     vars: ModelVar[] = [];
-    config: ModelConfig = {} as ModelConfig;
+    config: RunnerConfig = {} as RunnerConfig;
     type: GraphType = GraphType.SingleOutput;
     plugins: GraphPlugins = null;
 
-    constructor(model: Model, config: ModelConfig) {
+    constructor(model: Model, config: RunnerConfig) {
         this.ops = model.ops;
         this.vars = model.vars;
         this.type = config.type || this.type;

packages/paddlejs-core/src/graph.ts

@@ -109,13 +109,11 @@ export default class MediaProcessor {
      */
     allReshapeToRGB(imageData, opt) {
         // mean和std是介于0-1之间的
-        const { mean, std, targetShape, bgr } = opt;
+        const { mean, std, targetShape, bgr, normalizeType = 0 } = opt;
         const [, c, h, w] = targetShape;
         const data = imageData.data || imageData;
         const result = new Float32Array(h * w * c);
         let offset = 0;
-        // 将数据映射为0~1， 1：映射为-1~1之间
-        const normalizeType = 0;
         // h w c
         for (let i = 0; i < h; ++i) {
             const iw = i * w;