building_with_espdl-2-failure

本人代码

正常部分

1
2
3

# 列出当前工作目录中的文件
%ls

这部分要注意windows的python库是_.pyd形式，而linux是esp-dl的git库中同名的_.so文件。

# 导入必要的库
from optimizer import *
from calibrator import *
from evaluator import *

1 2	`onnx_model = onnx.load("handrecognition_model.onnx")`

1 2	`optimized_model_path = optimize_fp_model("handrecognition_model.onnx")`

打印模型的计算图信息

代码

optimized_model = onnx.load(optimized_model_path)
print("Optimized ONNX model loaded successfully.")
print(onnx.helper.printable_graph(optimized_model.graph))

输出

Optimized ONNX model loaded successfully.
graph tf2onnx (
  %conv2d_input[FLOAT, unk__33x96x96x1]
) initializers (
  %new_shape__31[INT64, 4]
  %const_fold_opt__32[INT64, 2]
  %StatefulPartitionedCall/sequential/dense_1/MatMul/ReadVariableOp:0[FLOAT, 128x6]
  %StatefulPartitionedCall/sequential/dense_1/BiasAdd/ReadVariableOp:0[FLOAT, 6]
  %StatefulPartitionedCall/sequential/dense/MatMul/ReadVariableOp:0[FLOAT, 6400x128]
  %StatefulPartitionedCall/sequential/dense/BiasAdd/ReadVariableOp:0[FLOAT, 128]
  %StatefulPartitionedCall/sequential/conv2d_2/Conv2D/ReadVariableOp:0[FLOAT, 64x64x3x3]
  %StatefulPartitionedCall/sequential/conv2d_2/BiasAdd/ReadVariableOp:0[FLOAT, 64]
  %StatefulPartitionedCall/sequential/conv2d_1/Conv2D/ReadVariableOp:0[FLOAT, 64x32x3x3]
  %StatefulPartitionedCall/sequential/conv2d_1/BiasAdd/ReadVariableOp:0[FLOAT, 64]
  %StatefulPartitionedCall/sequential/conv2d/Conv2D/ReadVariableOp:0[FLOAT, 32x1x5x5]
  %StatefulPartitionedCall/sequential/conv2d/BiasAdd/ReadVariableOp:0[FLOAT, 32]
) {
  %StatefulPartitionedCall/sequential/conv2d/BiasAdd__6:0 = Reshape(%conv2d_input, %new_shape__31)
  %StatefulPartitionedCall/sequential/conv2d/BiasAdd:0 = Conv[dilations = [1, 1], group = 1, kernel_shape = [5, 5], strides = [1, 1]](%StatefulPartitionedCall/sequential/conv2d/BiasAdd__6:0, %StatefulPartitionedCall/sequential/conv2d/Conv2D/ReadVariableOp:0, %StatefulPartitionedCall/sequential/conv2d/BiasAdd/ReadVariableOp:0)
  %StatefulPartitionedCall/sequential/conv2d/Relu:0 = Relu(%StatefulPartitionedCall/sequential/conv2d/BiasAdd:0)
  %StatefulPartitionedCall/sequential/max_pooling2d/MaxPool:0 = MaxPool[kernel_shape = [2, 2], strides = [2, 2]](%StatefulPartitionedCall/sequential/conv2d/Relu:0)
  %StatefulPartitionedCall/sequential/conv2d_1/BiasAdd:0 = Conv[dilations = [1, 1], group = 1, kernel_shape = [3, 3], strides = [1, 1]](%StatefulPartitionedCall/sequential/max_pooling2d/MaxPool:0, %StatefulPartitionedCall/sequential/conv2d_1/Conv2D/ReadVariableOp:0, %StatefulPartitionedCall/sequential/conv2d_1/BiasAdd/ReadVariableOp:0)
  %StatefulPartitionedCall/sequential/conv2d_1/Relu:0 = Relu(%StatefulPartitionedCall/sequential/conv2d_1/BiasAdd:0)
  %StatefulPartitionedCall/sequential/max_pooling2d_1/MaxPool:0 = MaxPool[kernel_shape = [2, 2], strides = [2, 2]](%StatefulPartitionedCall/sequential/conv2d_1/Relu:0)
  %StatefulPartitionedCall/sequential/conv2d_2/BiasAdd:0 = Conv[dilations = [1, 1], group = 1, kernel_shape = [3, 3], strides = [1, 1]](%StatefulPartitionedCall/sequential/max_pooling2d_1/MaxPool:0, %StatefulPartitionedCall/sequential/conv2d_2/Conv2D/ReadVariableOp:0, %StatefulPartitionedCall/sequential/conv2d_2/BiasAdd/ReadVariableOp:0)
  %StatefulPartitionedCall/sequential/conv2d_2/Relu:0 = Relu(%StatefulPartitionedCall/sequential/conv2d_2/BiasAdd:0)
  %StatefulPartitionedCall/sequential/max_pooling2d_2/MaxPool:0 = MaxPool[kernel_shape = [2, 2], strides = [2, 2]](%StatefulPartitionedCall/sequential/conv2d_2/Relu:0)
  %StatefulPartitionedCall/sequential/max_pooling2d_2/MaxPool__28:0 = Transpose[perm = [0, 2, 3, 1]](%StatefulPartitionedCall/sequential/max_pooling2d_2/MaxPool:0)
  %StatefulPartitionedCall/sequential/flatten/Reshape:0 = Reshape(%StatefulPartitionedCall/sequential/max_pooling2d_2/MaxPool__28:0, %const_fold_opt__32)
  %StatefulPartitionedCall/sequential/dense/MatMul:0 = MatMul(%StatefulPartitionedCall/sequential/flatten/Reshape:0, %StatefulPartitionedCall/sequential/dense/MatMul/ReadVariableOp:0)
  %StatefulPartitionedCall/sequential/dense/BiasAdd:0 = Add(%StatefulPartitionedCall/sequential/dense/MatMul:0, %StatefulPartitionedCall/sequential/dense/BiasAdd/ReadVariableOp:0)
  %StatefulPartitionedCall/sequential/dense/Relu:0 = Relu(%StatefulPartitionedCall/sequential/dense/BiasAdd:0)
  %StatefulPartitionedCall/sequential/dense_1/MatMul:0 = MatMul(%StatefulPartitionedCall/sequential/dense/Relu:0, %StatefulPartitionedCall/sequential/dense_1/MatMul/ReadVariableOp:0)
  %StatefulPartitionedCall/sequential/dense_1/BiasAdd:0 = Add(%StatefulPartitionedCall/sequential/dense_1/MatMul:0, %StatefulPartitionedCall/sequential/dense_1/BiasAdd/ReadVariableOp:0)
  %dense_1 = Softmax(%StatefulPartitionedCall/sequential/dense_1/BiasAdd:0)
  return %dense_1
}

加载pickle数据集

代码

import pickle

# 加载校准数据集
with open('X_cal.pkl', 'rb') as f:
    test_images = pickle.load(f)
with open('y_cal.pkl', 'rb') as f:
    test_labels = pickle.load(f)

# 提取校准数据子集
calib_dataset = test_images[0:1800:20]
pickle_file_path = 'handrecognition_calib.pickle'

print("Calibration dataset loaded and subset selected.")

输出

Calibration dataset loaded and subset selected.

# 加载优化后的模型
model_proto = onnx.load(optimized_model_path)
print('Generating the quantization table:')

# 初始化校准器
calib = Calibrator('int16', 'per-tensor', 'minmax')
# 如果需要 int8 量化，可以使用以下替代代码：
# calib = Calibrator('int8', 'per-channel', 'minmax')

# 设置执行设备
calib.set_providers(['CPUExecutionProvider'])

报错部分

代码

就是这部分一直报错，不管是使用自己的1通道、10种代码，还是使用自己的3通道、6种代码，甚至直接运行原作者的model_development文件夹的model_development.ipynb和esp_dl_formate_conversionw文件夹的esp_dl_formate_conversion.py，也是失败，显示ValueError: current model is not supported by esp-dl。

# 生成量化参数
calib.generate_quantization_table(model_proto, calib_dataset, pickle_file_path)

# 导出系数文件
calib.export_coefficient_to_cpp(model_proto, pickle_file_path, 'esp32s3', '.', 'handrecognition_coefficient', True)

print("Calibration and quantization completed successfully.")

报错输出

Generating the quantization table:
MatMul is not supported on esp-dl yet



---------------------------------------------------------------------------

ValueError                                Traceback (most recent call last)

/tmp/ipykernel_107082/766837025.py in <module>
     12 
     13 # 生成量化参数
---> 14 calib.generate_quantization_table(model_proto, calib_dataset, pickle_file_path)
     15 
     16 # 导出系数文件


~/code/gesture recognition/calibrator.so in calibrator.Calibrator.generate_quantization_table()


~/code/gesture recognition/calibrator.so in calibrator.Calibrator.generate_output_model()


~/code/gesture recognition/calibrator.so in calibrator.Calibrator.check_model()
（执行到calibrator.Calibrator.check_model的部分有问题）

ValueError: current model is not supported by esp-dl

尝试过的检验方向（均已失败）

模型有效性

代码

import onnx
model_proto = onnx.load(optimized_model_path)
onnx.checker.check_model(model_proto)  # 验证 ONNX 模型是否有效
print("Model loaded and verified successfully.")

输出

Model loaded and verified successfully.

数据形状是否符合

代码

print(type(calib_dataset))
print(calib_dataset.shape)  # 确认它的形状
print(calib_dataset[:5])  # 查看部分数据是否符合预期

输出

<class 'numpy.ndarray'>
(90, 96, 96, 1)
[[[[0.01176471]
   [0.01568627]
   [0.01568627]
   ...
   [0.02352941]
   [0.01960784]
   [0.01960784]]

  [[0.01568627]
   [0.01960784]
   [0.01568627]
   ...
   [0.01960784]
   [0.01568627]
   [0.01176471]]

  [[0.01960784]
   [0.01960784]
   [0.01568627]
   ...
   [0.01568627]
   [0.01568627]
   [0.01176471]]

  ...

  [[0.04313725]
   [0.07058824]
   [0.08627451]
   ...
   [0.01568627]
   [0.01568627]
   [0.01568627]]

  [[0.04705882]
   [0.07058824]
   [0.07843137]
   ...
   [0.01960784]
   [0.01960784]
   [0.01568627]]

  [[0.04705882]
   [0.07058824]
   [0.07843137]
   ...
   [0.01960784]
   [0.01568627]
   [0.01568627]]]


 [[[0.01960784]
   [0.02352941]
   [0.01960784]
   ...
   [0.01960784]
   [0.01568627]
   [0.01568627]]

  [[0.01568627]
   [0.01568627]
   [0.01960784]
   ...
   [0.01568627]
   [0.01960784]
   [0.01176471]]

  [[0.01960784]
   [0.01960784]
   [0.02352941]
   ...
   [0.01568627]
   [0.01568627]
   [0.01568627]]

  ...

  [[0.01960784]
   [0.01960784]
   [0.01960784]
   ...
   [0.01960784]
   [0.01960784]
   [0.01960784]]

  [[0.01568627]
   [0.01960784]
   [0.01960784]
   ...
   [0.01176471]
   [0.01568627]
   [0.01568627]]

  [[0.01960784]
   [0.01960784]
   [0.01960784]
   ...
   [0.01960784]
   [0.01960784]
   [0.01568627]]]


 [[[0.02352941]
   [0.01960784]
   [0.02352941]
   ...
   [0.02352941]
   [0.02352941]
   [0.02352941]]

  [[0.02352941]
   [0.02352941]
   [0.02745098]
   ...
   [0.02352941]
   [0.02352941]
   [0.01568627]]

  [[0.02352941]
   [0.02352941]
   [0.01960784]
   ...
   [0.01960784]
   [0.01960784]
   [0.02352941]]

  ...

  [[0.01960784]
   [0.01960784]
   [0.02352941]
   ...
   [0.02352941]
   [0.01960784]
   [0.01960784]]

  [[0.01568627]
   [0.01960784]
   [0.01960784]
   ...
   [0.02352941]
   [0.02352941]
   [0.02745098]]

  [[0.02352941]
   [0.02352941]
   [0.02352941]
   ...
   [0.02352941]
   [0.02352941]
   [0.03137255]]]


 [[[0.01568627]
   [0.01960784]
   [0.01960784]
   ...
   [0.01568627]
   [0.01960784]
   [0.01568627]]

  [[0.01176471]
   [0.01960784]
   [0.01568627]
   ...
   [0.01568627]
   [0.01568627]
   [0.01176471]]

  [[0.01960784]
   [0.01960784]
   [0.01960784]
   ...
   [0.01176471]
   [0.01568627]
   [0.01568627]]

  ...

  [[0.06666667]
   [0.05882353]
   [0.06666667]
   ...
   [0.01960784]
   [0.01568627]
   [0.01568627]]

  [[0.05882353]
   [0.05490196]
   [0.05882353]
   ...
   [0.01568627]
   [0.01568627]
   [0.01568627]]

  [[0.05098039]
   [0.05098039]
   [0.0627451 ]
   ...
   [0.01960784]
   [0.01568627]
   [0.02352941]]]


 [[[0.02745098]
   [0.01960784]
   [0.02352941]
   ...
   [0.01568627]
   [0.01568627]
   [0.01568627]]

  [[0.01960784]
   [0.01960784]
   [0.02352941]
   ...
   [0.01960784]
   [0.01568627]
   [0.01960784]]

  [[0.01568627]
   [0.01960784]
   [0.01960784]
   ...
   [0.01568627]
   [0.01568627]
   [0.01568627]]

  ...

  [[0.01568627]
   [0.01568627]
   [0.01960784]
   ...
   [0.01568627]
   [0.01960784]
   [0.01568627]]

  [[0.02352941]
   [0.01960784]
   [0.01960784]
   ...
   [0.01568627]
   [0.01960784]
   [0.01568627]]

  [[0.01960784]
   [0.01568627]
   [0.01568627]
   ...
   [0.01960784]
   [0.01568627]
   [0.02352941]]]]

pickel路径有效性

代码

import os

# 设置文件路径
pickle_file_path = 'output/handrecognition_calib_params.pickle'

# 检查路径是否存在，不存在则创建
output_dir = os.path.dirname(pickle_file_path)
if not os.path.exists(output_dir):
    os.makedirs(output_dir)
    print(f"Created directory: {output_dir}")

# 检查是否可写
is_writeable = os.access(output_dir, os.W_OK)
print("Writable:", is_writeable)

输出

Writable: True

在第一次显示不可写并且创建了文件夹以后，仍然无法被esp-dl支持，因此也不是读写性的问题。

原作者代码运行证明失败

这部分我把原作者esp_dl_formate_conversion.py拆开到jupyter notebook中创建ipynb（当然前面模型训练部分也是直接使用model_development.ipynb），一步一步执行，最终同样卡在了同样的位置。

#import libraries 
from optimizer import *
from calibrator import *
from evaluator import *
import pickle

1 2	`#load ONNX model onnx_model = onnx.load("handrecognition_model.onnx")`

1 2	`#optimize ONNX model optimized_model_path = optimize_fp_model("handrecognition_model.onnx")`

#load calibration dataset
with open('X_cal.pkl', 'rb') as f:
    (test_images) = pickle.load(f)
with open('y_cal.pkl', 'rb') as f:
    (test_labels) = pickle.load(f)

calib_dataset = test_images[0:1800:20]
pickle_file_path = 'handrecognition_calib.pickle'

#calibration 
model_proto = onnx.load(optimized_model_path)
print('Generating the quantization table:')
calib = Calibrator('int16', 'per-tensor', 'minmax')

Generating the quantization table:

出错位置

代码

###for in8 conversion 
#calib = Calibrator('int8', 'per-channel', 'minmax') 

calib.set_providers(['CPUExecutionProvider'])
# Obtain the quantization parameter
calib.generate_quantization_table(model_proto,calib_dataset, pickle_file_path)
# Generate the coefficient files for esp32s3
calib.export_coefficient_to_cpp(model_proto,  pickle_file_path, 'esp32s3', '.', 'handrecognition_coefficient', True)

输出

MatMul is not supported on esp-dl yet



---------------------------------------------------------------------------

ValueError                                Traceback (most recent call last)

/tmp/ipykernel_102222/2534009994.py in <module>
      5 calib.set_providers(['CPUExecutionProvider'])
      6 # Obtain the quantization parameter
----> 7 calib.generate_quantization_table(model_proto,calib_dataset, pickle_file_path)
      8 # Generate the coefficient files for esp32s3
      9 calib.export_coefficient_to_cpp(model_proto,  pickle_file_path, 'esp32s3', '.', 'handrecognition_coefficient', True)


~/code/Blogs/ESP-DL/esp_dl_formate_conversion/calibrator.so in calibrator.Calibrator.generate_quantization_table()


~/code/Blogs/ESP-DL/esp_dl_formate_conversion/calibrator.so in calibrator.Calibrator.generate_output_model()


~/code/Blogs/ESP-DL/esp_dl_formate_conversion/calibrator.so in calibrator.Calibrator.check_model()


ValueError: current model is not supported by esp-dl

最终，感谢在薛兄的提醒下，发现有人给作者提过这个问题，但作者至今也并未解决：https://github.com/alibukharai/Blogs/issues/9

考虑可能成功的新方向

在windows中搭conda尝试，因为之前只在ubuntu 24.04中尝试过，有可能esp-dl官方库的.pyd能成功，只是.so文件有问题。
更换esp-idf和esp-dl的branch和tag，因为目前使用的是v5.3的idf和release/v1.1的dl，原作者采用的是v4.4的idf和idf4.4的idf。
使用其他的优化和量化模型的工具，esp-dl确实不是很完善。

fyp

building_with_espdl-2-failure

https://blakehansen130.github.io/2024/11/05/model_quantization_and_optimization/

发布于

2024年11月5日

许可协议

building_with_espdl-3-failure 上一篇

building_with_espdl-1-supplement 下一篇