diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/gkdropout.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/gkdropout.py index 8101c6c7..74870291 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/gkdropout.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/gkdropout.py @@ -22,12 +22,27 @@ class GkDropout(Expander): """GkDropout expander""" def _expand(self, graph_builder): + """ + 对输入数据进行dropout操作。 + + Args: + graph_builder (GraphBuilder): 图构建器对象。 + + Returns: + tuple: 包含两个元素,第一个是执行dropout操作后的结果,第二个是生成的掩码。 + + """ + # 获取输入数据和掩码 input_x, input_mask = self.inputs + # 获取保持概率 keep_prob = self.attrs['keep_prob'] + # 计算保持概率的倒数 r_keep_prob = graph_builder.value(input_x.dtype, 1.0 / keep_prob) + # 计算保持概率 keep_prob = graph_builder.value(input_x.dtype, keep_prob) + # 如果掩码的数据类型与输入数据类型不同,则进行类型转换 if input_mask.dtype != input_x.dtype: input_mask = graph_builder.emit('Cast', [input_mask], attrs={'dst_type': input_x.dtype}) mask = graph_builder.emit('LessEqual', [input_mask, keep_prob]) # output is bool type diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/identity.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/identity.py index fe500660..d0abf4c3 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/identity.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/identity.py @@ -20,6 +20,16 @@ class Identity(Expander): """Identity expander""" def _expand(self, graph_builder): - input_x = self.inputs[0] - result = graph_builder.emit('Reshape', [input_x], attrs={'shape': input_x.shape}) - return result + """ + 对输入数据进行重塑操作。 + + Args: + graph_builder (GraphBuilder): 图构建器对象,用于构建计算图。 + + Returns: + Tensor: 重塑后的输入数据。 + + """ + input_x = self.inputs[0] # 获取输入数据 + result = graph_builder.emit('Reshape', [input_x], attrs={'shape': input_x.shape}) # 使用图构建器对象构建计算图,对输入数据进行重塑操作 + return result # 返回重塑后的输入数据 diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/layernorm.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/layernorm.py index eaa44140..791445c1 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/layernorm.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/layernorm.py @@ -25,67 +25,107 @@ class LayerNorm(Expander): """LayerNorm expander""" def _expand(self, graph_builder): + """ + 对输入进行扩展处理,包括批量归一化操作。 + + Args: + graph_builder (GraphBuilder): 图构建器对象,用于构建计算图。 + + Returns: + tuple: 包含三个元素的元组,分别是处理后的输入、均值和方差。 + - res (Tensor): 处理后的输入张量。 + - mean (Tensor): 输入的均值张量。 + - variance (Tensor): 输入的方差张量。 + + """ + # 获取输入数据 input_x, input_gamma, input_beta = self.inputs + # 获取处理器类型 processor = self.processor + # 获取归一化开始轴 begin_norm_axis = self.attrs['begin_norm_axis'] + # 获取epsilon值 epsilon = self.attrs['epsilon'] + # 获取输入数据的原始数据类型 ori_dtype = input_x.dtype + # 如果处理器类型为aicore且输入数据类型为float16,则将输入数据类型转换为float32 if processor == 'aicore' and ori_dtype == 'float16': input_x = graph_builder.emit('Cast', [input_x], attrs={'dst_type': 'float32'}) input_gamma = graph_builder.emit('Cast', [input_gamma], attrs={'dst_type': 'float32'}) input_beta = graph_builder.emit('Cast', [input_beta], attrs={'dst_type': 'float32'}) + # 获取输入数据的原始形状 ori_shape_x = input_x.shape + # 如果输入数据的格式为FRAC_NZ,则根据FRAC_NZ格式获取输入数据的形状 if input_x.data_format == DF.FRAC_NZ: ori_shape_x = infer_shape_from_fractalnz(input_x.shape) # Calculate the scaling ratio of the average + # 如果begin_norm_axis小于0,则将其加上ori_shape_x的长度 if begin_norm_axis < 0: begin_norm_axis += len(ori_shape_x) + # 定义reduce_axis,用于存储需要归一化的维度 reduce_axis = () + # 遍历ori_shape_x,如果维度大于begin_norm_axis或者等于begin_norm_axis,则将其加入reduce_axis for i, _ in enumerate(ori_shape_x): if i > begin_norm_axis or i == begin_norm_axis: reduce_axis = reduce_axis + (i,) + # 计算reduce_elts,即需要归一化的维度上的元素个数 reduce_elts = 1.0 for i in reduce_axis: reduce_elts *= ori_shape_x[i] # after reduced + # 获取归一化后的ori_shape_x ori_reduced_shape_x = get_reduced_ori_shape(ori_shape_x, reduce_axis) + # 定义axis,用于存储归一化的维度 axis = reduce_axis + # 如果input_x的数据格式为DF.FRAC_NZ,则将axis转换为frac_z轴 if input_x.data_format == DF.FRAC_NZ: axis = to_frac_z_axis(ori_shape_x, reduce_axis) + # 计算mean_cof_v,即归一化系数 mean_cof_v = graph_builder.value(input_x.dtype, 1.0 / reduce_elts) # Calculate mean + # 计算输入张量的均值 mean_red = graph_builder.emit('ReduceSum', [input_x], attrs={'reduce_axis': axis, 'keep_dims': True}) + # 将均值乘以系数 mean = graph_builder.emit('Mul', [mean_red, mean_cof_v]) + # 如果输入张量的数据格式为DF.FRAC_NZ,则对均值进行重整 if input_x.data_format == DF.FRAC_NZ: mean = graph_builder.emit('Reshape', [mean], attrs={'shape': ori_reduced_shape_x}) # Calculate variance - variance_sub = graph_builder.emit('Sub', [input_x, mean]) - variance_mul = graph_builder.emit('Mul', [variance_sub, variance_sub]) - variance_red = graph_builder.emit('ReduceSum', [variance_mul], attrs={'reduce_axis': axis, 'keep_dims': True}) - variance = graph_builder.emit('Mul', [variance_red, mean_cof_v]) + # 计算方差 + variance_sub = graph_builder.emit('Sub', [input_x, mean]) # 计算输入与均值的差值 + variance_mul = graph_builder.emit('Mul', [variance_sub, variance_sub]) # 计算差值的平方 + variance_red = graph_builder.emit('ReduceSum', [variance_mul], attrs={'reduce_axis': axis, 'keep_dims': True}) # 对差值的平方求和 + variance = graph_builder.emit('Mul', [variance_red, mean_cof_v]) # 计算方差 + # 如果输入数据的格式为DF.FRAC_NZ,则对方差进行reshape操作 if input_x.data_format == DF.FRAC_NZ: variance = graph_builder.emit('Reshape', [variance], attrs={'shape': ori_reduced_shape_x}) # Calculate normalize + # 计算输入x与均值之间的差值 normalize_sub = graph_builder.emit('Sub', [input_x, mean]) + # 创建一个epsilon值,用于防止除零错误 epsilon_v = graph_builder.value(input_x.dtype, epsilon) + # 计算方差加上epsilon的值 normalize_add = graph_builder.emit('Add', [variance, epsilon_v]) normlize_rsqrt = graph_builder.emit('Rsqrt', [normalize_add]) normalize_mul = graph_builder.emit('Mul', [normalize_sub, normlize_rsqrt]) # Calculate scale and translate + # 计算归一化后的乘积 scale_mul = graph_builder.emit('Mul', [normalize_mul, input_gamma]) + # 计算最终结果 res = graph_builder.emit('Add', [scale_mul, input_beta]) + # 如果处理器为aicore且原始数据类型为float16,则将结果、均值和方差转换为float16 if processor == 'aicore' and ori_dtype == 'float16': res = graph_builder.emit('Cast', [res], attrs={'dst_type': 'float16'}) mean = graph_builder.emit('Cast', [mean], attrs={'dst_type': 'float16'}) diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/layernorm_grad.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/layernorm_grad.py index 2ae7078b..4facbef1 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/layernorm_grad.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/layernorm_grad.py @@ -23,13 +23,33 @@ class LayerNormGrad(Expander): """LayerNormGrad expander""" def _expand(self, graph_builder): + """ + 对输入进行扩展操作。 + + Args: + graph_builder (GraphBuilder): 图构建器对象。 + + Returns: + tuple: 包含dx, dg, db的元组。 + dx (Tensor): 梯度相对于输入x的导数。 + dg (Tensor): 梯度相对于gamma的导数。 + db (Tensor): 梯度相对于beta的导数。 + + """ + # 获取输入参数 x, dy, variance, mean, gamma = self.inputs + # 获取处理器类型 processor = self.processor + # 获取归一化轴的起始位置 begin_norm_axis = self.attrs['begin_norm_axis'] + # 获取参数轴的起始位置 begin_params_axis = self.attrs['begin_params_axis'] + # 获取epsilon值,默认为1e-12 epsilon = self.attrs['epsilon'] if 'epsilon' in self.attrs else 1e-12 + # 获取输入数据的原始数据类型 ori_dtype = x.dtype + # 如果处理器类型为aicore且数据类型为float16,则将输入数据转换为float32 if processor == 'aicore' and ori_dtype == 'float16': x = graph_builder.emit('Cast', [x], attrs={'dst_type': 'float32'}) dy = graph_builder.emit('Cast', [dy], attrs={'dst_type': 'float32'}) @@ -37,77 +57,121 @@ class LayerNormGrad(Expander): mean = graph_builder.emit('Cast', [mean], attrs={'dst_type': 'float32'}) gamma = graph_builder.emit('Cast', [gamma], attrs={'dst_type': 'float32'}) + # 如果归一化轴的起始位置小于0,则将其转换为正数 if begin_norm_axis < 0: begin_norm_axis += len(x.shape) + # 如果参数轴的起始位置小于0,则将其转换为正数 if begin_params_axis < 0: begin_params_axis += len(x.shape) + # 获取归一化轴和参数轴的范围 norm_axis = tuple(range(begin_norm_axis, len(x.shape))) param_axis = tuple(range(0, begin_params_axis)) + # 计算归一化轴的维度乘积 reduce_size = 1.0 for i in norm_axis: reduce_size *= x.shape[i] # set some constant val. + # 计算epsilon的值 eps = graph_builder.value(x.dtype, epsilon) + # 计算-0.5的值 const_neg_half = graph_builder.value(x.dtype, -0.5) + # 计算-2.0的值 const_neg_two = graph_builder.value(x.dtype, -2.0) + # 计算2.0的值 const_two = graph_builder.value(x.dtype, 2.0) + # 计算-1.0的值 const_neg_one = graph_builder.value(x.dtype, -1.0) + # 计算mean_cof的值 mean_cof = graph_builder.value(x.dtype, (1.0 / reduce_size)) # cal dg db + # 计算方差和eps的和 var_eps = graph_builder.emit('Add', [variance, eps]) + # 计算方差和eps的和的对数 var_eps_log = graph_builder.emit('Log', [var_eps]) + # 计算方差和eps的对数乘以-0.5 var_eps_mul = graph_builder.emit('Mul', [var_eps_log, const_neg_half]) + # 计算方差和eps的对数乘以-0.5的指数 rsqrt_var_eps = graph_builder.emit('Exp', [var_eps_mul]) + # 计算x和mean的差 + # 计算输入x减去均值 x_sub_mean = graph_builder.emit('Sub', [x, mean]) + # 计算x减去均值乘以rsqrt_var_eps x_sub_mean_mul_rsqrt_var_eps = graph_builder.emit('Mul', [rsqrt_var_eps, x_sub_mean]) + # 计算dy乘以x减去均值乘以rsqrt_var_eps dg_mul = graph_builder.emit('Mul', [dy, x_sub_mean_mul_rsqrt_var_eps]) + # 计算dg,对dg_mul进行求和,reduce_axis为param_axis,keep_dims为False dg = graph_builder.emit('ReduceSum', [dg_mul], attrs={'reduce_axis': param_axis, 'keep_dims': False}) + # 计算db,对dy进行求和,reduce_axis为param_axis,keep_dims为False db = graph_builder.emit('ReduceSum', [dy], attrs={'reduce_axis': param_axis, 'keep_dims': False}) # pd_var + # 计算tmp_var_eps tmp_var_eps = graph_builder.emit('Mul', [rsqrt_var_eps, rsqrt_var_eps]) + # 计算r_tmp_var_eps r_tmp_var_eps = graph_builder.emit('Mul', [rsqrt_var_eps, tmp_var_eps]) + # 计算dy_mul_gamma dy_mul_gamma = graph_builder.emit('Mul', [dy, gamma]) + # 计算tmp_mul tmp_mul = graph_builder.emit('Mul', [dy_mul_gamma, x_sub_mean]) + # 计算padvar_mul1 padvar_mul1 = graph_builder.emit('ReduceSum', [tmp_mul], attrs={'reduce_axis': norm_axis, 'keep_dims': True}) + # 计算padvar_mul3 padvar_mul3 = graph_builder.emit('Mul', [padvar_mul1, r_tmp_var_eps]) + # 计算pd_var pd_var = graph_builder.emit('Mul', [padvar_mul3, const_neg_half]) # pd_mean + # 计算pdmean1_sum,使用ReduceSum函数,输入为dy_mul_gamma,归约轴为norm_axis,保持维度为True pdmean1_sum = graph_builder.emit('ReduceSum', [dy_mul_gamma], attrs={'reduce_axis': norm_axis, 'keep_dims': True}) + # 计算neg_rsqrt_var_eps,使用Mul函数,输入为rsqrt_var_eps和const_neg_one neg_rsqrt_var_eps = graph_builder.emit('Mul', [rsqrt_var_eps, const_neg_one]) + # 计算pd_mean_1,使用Mul函数,输入为neg_rsqrt_var_eps和pdmean1_sum pd_mean_1 = graph_builder.emit('Mul', [neg_rsqrt_var_eps, pdmean1_sum]) + # 计算pdmean2_mul1,使用Mul函数,输入为const_neg_two和x_sub_mean pdmean2_mul1 = graph_builder.emit('Mul', [const_neg_two, x_sub_mean]) + # 计算pdmean2_sum,使用ReduceSum函数,输入为pdmean2_mul1,归约轴为norm_axis,保持维度为True pdmean2_sum = graph_builder.emit('ReduceSum', [pdmean2_mul1], attrs={'reduce_axis': norm_axis, 'keep_dims': True}) + # 计算pdmean2_mul3,使用Mul函数,输入为pdmean2_sum和mean_cof pdmean2_mul3 = graph_builder.emit('Mul', [pdmean2_sum, mean_cof]) + # 计算pd_mean_2,使用Mul函数,输入为pdmean2_mul3和pd_var pd_mean_2 = graph_builder.emit('Mul', [pdmean2_mul3, pd_var]) + # 计算pd_mean,使用Add函数,输入为pd_mean_1和pd_mean_2 pd_mean = graph_builder.emit('Add', [pd_mean_1, pd_mean_2]) # cal dx + # 计算pd_x_1 pd_x_1 = graph_builder.emit('Mul', [dy_mul_gamma, rsqrt_var_eps]) + # 计算pdx2_mul pdx2_mul = graph_builder.emit('Mul', [pd_var, x_sub_mean]) + # 计算pdx2_mul_two pdx2_mul_two = graph_builder.emit('Mul', [pdx2_mul, const_two]) + # 计算pd_x_2 pd_x_2 = graph_builder.emit('Mul', [pdx2_mul_two, mean_cof]) + # 计算pd_x_3 pd_x_3 = graph_builder.emit('Mul', [pd_mean, mean_cof]) + # 计算dx_tmp dx_tmp = graph_builder.emit('Add', [pd_x_1, pd_x_2]) + # 计算dx dx = graph_builder.emit('Add', [dx_tmp, pd_x_3]) + # 如果处理器为aicore且原始数据类型为float16,则将dx、dg、db转换为float16 if processor == 'aicore' and ori_dtype == 'float16': dx = graph_builder.emit('Cast', [dx], attrs={'dst_type': 'float16'}) dg = graph_builder.emit('Cast', [dg], attrs={'dst_type': 'float16'}) db = graph_builder.emit('Cast', [db], attrs={'dst_type': 'float16'}) + # 返回dx、dg、db return dx, dg, db diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/logsoftmax.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/logsoftmax.py index 27aa8035..691f4780 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/logsoftmax.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/logsoftmax.py @@ -23,24 +23,49 @@ class LogSoftmax(Expander): """LogSoftmax expander""" def _expand(self, graph_builder): + """ + 对输入数据进行Softmax归一化。 + + Args: + graph_builder (GraphBuilder): 图构建器对象,用于构建计算图。 + + Returns: + Tensor: Softmax归一化后的结果。 + + """ + # 获取输入数据 input_x = self.inputs[0] + # 获取轴参数 axis = self.attrs['axis'] + # 获取处理器类型 processor = self.processor + # 如果轴参数是整数,则将其转换为元组 if isinstance(axis, int): axis = (axis,) + # 获取输入数据的原始数据类型 ori_dtype = input_x.dtype + # 如果原始数据类型不是float16且处理器类型是aicore,则将输入数据转换为float16 if ori_dtype != "float16" and processor == "aicore": input_x_f16 = graph_builder.emit('Cast', [input_x], attrs={'dst_type': 'float16'}) + # 对转换后的数据进行ReduceMax操作 max_x_f16 = graph_builder.emit('ReduceMax', [input_x_f16], attrs={'reduce_axis': axis, 'keep_dims': True}) + # 将ReduceMax操作的结果转换回原始数据类型 max_x = graph_builder.emit('Cast', [max_x_f16], attrs={'dst_type': ori_dtype}) else: + # 对输入数据进行ReduceMax操作 max_x = graph_builder.emit('ReduceMax', [input_x], attrs={'reduce_axis': axis, 'keep_dims': True}) + # 计算输入数据与ReduceMax操作结果的差值 data_sub = graph_builder.emit('Sub', [input_x, max_x]) + # 计算差值的指数 data_exp = graph_builder.emit('Exp', [data_sub]) + # 对指数结果进行ReduceSum操作 data_expsum = graph_builder.emit('ReduceSum', [data_exp], attrs={'reduce_axis': axis, 'keep_dims': True}) + # 计算ReduceSum结果的log log_expsum = graph_builder.emit('Log', [data_expsum]) + # 计算差值与log的差值 result = graph_builder.emit('Sub', [data_sub, log_expsum]) + # 返回结果 return result diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/logsoftmax_grad.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/logsoftmax_grad.py index 9a075d7c..7af59ed0 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/logsoftmax_grad.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/logsoftmax_grad.py @@ -23,14 +23,32 @@ class LogSoftmaxGrad(Expander): """LogSoftmaxGrad expander""" def _expand(self, graph_builder): + """ + 对输入进行扩展操作。 + + Args: + graph_builder (GraphBuilder): 图构建器对象。 + + Returns: + Tensor: 扩展操作的结果。 + + """ + # 获取输入的logits和dy input_logits, input_dy = self.inputs + # 获取axis参数 axis = self.attrs['axis'] + # 如果axis是整数,则将其转换为元组 if isinstance(axis, int): axis = (axis,) + # 计算softmax softmax = graph_builder.emit('Exp', [input_logits]) + # 计算dy的sum dy_sum = graph_builder.emit('ReduceSum', [input_dy], attrs={'reduce_axis': axis, 'keep_dims': True}) + # 计算softmax和dy_sum的乘积 mul_result = graph_builder.emit('Mul', [softmax, dy_sum]) + # 计算input_dy和mul_result的差 result = graph_builder.emit('Sub', [input_dy, mul_result]) + # 返回结果 return result diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/matmul.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/matmul.py index 8af69e0b..1d888f58 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/matmul.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/matmul.py @@ -25,48 +25,139 @@ class MatMul(Expander): """ def __init__(self, expand_info): + """ + 初始化MatMul类实例。 + + Args: + expand_info (dict): 扩展信息字典,包含操作所需的额外信息。 + + Attributes: + transpose_a (bool): 是否对矩阵A进行转置。 + transpose_b (bool): 是否对矩阵B进行转置。 + left_format (str): 矩阵A的数据格式。 + right_format (str): 矩阵B的数据格式。 + shape_a (tuple): 矩阵A的形状。 + shape_b (tuple): 矩阵B的形状。 + + """ + # 调用父类的初始化方法 super(MatMul, self).__init__(expand_info) + # 获取transpose_a属性 self.transpose_a = self.attrs['transpose_a'] + # 获取transpose_b属性 self.transpose_b = self.attrs['transpose_b'] + # 获取left_format属性 self.left_format = self.attrs['left_format'] + # 获取right_format属性 self.right_format = self.attrs['right_format'] + # 获取输入A的shape self.shape_a = self.inputs[0]['shape'] + # 获取输入B的shape self.shape_b = self.inputs[1]['shape'] def _optimize_to_mul(self): + """ + 检查是否可以用乘法(mul)替换矩阵乘法(matmul) + + Args: + 无 + + Returns: + bool: 如果可以用乘法替换矩阵乘法,返回True;否则返回False。 + + """ """check if matmul can be replace by mul""" + # 如果处理器不是'aicore',或者左格式或右格式不是默认格式,则返回False if self.processor != 'aicore' or self.left_format != DF.DEFAULT or self.right_format != DF.DEFAULT: return False + # 如果transpose_a为True,则k_a为shape_a的倒数第二个维度,否则为shape_a的倒数第一个维度 k_a = self.shape_a[-2] if self.transpose_a else self.shape_a[-1] + # 如果transpose_b为True,则k_b为shape_b的倒数第一个维度,否则为shape_b的倒数第二个维度 k_b = self.shape_b[-1] if self.transpose_b else self.shape_b[-2] + # 如果k_a或k_b不等于1,则返回False if k_a != 1 or k_b != 1: return False + # 否则返回True return True def _check(self): + """ + 检查输入个数是否满足矩阵乘法的要求。 + + Args: + 无 + + Returns: + 无 + + Raises: + GKException: 如果输入的个数小于2,则抛出GKException异常,提示信息为 "For 'MatMul', inputs number should bigger than 1, but got {}.",其中{}为输入的个数。 + + """ + # 获取输入的个数 input_num = len(self.inputs) + # 如果输入的个数小于2,抛出异常 if input_num < 2: raise GKException("For 'MatMul', inputs number should bigger than 1, but got {}.".format(input_num)) def _expand(self, graph_builder): + """ + 将MatMul或BatchMatMul操作替换为Mul操作。 + + Args: + graph_builder (GraphBuilder): 图构建器对象。 + + Returns: + Node: Mul操作的结果节点。 + + Raises: + GKException: 如果不需要将MatMul/BatchMatMul替换为Mul操作,则引发异常。 + + """ + # 定义一个函数,用于转置shape def transpose(shape): + """ + 将给定的shape进行转置操作。 + + Args: + shape (tuple): 输入的shape,为一个元组,表示多维数组的形状。 + + Returns: + list: 转置后的shape,以列表形式返回。 + + """ + # 将shape转换为列表 trans_shape = list(shape) + # 将shape的倒数第二个元素和倒数第一个元素交换位置 trans_shape[-2] = shape[-1] trans_shape[-1] = shape[-2] + # 返回转置后的shape return trans_shape + # 如果不需要优化为乘法,则抛出异常 if not self._optimize_to_mul(): raise GKException("MatMul/BatchMatMul do not need to be replaced by Mul") # Matmul is replaced by Mul([b m k], [b k n]) when k==1 + # 获取输入a input_a = self.inputs[0] + # 获取输入b input_b = self.inputs[1] + # 如果transpose_a为True,则对输入a进行转置 if self.transpose_a: + # 获取输入a的转置形状 shape_a_trans = transpose(self.shape_a) + # 对输入a进行转置 input_a = graph_builder.emit('Reshape', [input_a], attrs={'shape': shape_a_trans}) + # 如果transpose_b为True,则对输入b进行转置 if self.transpose_b: + # 获取输入b的转置形状 shape_b_trans = transpose(self.shape_b) + # 对输入b进行转置 input_b = graph_builder.emit('Reshape', [input_b], attrs={'shape': shape_b_trans}) + # 对输入a和输入b进行乘法运算 result = graph_builder.emit('Mul', [input_a, input_b]) + # 如果dst_type在attrs中,并且输入a的数据类型与dst_type不同,则对结果进行类型转换 if 'dst_type' in self.attrs and self.inputs[0].dtype != self.attrs['dst_type']: + # 对结果进行类型转换 result = graph_builder.emit('Cast', [result], attrs={'dst_type': self.attrs['dst_type']}) return result diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/maximum_grad.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/maximum_grad.py index d29989cb..0e62e7b7 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/maximum_grad.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/maximum_grad.py @@ -23,35 +23,76 @@ class MaximumGrad(Expander): """MaximumGrad expander""" def _check(self): + """ + 检查MaximumGrad的属性是否符合要求。 + + Args: + 无 + + Returns: + 返回父类的检查结果。 + + Raises: + GKException: 当 'grad_x' 和 'grad_y' 的值都为 False 时抛出异常。 + + """ + # 如果attr 'grad_x'和'grad_y'的值都为False,则抛出异常 if not self.attrs.get('grad_x', True) and not self.attrs.get('grad_y', True): raise GKException("For 'MaximumGrad', value of attr 'grad_x' and 'grad_y' should be False, but got {} and " "{}".format(self.attrs.get('grad_x'), self.attrs.get('grad_y'))) + # 调用父类的方法 return super()._check() def _expand(self, graph_builder): + """ + 根据输入计算梯度,并返回两个梯度结果。 + + Args: + graph_builder (GraphBuilder): 图构建器对象,用于构建计算图。 + + Returns: + tuple: 包含两个梯度结果的元组,第一个元素为对输入x的梯度,第二个元素为对输入y的梯度。 + + """ + # 获取输入的x、y和dout input_x, input_y, input_dout = self.inputs + # 比较x和y的大小,返回一个布尔值 ge_result = graph_builder.emit('GreaterEqual', [input_x, input_y]) + # 将布尔值转换为与x相同的类型 ge_result = graph_builder.emit('Cast', [ge_result], attrs={'dst_type': input_x.dtype}) + # 计算dx,即x的梯度 dx = graph_builder.emit('Mul', [ge_result, input_dout]) + # 计算dy,即y的梯度 dy = graph_builder.emit('Sub', [input_dout, dx]) + # 获取dx和dy的reduce轴 reduce_axis_x = MinimumGrad.get_reduce_axis(input_x.shape, dx.shape) reduce_axis_y = MinimumGrad.get_reduce_axis(input_y.shape, dy.shape) + # 如果dx有reduce轴 if reduce_axis_x: + # 对dx进行求和 dx_reduce = graph_builder.emit('ReduceSum', [dx], attrs={'reduce_axis': reduce_axis_x, 'keep_dims': False}) + # 如果dx_reduce的形状与input_x的形状不同,则进行reshape if dx_reduce.shape != input_x.shape: dx_out = graph_builder.emit('Reshape', [dx_reduce], attrs={'shape': input_x.shape}) + # 否则,dx_out等于dx_reduce else: dx_out = dx_reduce + # 否则,dx_out等于dx else: dx_out = dx + # 如果dy有reduce轴 if reduce_axis_y: + # 对dy进行求和 dy_reduce = graph_builder.emit('ReduceSum', [dy], attrs={'reduce_axis': reduce_axis_y, 'keep_dims': False}) + # 如果dy_reduce的形状与input_y的形状不同,则进行reshape if dy_reduce.shape != input_y.shape: dy_out = graph_builder.emit('Reshape', [dy_reduce], attrs={'shape': input_y.shape}) + # 否则,dy_out等于dy_reduce else: dy_out = dy_reduce + # 否则,dy_out等于dy else: dy_out = dy diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/minimum_grad.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/minimum_grad.py index 8772fb0e..0a98d1df 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/minimum_grad.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/minimum_grad.py @@ -22,59 +22,117 @@ class MinimumGrad(Expander): """MinimumGrad expander""" def _check(self): + """ + 检查MinimumGrad类的属性是否满足要求。 + + Args: + 无 + + Returns: + bool: 如果属性符合要求,则返回True,否则抛出GKException异常。 + + Raises: + GKException: 如果MinimumGrad类的属性'grad_x'和'grad_y'均为False,则抛出此异常。 + + """ + # 如果attr 'grad_x'和'grad_y'的值都为False,则抛出异常 if not self.attrs.get('grad_x', True) and not self.attrs.get('grad_y', True): raise GKException("For 'MinimumGrad', value of attr 'grad_x' and 'grad_y' should be False, but got {} and " "{}".format(self.attrs.get('grad_x'), self.attrs.get('grad_y'))) + # 调用父类的方法 return super(MinimumGrad, self)._check() def _expand(self, graph_builder): + """ + 计算两个输入的梯度。 + + Args: + graph_builder (GraphBuilder): 图构建器对象,用于在图中执行操作。 + + Returns: + tuple: 包含两个梯度结果的元组。 + + """ + # 输入参数 input_x, input_y, input_dout = self.inputs - le_result = graph_builder.emit('LessEqual', [input_x, input_y]) - le_result = graph_builder.emit('Cast', [le_result], attrs={'dst_type': input_x.dtype}) - dx = graph_builder.emit('Mul', [le_result, input_dout]) - dy = graph_builder.emit('Sub', [input_dout, dx]) + # 执行 LessEqual 操作 + le_result = graph_builder.emit('LessEqual', [input_x, input_y]) # 执行 LessEqual 操作 + # 将结果转换为与 input_x 相同的数据类型 + le_result = graph_builder.emit('Cast', [le_result], attrs={'dst_type': input_x.dtype}) # 将结果转换为与 input_x 相同的数据类型 + # 执行 Mul 操作,将 le_result 和 input_dout 相乘 + dx = graph_builder.emit('Mul', [le_result, input_dout]) # 执行 Mul 操作,将 le_result 和 input_dout 相乘 + # 执行 Sub 操作,用 input_dout 减去 dx + dy = graph_builder.emit('Sub', [input_dout, dx]) # 执行 Sub 操作,用 input_dout 减去 dx + # 对于 minimumgrad 操作,输出形状应与输入形状相同, + # 但某些元素级操作可能会广播输入形状, + # 导致输出形状不等于原始输入形状,因此需要减少输出来使它们相等 # for minimumgrad op, output_shape should be equal to input_shape, # but some elementwise operating may broadcast input_shape # then output_shape not equal to original input_shape, so need to reduce output to let them equal - reduce_axis_x = self.get_reduce_axis(input_x.shape, dx.shape) - reduce_axis_y = self.get_reduce_axis(input_y.shape, dy.shape) + reduce_axis_x = self.get_reduce_axis(input_x.shape, dx.shape) # 获取 x 的减少轴 + reduce_axis_y = self.get_reduce_axis(input_y.shape, dy.shape) # 获取 y 的减少轴 if reduce_axis_x: - dx_reduce = graph_builder.emit('ReduceSum', [dx], attrs={'reduce_axis': reduce_axis_x, 'keep_dims': False}) + # 如果存在减少轴,执行 ReduceSum 操作 + dx_reduce = graph_builder.emit('ReduceSum', [dx], attrs={'reduce_axis': reduce_axis_x, 'keep_dims': False}) # 执行 ReduceSum 操作 if dx_reduce.shape != input_x.shape: - dx_out = graph_builder.emit('Reshape', [dx_reduce], attrs={'shape': input_x.shape}) + # 如果减少后的形状不等于原始输入形状,则进行 Reshape 操作 + dx_out = graph_builder.emit('Reshape', [dx_reduce], attrs={'shape': input_x.shape}) # 如果减少后的形状不等于原始输入形状,则进行 Reshape 操作 else: - dx_out = dx_reduce + dx_out = dx_reduce # 否则直接使用减少后的结果 else: - dx_out = dx + dx_out = dx # 如果没有减少轴,则直接使用 dx if reduce_axis_y: - dy_reduce = graph_builder.emit('ReduceSum', [dy], attrs={'reduce_axis': reduce_axis_y, 'keep_dims': False}) + # 如果存在减少轴,执行 ReduceSum 操作 + dy_reduce = graph_builder.emit('ReduceSum', [dy], attrs={'reduce_axis': reduce_axis_y, 'keep_dims': False}) # 执行 ReduceSum 操作 if dy_reduce.shape != input_y.shape: - dy_out = graph_builder.emit('Reshape', [dy_reduce], attrs={'shape': input_y.shape}) + # 如果减少后的形状不等于原始输入形状,则进行 Reshape 操作 + dy_out = graph_builder.emit('Reshape', [dy_reduce], attrs={'shape': input_y.shape}) # 如果减少后的形状不等于原始输入形状,则进行 Reshape 操作 else: - dy_out = dy_reduce + dy_out = dy_reduce # 否则直接使用减少后的结果 else: - dy_out = dy + dy_out = dy # 如果没有减少轴,则直接使用 dy - # output two results, regardless of grad_x and grad_y + # 输出两个结果, return dx_out, dy_out @staticmethod def get_reduce_axis(original_shape, broadcast_shape): + """ + 计算最终输出形状的归约轴。 + + Args: + original_shape (tuple of int): 原始形状,一个包含整数的元组。 + broadcast_shape (tuple of int): 广播形状,一个包含整数的元组。 + + Returns: + list of int: 归约轴列表,表示在最终输出形状中需要归约的轴索引。 + + Raises: + ValueError: 如果original_shape的长度大于broadcast_shape的长度,或者original_shape和broadcast_shape无法广播。 + + """ """compute reduce axis for final output_shape""" + # 如果original_shape的长度大于broadcast_shape的长度 if len(original_shape) > len(broadcast_shape): raise ValueError("For 'MinimumGrad', the length of original_shape should be less than or equal to the " "length of broadcast_shape, but got {} and {}".format(original_shape, broadcast_shape)) + # 创建一个tmp_shape列表,长度为broadcast_shape的长度,前面填充1,后面填充original_shape的元素 tmp_shape = [1] * (len(broadcast_shape) - len(original_shape)) + original_shape reduce_axis = [] + # 遍历tmp_shape中的每个元素 for idx, _ in enumerate(tmp_shape): + # 如果tmp_shape中的元素与broadcast_shape中的对应元素不相等 if tmp_shape[idx] != broadcast_shape[idx]: + # 如果tmp_shape中的元素为1 if tmp_shape[idx] == 1: + # 将当前索引添加到reduce_axis列表中 reduce_axis.append(idx) else: + # 抛出异常,表示original_shape和broadcast_shape无法广播 raise ValueError("For 'MinimumGrad', original_shape {} and broadcast_shape {} can not broadcast." .format(original_shape, broadcast_shape)) return reduce_axis diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/oneslike.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/oneslike.py index e4a4a4d0..7b91cb2a 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/oneslike.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/oneslike.py @@ -20,7 +20,24 @@ class OnesLike(Expander): """OnesLike expander""" def _expand(self, graph_builder): + """ + 将输入张量扩展至指定形状。 + + Args: + graph_builder: 图构建器对象,用于构建图结构。 + + Returns: + 扩展后的张量。 + + """ + # 获取输入张量 input_x = self.inputs[0] + + # 创建一个值为1的常量,数据类型与输入张量相同 const_one = graph_builder.value(input_x.dtype, 1) + + # 使用BroadcastTo操作将常量扩展至输入张量的形状 result = graph_builder.emit('BroadcastTo', [const_one], attrs={'shape': input_x.shape}) + + # 返回扩展后的张量 return result diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/reduce_mean.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/reduce_mean.py index abb2df57..2258135a 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/reduce_mean.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/reduce_mean.py @@ -22,22 +22,41 @@ from ._utils import Expander, ExpanderInfoValidator as VLD class ReduceMean(Expander): """ReduceMean expander""" - def _expand(self, graph_builder): +def _expand(self, graph_builder): + """ + 对输入张量进行扩展操作。 + + Args: + graph_builder (GraphBuilder): 图构建器对象。 + + Returns: + Tensor: 扩展操作后的张量。 + + """ + # 获取输入张量 x = self.inputs[0] + # 获取扩展操作的轴 axis = self.attrs['axis'] + # 获取是否保持维度 keep_dims = self.attrs['keep_dims'] + # 如果轴不是元组或列表,则将其转换为元组 if not isinstance(axis, (tuple, list)): axis = (axis,) + # 如果轴为空,则将其设置为张量的所有维度 elif not axis: axis = list(range(len(x.shape))) + # 计算缩减的大小 reduce_size = 1.0 for idx in axis: reduce_size *= x.shape[idx] + # 创建一个与输入张量相同数据类型的值,值为缩减的大小 reduce_size_value = graph_builder.value(x.dtype, reduce_size) + # 沿指定轴对输入张量进行求和操作 sum_x = graph_builder.emit('ReduceSum', [x], attrs={'reduce_axis': axis, 'keep_dims': keep_dims}) + # 将求和结果除以缩减的大小,得到扩展后的张量 result = graph_builder.emit('RealDiv', [sum_x, reduce_size_value]) return result diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/relu_grad.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/relu_grad.py index d2e7f740..1fe53f22 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/relu_grad.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/relu_grad.py @@ -21,12 +21,28 @@ class ReluGrad(Expander): """ReLU expander""" def _expand(self, graph_builder): + """ + 在指定的图构建器中扩展当前节点。 + + Args: + graph_builder (GraphBuilder): 图构建器实例,用于在图中生成新的节点。 + + Returns: + Tensor: 返回计算后的结果张量。 + + """ + # 获取输入张量 input_x = self.inputs[0] input_y = self.inputs[1] + # 生成一个与input_y相同数据类型的0值张量 + # 生成一个与input_y相同数据类型的0值张量 const_zero = graph_builder.value(input_y.dtype, 0) + # 判断input_y是否大于0,生成布尔张量 ge_result = graph_builder.emit('Greater', [input_y, const_zero]) + # 将布尔张量转换为与input_x相同数据类型的张量 ge_result = graph_builder.emit('Cast', [ge_result], attrs={'dst_type': input_x.dtype}) + # 将转换后的张量与input_x相乘 result = graph_builder.emit('Mul', [ge_result, input_x]) return result diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/sigmoid_cross_entropy_with_logits.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/sigmoid_cross_entropy_with_logits.py index c2c9d20e..c8ed536d 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/sigmoid_cross_entropy_with_logits.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/sigmoid_cross_entropy_with_logits.py @@ -21,21 +21,46 @@ class SigmoidCrossEntropyWithLogits(Expander): """SigmoidCrossEntropyWithLogits expander""" def _expand(self, graph_builder): + """ + 计算sigmoid交叉熵损失。 + + Args: + graph_builder: 图构建器对象,用于构建计算图。 + + Returns: + 计算得到的sigmoid交叉熵损失值。 + + """ logits, labels = self.inputs + # 计算 logits 和 labels 的 sigmoid_cross_entropy_with_logits # Calculate sigmoid_cross_entropy_with_logits(logits, labels) - # formula of sigmoid_cross_entropy_with_logits is: - # -(labels * log(sigmoid(logits)) + (1 - labels) * log(1 - sigmoid(logits))) - # To ensure stability and avoid overflow, the formula equal to : - # max(logits, 0) - logits * labels + log(1 + exp(-abs(logits))) + # sigmoid_cross_entropy_with_logits 的公式为: + # -(labels * log(sigmoid(logits)) + (1 - labels) * log(1 - sigmoid(logits))) + # 为了确保稳定性并避免溢出,该公式等价于: + # max(logits, 0) - logits * labels + log(1 + exp(-abs(logits))) + + # 创建一个值为 1.0 的常量 const_one = graph_builder.value(logits.dtype, 1.0) + # 创建一个值为 0.0 的常量 const_zero = graph_builder.value(logits.dtype, 0.0) + + # 计算 logits 和 0 的最大值 max_logits = graph_builder.emit('Maximum', [logits, const_zero]) + # 计算 logits 和 labels 的乘积 logits_mul_labels = graph_builder.emit('Mul', [logits, labels]) + # 计算 logits 的绝对值 abs_logits = graph_builder.emit('Abs', [logits]) + # 计算 logits 的负值 neg_abs_logits = graph_builder.emit('Neg', [abs_logits]) + # 计算 exp(-abs(logits)) exp_neg_abs_logits = graph_builder.emit('Exp', [neg_abs_logits]) + # 计算 1 + exp(-abs(logits)) one_add_exp_neg_abs_logits = graph_builder.emit('Add', [const_one, exp_neg_abs_logits]) + # 计算 log(1 + exp(-abs(logits))) log_one_add_exp_neg_abs_logits = graph_builder.emit('Log', [one_add_exp_neg_abs_logits]) + # 计算 max(logits, 0) - logits * labels res_tmp = graph_builder.emit('Sub', [max_logits, logits_mul_labels]) + # 计算最终结果 res = graph_builder.emit('Add', [res_tmp, log_one_add_exp_neg_abs_logits]) + return res diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/sigmoid_cross_entropy_with_logits_grad.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/sigmoid_cross_entropy_with_logits_grad.py index 07f9dba3..cf49d4cc 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/sigmoid_cross_entropy_with_logits_grad.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/sigmoid_cross_entropy_with_logits_grad.py @@ -21,15 +21,29 @@ class SigmoidCrossEntropyWithLogitsGrad(Expander): """SigmoidCrossEntropyWithLogitsGrad expander""" def _expand(self, graph_builder): + """ + 计算sigmoid交叉熵损失的梯度。 + + Args: + graph_builder: 图构建器对象,用于构建计算图。 + + Returns: + 计算得到的梯度值。 + + """ logits, label, dout = self.inputs - # Calculate sigmoid_cross_entropy_with_logits_grad(logits, label, dout) - # formula of sigmoid_cross_entropy_with_logits_grad is : + # 计算sigmoid_cross_entropy_with_logits_grad(logits, label, dout) + # sigmoid_cross_entropy_with_logits_grad的公式为: # (sigmoid(logits) - label) * dout + # 计算sigmoid(logits) + # Calculate sigmoid(logits) const_one = graph_builder.value(logits.dtype, 1.0) - neg_x = graph_builder.emit('Neg', [logits]) - exp_neg_x = graph_builder.emit('Exp', [neg_x]) - add_exp = graph_builder.emit('Add', [const_one, exp_neg_x]) - sigmoid_res = graph_builder.emit('RealDiv', [const_one, add_exp]) + neg_x = graph_builder.emit('Neg', [logits]) # 计算-logits + exp_neg_x = graph_builder.emit('Exp', [neg_x]) # 计算e^(-logits) + add_exp = graph_builder.emit('Add', [const_one, exp_neg_x]) # 计算1 + e^(-logits) + sigmoid_res = graph_builder.emit('RealDiv', [const_one, add_exp]) # 计算1 / (1 + e^(-logits)),即sigmoid(logits) + # 计算(sigmoid(logits) - label) sigmoid_res_sub_label = graph_builder.emit('Sub', [sigmoid_res, label]) - res = graph_builder.emit('Mul', [sigmoid_res_sub_label, dout]) + # 计算最终结果 + res = graph_builder.emit('Mul', [sigmoid_res_sub_label, dout]) # 计算(sigmoid(logits) - label) * dout return res diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/sigmoid_grad.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/sigmoid_grad.py index 56d9413a..72564754 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/sigmoid_grad.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/sigmoid_grad.py @@ -16,16 +16,31 @@ from ._utils import Expander, ExpanderInfoValidator as VLD -@VLD.check_all_formats_same +@VLD.check_all_formats_same # 定义一个SigmoidGrad类,继承自Expander类 class SigmoidGrad(Expander): """SigmoidGrad expander""" def _expand(self, graph_builder): + """ + 计算 sigmoid 函数的梯度。 + + Args: + graph_builder (GraphBuilder): 图构建器对象,用于在图中添加操作。 + + Returns: + Tensor: 计算得到的 sigmoid 梯度。 + + """ input_y, dy = self.inputs + # 计算 sigmoid_grad(y, dy) + # sigmoid_grad 的公式是: (1 - y) * y * dy # Calculate sigmoid_grad(y, dy) # formula of sigmoid_grad is : (1 - y) * y * dy const_one = graph_builder.value(input_y.dtype, 1.0) + # 1 - y one_mins_y = graph_builder.emit('Sub', [const_one, input_y]) + # y * dy y_mul_dy = graph_builder.emit('Mul', [input_y, dy]) + # (1 - y) * (y * dy) res = graph_builder.emit('Mul', [one_mins_y, y_mul_dy]) return res diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/slice.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/slice.py index 5bc265ea..fb847dd2 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/slice.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/slice.py @@ -21,15 +21,41 @@ class Slice(Expander): """Slice expander""" def _expand(self, graph_builder): + """ + 在图中扩展输入张量。 + + Args: + graph_builder (GraphBuilder): 图构建器对象。 + + Returns: + Tensor: 扩展后的输出张量。 + + """ + # 获取输入张量 input_x = self.inputs[0] + + # 获取开始索引 begin = self.attrs['begin'] + # 获取切片大小 size = self.attrs['size'] + + # 初始化结束索引列表 end = [] + # 初始化步长列表 strides = [] + + # 遍历每个维度,计算结束索引和步长 for i, begin_idx in enumerate(begin): + # 步长设置为1 strides.append(1) + # 计算结束索引 end.append(begin_idx + size[i]) + + # 创建一个新的张量作为输出 output = graph_builder.tensor(size, input_x.dtype, input_x.data_format) + + # 执行StridedSlice操作,对输入张量进行切片 graph_builder.op('StridedSlice', output, [input_x], attrs={'begin': begin, 'end': end, 'strides': strides}) + # 返回输出张量 return output diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/softmax.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/softmax.py index 991d5e6a..9bad459e 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/softmax.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/softmax.py @@ -25,45 +25,75 @@ class Softmax(Expander): """Softmax expander""" def _expand(self, graph_builder): + """ + 计算Softmax函数值。 + + Args: + graph_builder: 图构建器对象。 + + Returns: + Softmax函数的计算结果。 + + """ + # 获取输入数据 input_x = self.inputs[0] + # 获取处理器 processor = self.processor + # 获取轴信息 axis = self.attrs['axis'] + # 获取输入数据的原始形状 ori_shape = input_x.shape + # 如果输入数据格式为FRAC_NZ,则推断其形状 if input_x.data_format == DF.FRAC_NZ: ori_shape = infer_shape_from_fractalnz(input_x.shape) + # 遍历轴信息,处理负数轴索引 for i, _ in enumerate(list(axis)): if axis[i] < 0: axis[i] += len(ori_shape) + # 获取减少维度后的原始形状 ori_reduced_shape = get_reduced_ori_shape(ori_shape, axis) + # 获取减少的轴 reduce_axis = axis + # 如果输入数据格式为FRAC_NZ,则转换轴 if input_x.data_format == DF.FRAC_NZ: reduce_axis = to_frac_z_axis(ori_shape, axis) + # 获取输入数据的原始数据类型 ori_dtype = input_x.dtype + # 如果原始数据类型不是float16且处理器为aicore,则进行类型转换 if ori_dtype != "float16" and processor == "aicore": input_x_f16 = graph_builder.emit('Cast', [input_x], attrs={'dst_type': 'float16'}) max_x_f16 = graph_builder.emit('ReduceMax', [input_x_f16], attrs={'reduce_axis': reduce_axis, 'keep_dims': True}) max_x = graph_builder.emit('Cast', [max_x_f16], attrs={'dst_type': ori_dtype}) else: + # 计算最大值 max_x = graph_builder.emit('ReduceMax', [input_x], attrs={'reduce_axis': reduce_axis, 'keep_dims': True}) + # 如果原始数据类型为float16且处理器为aicore,则进行类型转换 if ori_dtype == "float16" and processor == "aicore": max_x = graph_builder.emit('Cast', [max_x], attrs={'dst_type': "float32"}) input_x = graph_builder.emit('Cast', [input_x], attrs={'dst_type': "float32"}) + # 如果输入数据格式为FRAC_NZ,则重新调整最大值的形状 if input_x.data_format == DF.FRAC_NZ: max_x = graph_builder.emit('Reshape', [max_x], attrs={'shape': ori_reduced_shape}) + # 计算输入数据减去最大值的差值 data_sub = graph_builder.emit('Sub', [input_x, max_x]) + # 计算差值的指数 data_exp = graph_builder.emit('Exp', [data_sub]) + # 计算指数的和 data_expsum = graph_builder.emit('ReduceSum', [data_exp], attrs={'reduce_axis': reduce_axis, 'keep_dims': True}) + # 如果输入数据格式为FRAC_NZ,则重新调整指数和的形状 if input_x.data_format == DF.FRAC_NZ: data_expsum = graph_builder.emit('Reshape', [data_expsum], attrs={'shape': ori_reduced_shape}) + # 计算Softmax值 result = graph_builder.emit('RealDiv', [data_exp, data_expsum]) + # 如果原始数据类型为float16且处理器为aicore,则进行类型转换 if ori_dtype == "float16" and processor == "aicore": result = graph_builder.emit('Cast', [result], attrs={'dst_type': ori_dtype}) diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/softmax_cross_entropy_with_logits.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/softmax_cross_entropy_with_logits.py index e28e74f4..12cae367 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/softmax_cross_entropy_with_logits.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/softmax_cross_entropy_with_logits.py @@ -22,21 +22,45 @@ class SoftmaxCrossEntropyWithLogits(Expander): """SoftmaxCrossEntropyWithLogits expander""" def _expand(self, graph_builder): + """ + 计算损失值和 logits 的梯度。 + + Args: + graph_builder (GraphBuilder): 图构建器对象。 + + Returns: + Tuple[Tensor, Tensor]: 损失值和 logits 的梯度。 + + """ logits, label = self.inputs + # 计算 softmax_cross_entropy_with_logits(logits, label) + # softmax_cross_entropy_with_logits 的公式是: -reduce_sum(label * log(softmax(logits))) # Calculate softmax_cross_entropy_with_logits(logits, label) # formula of softmax_cross_entropy_with_logits is : -reduce_sum(label * log(softmax(logits))) axis = (-1,) max_x = graph_builder.emit('ReduceMax', [logits], attrs={'reduce_axis': axis, 'keep_dims': True}) + # 计算 logits 的最大值 data_sub = graph_builder.emit('Sub', [logits, max_x]) + # logits 减去最大值 data_exp = graph_builder.emit('Exp', [data_sub]) + # 对上一步结果进行指数运算 data_expsum = graph_builder.emit('ReduceSum', [data_exp], attrs={'reduce_axis': axis, 'keep_dims': True}) + # 对指数运算结果求和 data_softmax = graph_builder.emit('RealDiv', [data_exp, data_expsum]) + # 计算 softmax const_eps = graph_builder.value(logits.dtype, 0.000001) + # 定义一个极小的常数,用于防止除以零的错误 data_softmax_safety = graph_builder.emit("Maximum", [data_softmax, const_eps]) + # 确保 softmax 的值不为零 softmax_log = graph_builder.emit('Log', [data_softmax_safety]) + # 对 softmax 结果取对数 label_mul_log = graph_builder.emit('Mul', [label, softmax_log]) + # 将 label 与 softmax 的对数相乘 tmp_res = data_expsum = graph_builder.emit('ReduceSum', [label_mul_log], attrs={ 'reduce_axis': axis, 'keep_dims': False}) + # 对上一步结果进行求和 loss = graph_builder.emit('Neg', [tmp_res]) + # 计算损失值,即上一步结果的负值 dlogits = graph_builder.emit('Sub', [data_softmax, label]) + # 计算 logits 的梯度 return loss, dlogits diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/softmax_grad_ext.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/softmax_grad_ext.py index d4ccff60..d3c23a4e 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/softmax_grad_ext.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/softmax_grad_ext.py @@ -13,29 +13,48 @@ # limitations under the License. # =========================================================================== """generate json desc for SoftmaxGradExt""" -from mindspore._extends.graph_kernel.model.model import DataFormat as DF -from ._utils import Expander, ExpanderInfoValidator as VLD -from ._utils import get_reduce_axis_shape +from mindspore._extends.graph_kernel.model.model import DataFormat as DF # 导入DataFormat类 +from ._utils import Expander, ExpanderInfoValidator as VLD # 导入Expander和ExpanderInfoValidator类 +from ._utils import get_reduce_axis_shape # 导入get_reduce_axis_shape函数 -@VLD.add_format(DF.FRAC_NZ, DF.FRAC_NZ, DF.DEFAULT) +@VLD.add_format(DF.FRAC_NZ, DF.FRAC_NZ, DF.DEFAULT) # 使用ExpanderInfoValidator类添加FRAC_NZ格式 @VLD.add_format(DF.DEFAULT, DF.DEFAULT, DF.DEFAULT) @VLD.check_attrs('axis') class SoftmaxGradExt(Expander): """SoftmaxGradExt expander""" def _expand(self, graph_builder): + """ + 对输入数据进行扩展处理。 + + Args: + graph_builder (GraphBuilder): 图构建器对象。 + + Returns: + Tensor: 处理后的数据。 + + """ + # 获取输入参数 x, y, z = self.inputs + # 获取指定的轴 axis = self.attrs['axis'] + # 获取需要减少的轴和原始减少的形状 reduce_axis, ori_reduced_shape = get_reduce_axis_shape(x.shape, x.data_format, axis) + # 将x和y相乘 data_mul = graph_builder.emit('Mul', [x, y]) + # 对乘积进行求和,并保留维度 data_sum = graph_builder.emit('ReduceSum', [data_mul], attrs={'reduce_axis': reduce_axis, 'keep_dims': True, 'reduce_output_fuse': True}) + # 如果x的数据格式为FRAC_NZ,则对求和结果进行重塑 if x.data_format == DF.FRAC_NZ: data_sum = graph_builder.emit('Reshape', [data_sum], attrs={'shape': ori_reduced_shape}) + # 从x中减去求和结果 data_sub = graph_builder.emit('Sub', [x, data_sum]) + # 将减法结果与y相乘 data_mul2 = graph_builder.emit('Mul', [data_sub, y]) + # 将结果与z相乘得到最终结果 result = graph_builder.emit('Mul', [data_mul2, z]) return result diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/sqrt_grad.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/sqrt_grad.py index 9f072e7a..8d1bb0bd 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/sqrt_grad.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/sqrt_grad.py @@ -21,9 +21,24 @@ class SqrtGrad(Expander): """SqrtGrad expander""" def _expand(self, graph_builder): + """ + 计算并返回给定输入 x 的平方根的梯度。 + + Args: + graph_builder (GraphBuilder): 图构建器对象,用于构建计算图。 + + Returns: + Tensor: 返回给定输入 x 的平方根的梯度。 + + """ + # 获取输入 x 和梯度 dout # formula of sqrt_grad is dout / (2 * x) x, dout = self.inputs + # 创建一个常数值 2 const_two = graph_builder.value(x.dtype, 2) + # 计算 2 * x dividend = graph_builder.emit('Mul', [x, const_two]) + # 计算梯度:dout / (2 * x) result = graph_builder.emit('RealDiv', [dout, dividend]) + # 返回计算结果 return result diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/square_sum_all.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/square_sum_all.py index a06c1a39..f38953d8 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/square_sum_all.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/square_sum_all.py @@ -21,24 +21,57 @@ class SquareSumAll(Expander): """SquareSumAll expander""" def _check(self): + """ + 检查输入是否合法。 + + Args: + 无。 + + Returns: + 无。 + + Raises: + GKException: 如果输入的数量不等于2,则抛出GKException异常。 + + """ """check inputs""" + # 获取输入的数量 input_num = len(self.inputs) if input_num != 2: + # 如果输入的数量不等于2,则抛出异常 raise GKException("For 'SquareSumAll', the inputs number should be 2, but got {}.".format(input_num)) def _expand(self, graph_builder): + """ + 对输入的两个变量进行扩展操作。 + + Args: + graph_builder (GraphBuilder): 图构建器对象,用于在图构建过程中发射操作。 + + Returns: + tuple: 包含两个元素的元组,每个元素为扩展操作的结果。 + + """ """do expand""" + # 获取输入的两个变量 x0 = self.inputs[0] x1 = self.inputs[1] + # 获取x0的形状 ori_shape = x0.shape + # 初始化一个空列表,用于存储维度索引 axis = [] + # 遍历ori_shape,将每个维度的索引添加到axis列表中 for i, _ in enumerate(ori_shape): axis.append(i) + # 对x0进行平方运算 square_res0 = graph_builder.emit('Mul', [x0, x0]) + # 对x1进行平方运算 square_res1 = graph_builder.emit('Mul', [x1, x1]) + # 对square_res0进行求和运算,求和的维度为axis,并保持维度不变 result0 = graph_builder.emit('ReduceSum', [square_res0], attrs={'reduce_axis': axis, 'keep_dims': False}) + # 对square_res1进行求和运算,求和的维度为axis,并保持维度不变 result1 = graph_builder.emit('ReduceSum', [square_res1], attrs={'reduce_axis': axis, 'keep_dims': False}) return result0, result1 diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/square_sum_v1.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/square_sum_v1.py index 8774dcbe..67d6f28b 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/square_sum_v1.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/square_sum_v1.py @@ -25,13 +25,30 @@ class SquareSumV1(Expander): """Square expander""" def _expand(self, graph_builder): + """ + 计算输入张量的平方并沿指定轴进行求和。 + + Args: + graph_builder (GraphBuilder): 图构建器对象。 + + Returns: + Tensor: 计算得到的张量。 + + """ + # 获取输入的第一个元素 x = self.inputs[0] + # 获取属性中的axis值 axis = self.attrs['axis'] + # 获取需要reduce的axis和原始的reduced shape reduce_axis, ori_reduced_shape = get_reduce_axis_shape(x.shape, x.data_format, axis) + # 计算x的平方 square_res = graph_builder.emit('Mul', [x, x]) + # 对平方结果进行ReduceSum操作 result = graph_builder.emit('ReduceSum', [square_res], attrs={'reduce_axis': reduce_axis, 'keep_dims': False}) + # 如果数据格式为FRAC_NZ,则对结果进行Reshape操作 if x.data_format == DF.FRAC_NZ: result = graph_builder.emit('Reshape', [result], attrs={'shape': ori_reduced_shape}) + # 返回最终结果 return result diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/squared_difference.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/squared_difference.py index 316b000e..ca0dc687 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/squared_difference.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/squared_difference.py @@ -21,10 +21,24 @@ class SquaredDifference(Expander): """SquaredDifference expander""" def _expand(self, graph_builder): + """ + 根据输入的两个输入值计算并返回它们的平方差的计算结果。 + + Args: + graph_builder (GraphBuilder): 图构建器对象,用于在图中生成节点和边。 + + Returns: + Node: 计算结果节点。 + + """ + # 获取输入的第一个值 input_x = self.inputs[0] + # 获取输入的第二个值 input_y = self.inputs[1] + # 使用图构建器计算输入值的差值 sub_val = graph_builder.emit('Sub', [input_x, input_y]) + # 使用图构建器计算差值的平方 result = graph_builder.emit('Mul', [sub_val, sub_val]) return result diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/squeeze.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/squeeze.py index a5e631b5..66178c0b 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/squeeze.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/squeeze.py @@ -21,27 +21,67 @@ class Squeeze(Expander): """Squeeze expander""" def _expand(self, graph_builder): + """ + 扩展输入的维度。 + + Args: + graph_builder (GraphBuilder): 图构建器对象,用于构建图结构。 + + Returns: + Tensor: 扩展维度后的输入。 + + """ + # 获取输入的第一个元素 input_x = self.inputs[0] + + # 根据输入的shape和axis属性推断输出shape out_shape = self.infer_shape(input_x.shape, self.attrs['axis']) + + # 使用graph_builder发射Reshape操作,并设置shape属性为out_shape result = graph_builder.emit('Reshape', [input_x], attrs={'shape': out_shape}) + # 返回结果 return result @staticmethod def infer_shape(shape, axis): + """ + 根据指定的axis推断squeeze后的shape。 + + Args: + shape (list, tuple): 原始数据的shape。 + axis (int, list, tuple): 需要被squeeze的维度。如果为int,则只squeeze该维度; + 如果为list或tuple,则squeeze列表或元组中的每个维度。如果为空,则squeeze所有维度为1的维度。 + + Returns: + list: squeeze后的shape。 + + Raises: + ValueError: 如果输入的axis类型不符合要求,抛出异常。 + + """ """infer shape for squeeze""" def squeeze_axis(shape, axis): + # 如果axis为空,移除shape中所有值为1的维度 if not axis: out_shape = list(d for d in shape if d != 1) else: + # 获取shape的维度数量 ndim = len(shape) + # 移除shape中指定的axis维度 out_shape = list(shape[i] for i in range(ndim) if not (i in axis or (i - ndim) in axis)) + # 如果out_shape为空,则将其设置为[1] if not out_shape: out_shape = [1] return out_shape + + # 如果shape是列表或元组类型 if isinstance(shape, (list, tuple)): + # 如果axis是整数类型,则将其转换为列表 if isinstance(axis, int): axis = [axis] + # 如果axis是列表或元组类型,则调用squeeze_axis函数处理 if isinstance(axis, (list, tuple)): return squeeze_axis(shape, axis) + # 如果输入不符合要求,则抛出异常 raise ValueError("Invalid axis for Squeeze.") diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/tanh_grad.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/tanh_grad.py index a43c7962..60681a54 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/tanh_grad.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/tanh_grad.py @@ -21,11 +21,29 @@ class TanhGrad(Expander): """TanhGrad expander""" def _expand(self, graph_builder): + """ + 计算1减去输入值的平方后,再与输入的导数相乘。 + + Args: + graph_builder (GraphBuilder): 图构建器对象,用于构建计算图。 + + Returns: + Tensor: 计算结果,类型为Tensor。 + + """ + # 获取输入值 input_y, input_dy = self.inputs + # 创建一个值为1的常量,数据类型与input_y相同 const_one = graph_builder.value(input_y.dtype, 1) + + # 计算input_y的平方 double_y = graph_builder.emit('Mul', [input_y, input_y]) + + # 计算1减去input_y的平方 one_sub_double_y = graph_builder.emit('Sub', [const_one, double_y]) + + # 计算input_dy与1减去input_y的平方的乘积 result = graph_builder.emit('Mul', [input_dy, one_sub_double_y]) return result diff --git a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/tile.py b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/tile.py index 918b7486..9b262d45 100644 --- a/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/tile.py +++ b/src/mindspore2022/mindspore/python/mindspore/_extends/graph_kernel/expanders/tile.py @@ -25,30 +25,48 @@ class Tile(Expander): def _get_output_shape(self): """Get output shape""" + # 获取输入形状的列表 shape = list(self.inputs[0].shape) + # 获取属性"multiples"的列表 multiples = list(self.attrs["multiples"]) + # 计算"multiples"和输入形状的长度差 diff_len = len(multiples) - len(shape) + # 如果长度差小于0,抛出异常 if diff_len < 0: raise GKException("For 'Tile', dimensions of attr 'multiples' should be greater than or equal to " "dimensions of input shape, but got {} and {}".format(multiples, shape)) + # 如果长度差大于0,则扩展输入形状的列表 if diff_len > 0: for _ in range(diff_len): shape.insert(0, 1) + # 初始化输出形状的列表 output_shape = [] + # 遍历输入形状和multiples的元组 for sh, mul in list(zip(shape, multiples)): + # 如果输入形状和multiples的值都不为1,则抛出异常 if sh != 1 and mul != 1: raise GKException("For 'Tile', input shape{} and attr 'multiples'{} can not broadcast." .format(self.inputs[0].shape, multiples)) + # 计算维度 dim = sh * mul + # 将计算得到的维度添加到输出形状的列表中 output_shape.append(dim) + # 返回输出形状的列表 return output_shape def _expand(self, graph_builder): + # 获取输入的第一个元素 input_x = self.inputs[0] + + # 获取输出形状 output_shape = self._get_output_shape() + # 使用graph_builder的emit方法生成BroadcastTo操作 + # 参数为[input_x]和输出形状 result = graph_builder.emit('BroadcastTo', [input_x], attrs={'shape': output_shape}) + + # 返回结果 return result