Go自研微服务框架-参数处理

参数处理

1. 频繁创建context的优化

sync.Pool用于存储那些被分配了但是没有被使用，但是未来可能被使用的值，这样可以不用再次分配内存，提高效率。

sync.Pool大小是可伸缩的，高负载是会动态扩容，存放在池中不活跃的对象会被自动清理。

type Engine struct {
	*router
	funcMap    template.FuncMap
	HTMLRender render.HTMLRender
	pool       sync.Pool
}

func New() *Engine {
	engine := &Engine{
		router:     &router{},
		funcMap:    nil,
		HTMLRender: render.HTMLRender{},
	}
	engine.pool.New = func() any {
		return engine.allocateContext()
	}
	return engine
}
func (e *Engine) allocateContext() any {
	return &Context{engin: e}
}

func (e *Engine) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ctx := e.pool.Get().(*Context)
	ctx.W = w
	ctx.R = r
	e.handleHttpRequest(ctx)
	e.pool.Put(ctx)
}

func (e *Engine) handleHttpRequest(ctx *Context) {

	groups := e.router.groups
	for _, g := range groups {
		uri := ctx.R.RequestURI
		routerName := SubStringLast(uri, "/"+g.groupName)
		node := g.treeNode.Get(routerName)
		if node == nil || !node.isEnd {
			ctx.W.WriteHeader(http.StatusNotFound)
			fmt.Fprintln(ctx.W, ctx.R.RequestURI+" not found")
			return
		}

		methodHandle, ok := g.handlerMap[routerName]
		if ok {
			_, ok = methodHandle[ANY]
			if ok {
				g.methodHandle(ANY, methodHandle, ctx)
				return
			}
			_, ok = methodHandle[ctx.R.Method]
			if ok {
				g.methodHandle(ctx.R.Method, methodHandle, ctx)
				return
			}
			ctx.W.WriteHeader(http.StatusMethodNotAllowed)
			fmt.Fprintln(ctx.W, ctx.R.Method+" not allowed")
		}
	}
}

2. query参数

首先我们来处理query参数，比如：http://xxx.com/user/add?id=1&age=20&username=张三

记得将路由的URL匹配改为：uri := ctx.R.URL.Path

type Context struct {
	W          http.ResponseWriter
	R          *http.Request
	engin      *Engine
	queryCache url.Values
}

func (c *Context) GetDefaultQuery(key string, defaultValue string) string {
	values, ok := c.GetQueryArray(key)
	if !ok {
		return defaultValue
	} else {
		return values[0]
	}
}


func (c *Context) GetQuery(key string) string {
	c.initQueryCache()
	return c.queryCache.Get(key)
}

func (c *Context) QueryArray(key string) (values []string) {
	c.initQueryCache()
	values, _ = c.queryCache[key]
	return
}

func (c *Context) GetQueryArray(key string) (values []string, ok bool) {
	c.initQueryCache()
	values, ok = c.queryCache[key]
	return
}

func (c *Context) initQueryCache() {
	if c.queryCache == nil {
		if c.R != nil {
			c.queryCache = c.R.URL.Query()
		} else {
			c.queryCache = url.Values{}
		}
	}
}

2.1 map类型参数

类似于http://localhost:8080/queryMap?user[id]=1&user[name]=张三

func (c *Context) QueryMap(key string) (dicts map[string]string) {
	dicts, _ = c.GetQueryMap(key)
	return
}

func (c *Context) GetQueryMap(key string) (map[string]string, bool) {
	c.initQueryCache()
	return c.get(c.queryCache, key)
}

func (c *Context) get(m map[string][]string, key string) (map[string]string, bool) {
	//user[id]=1&user[name]=张三
	dicts := make(map[string]string)
	exist := false
	for k, value := range m {
		if i := strings.IndexByte(k, '['); i >= 1 && k[0:i] == key {
			if j := strings.IndexByte(k[i+1:], ']'); j >= 1 {
				exist = true
				dicts[k[i+1:][:j]] = value[0]
			}
		}
	}
	return dicts, exist
}

3. Post表单参数

获取表单参数我们借助 http.Request.PostForm

Form属性包含了post表单和url后面跟的get参数。

PostForm属性只包含了post表单参数。

func (c *Context) initFormCache() {
	if c.formCache == nil {
		c.formCache = make(url.Values)
		req := c.R
		if err := req.ParseMultipartForm(defaultMultipartMemory); err != nil {
			if !errors.Is(err, http.ErrNotMultipart) {
				log.Println(err)
			}
		}
		c.formCache = c.R.PostForm
	}
}

func (c *Context) GetPostForm(key string) (string, bool) {
	if values, ok := c.GetPostFormArray(key); ok {
		return values[0], ok
	}
	return "", false
}

func (c *Context) PostFormArray(key string) (values []string) {
	values, _ = c.GetPostFormArray(key)
	return
}

func (c *Context) GetPostFormArray(key string) (values []string, ok bool) {
	c.initFormCache()
	values, ok = c.formCache[key]
	return
}

func (c *Context) GetPostFormMap(key string) (map[string]string, bool) {
	c.initFormCache()
	return c.get(c.formCache, key)
}

func (c *Context) PostFormMap(key string) (dicts map[string]string) {
	dicts, _ = c.GetPostFormMap(key)
	return
}

4. 文件参数

借助http.Request.FormFile

func (c *Context) FormFile(name string) (*multipart.FileHeader, error) {
	req := c.R
	if err := req.ParseMultipartForm(defaultMultipartMemory); err != nil {
		return nil, err
	}
	file, header, err := req.FormFile(name)
	if err != nil {
		return nil, err
	}
	err = file.Close()
	if err != nil {
		return nil, err
	}
	return header, nil
}

g.Post("/file", func(ctx *msgo.Context) {
		file, err := ctx.FormFile("file")
		if err != nil {
			log.Println(err)
		}
		src, err := file.Open()
		defer src.Close()
		if err != nil {
			log.Println(err)
		} else {
			out, err := os.Create("./upload/test.png")
			defer out.Close()
			if err != nil {
				log.Println(err)
			} else {
				io.Copy(out, src)
			}
		}
	})

一般会有将文件存储的需求，可以将上述的代码也提取为一个方法

func (c *Context) SaveUploadedFile(file *multipart.FileHeader, dst string) error {
	src, err := file.Open()
	if err != nil {
		return err
	}
	defer src.Close()

	out, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer out.Close()

	_, err = io.Copy(out, src)
	return err
}

func (c *Context) MultipartForm() (*multipart.Form, error) {
	err := c.R.ParseMultipartForm(defaultMultipartMemory)
	return c.R.MultipartForm, err
}

5. json参数

现在流行前后端分离开发，前后端交互以json的形式来通信

json参数：

1. content-type: application/json
2. post传参

一般在写代码时，我们期望这样的处理方式：

	g.Post("/jsonParam", func(ctx *msgo.Context) {
		user := &User{}
		err := ctx.DealJson(user)
		if err == nil {
			ctx.JSON(http.StatusOK, user)
		} else {
			log.Println(err)
		}
	})

func (c *Context) DealJson(data any) error {
	body := c.R.Body
	if c.R == nil || body == nil {
		return errors.New("invalid request")
	}
	decoder := json.NewDecoder(body)
	return decoder.Decode(data)
}

测试：

{
    "name":"张三",
    "age":10,
    "addresses":[
        "北京",
        "杭州"
    ]
}

[
    {
        "name":"11",
        "age":10,
        "addresses":[
            "北京"
        ]
    }
]

如果想要实现参数中有的属性，但是对应的结构体没有，报错，也就是检查结构体是否有效

decoder.DisallowUnknownFields()

如果结构体有的属性，但是参数中没有，想要校验这种错误，又该如何做呢？

5.1 结构体校验

如果想要达到上述的效果，我们可以写一个校验器，专门来处理这样的校验。

为了实现这种效果，我们可以改变一下思路，先将所有的参数解析为map，然后和对应的结构体进行比对

func (c *Context) DealJson(data any) error {
	body := c.R.Body
	if c.R == nil || body == nil {
		return errors.New("invalid request")
	}
	decoder := json.NewDecoder(body)
	if c.DisallowUnknownFields {
		decoder.DisallowUnknownFields()
	}
	if c.IsValidate {
		err := validateRequireParam(data, decoder)
		if err != nil {
			return err
		}
	} else {
		err := decoder.Decode(data)
		if err != nil {
			return err
		}
	}
	return nil
}

func validateRequireParam(data any, decoder *json.Decoder) error {
	if data == nil {
		return nil
	}
	valueOf := reflect.ValueOf(data)
	if valueOf.Kind() != reflect.Pointer {
		return errors.New("no ptr type")
	}

	t := valueOf.Elem().Interface()
	of := reflect.ValueOf(t)
	switch of.Kind() {
	case reflect.Struct:
		mapData := make(map[string]interface{})
		_ = decoder.Decode(&mapData)
		for i := 0; i < of.NumField(); i++ {
			field := of.Type().Field(i)
			tag := field.Tag.Get("json")
			value := mapData[tag]
			if value == nil {
				return errors.New(fmt.Sprintf("filed [%s] is not exist", tag))
			}
		}
		marshal, _ := json.Marshal(mapData)
		_ = json.Unmarshal(marshal, data)
	}

	return nil
}

如果并不是所有字段都是必须的呢?

type User struct {
	Name      string   `xml:"name" json:"name" msgo:"required"`
	Age       int      `xml:"name" json:"age"`
	Addresses []string `json:"addresses"`
}

	for i := 0; i < of.NumField(); i++ {
			field := of.Type().Field(i)
			required := field.Tag.Get("msgo")
			tag := field.Tag.Get("json")
			value := mapData[tag]
			if value == nil && required == "required" {
				return errors.New(fmt.Sprintf("filed [%s] is required", tag))
			}
		}

5.2 切片数组校验

func (c *Context) DealJson(data any) error {
	body := c.R.Body
	if c.R == nil || body == nil {
		return errors.New("invalid request")
	}
	decoder := json.NewDecoder(body)
	if c.DisallowUnknownFields {
		decoder.DisallowUnknownFields()
	}
	if c.IsValidate {
		err := validateRequireParam(data, decoder)
		if err != nil {
			return err
		}
	} else {
		err := decoder.Decode(data)
		if err != nil {
			return err
		}
	}
	return nil
}

func validateRequireParam(data any, decoder *json.Decoder) error {
	if data == nil {
		return nil
	}
	valueOf := reflect.ValueOf(data)
	if valueOf.Kind() != reflect.Pointer {
		return errors.New("no ptr type")
	}
	t := valueOf.Elem().Interface()
	of := reflect.ValueOf(t)
	switch of.Kind() {
	case reflect.Struct:
		return checkParam(of, data, decoder)
	case reflect.Slice, reflect.Array:
		elem := of.Type().Elem()
		elemType := elem.Kind()
		if elemType == reflect.Struct {
			return checkParamSlice(elem, data, decoder)
		}
	default:
		err := decoder.Decode(data)
		if err != nil {
			return err
		}
	}
	return nil
}

func checkParamSlice(elem reflect.Type, data any, decoder *json.Decoder) error {
	mapData := make([]map[string]interface{}, 0)
	_ = decoder.Decode(&mapData)
	if len(mapData) <= 0 {
		return nil
	}
	for i := 0; i < elem.NumField(); i++ {
		field := elem.Field(i)
		required := field.Tag.Get("msgo")
		tag := field.Tag.Get("json")
		value := mapData[0][tag]
		if value == nil && required == "required" {
			return errors.New(fmt.Sprintf("filed [%s] is required", tag))
		}
	}
	if data != nil {
		marshal, _ := json.Marshal(mapData)
		_ = json.Unmarshal(marshal, data)
	}
	return nil
}

5.3 引入第三方校验

gin等框架在做校验时，是使用了https://github.com/go-playground/validator 组件，我们也将其集成进来

type User struct {
	Name      string   `xml:"name" json:"name" msgo:"required"`
	Age       int      `xml:"name" json:"age" validate:"required,max=50,min=18"`
	Addresses []string `json:"addresses"`
}

func (c *Context) DealJson(data any) error {
	body := c.R.Body
	if c.R == nil || body == nil {
		return errors.New("invalid request")
	}
	decoder := json.NewDecoder(body)
	if c.DisallowUnknownFields {
		decoder.DisallowUnknownFields()
	}
	if c.IsValidate {
		err := validateRequireParam(data, decoder)
		if err != nil {
			return err
		}
	} else {
		err := decoder.Decode(data)
		if err != nil {
			return err
		}
	}
	return validate(data)
}

type SliceValidationError []error

func (err SliceValidationError) Error() string {
	n := len(err)
	switch n {
	case 0:
		return ""
	default:
		var b strings.Builder
		if err[0] != nil {
			fmt.Fprintf(&b, "[%d]: %s", 0, err[0].Error())
		}
		if n > 1 {
			for i := 1; i < n; i++ {
				if err[i] != nil {
					b.WriteString("\n")
					fmt.Fprintf(&b, "[%d]: %s", i, err[i].Error())
				}
			}
		}
		return b.String()
	}
}

func validate(obj any) error {
	if obj == nil {
		return nil
	}
	value := reflect.ValueOf(obj)
	switch value.Kind() {
	case reflect.Ptr:
		return validate(value.Elem().Interface())
	case reflect.Struct:
		return validateStruct(obj)
	case reflect.Slice, reflect.Array:
		count := value.Len()
		validateRet := make(SliceValidationError, 0)
		for i := 0; i < count; i++ {
			if err := validateStruct(value.Index(i).Interface()); err != nil {
				validateRet = append(validateRet, err)
			}
		}
		if len(validateRet) == 0 {
			return nil
		}
		return validateRet
	default:
		return nil
	}
}

func validateStruct(obj any) error {
	return validator.New().Struct(obj)
}

6. 优化验证器-接口+单例

在上面 验证的时候，每次都需要使用validator.New()，会极大的浪费性能，可以使用单例来做优化，同时验证器的实现可能有多种，提供接口，便于扩展

type StructValidator interface {
	//结构体验证，如果错误返回对应的错误信息
	ValidateStruct(any) error
	//返回对应使用的验证器
	Engine() any
}

package msgo

import (
	"github.com/go-playground/validator/v10"
	"reflect"
	"sync"
)

type StructValidator interface {
	// ValidateStruct 结构体验证，如果错误返回对应的错误信息
	ValidateStruct(any) error
	// Engine 返回对应使用的验证器
	Engine() any
}

type defaultValidator struct {
	one      sync.Once
	validate *validator.Validate
}

func (d *defaultValidator) ValidateStruct(obj any) error {
	if obj == nil {
		return nil
	}
	value := reflect.ValueOf(obj)
	switch value.Kind() {
	case reflect.Ptr:
		return d.ValidateStruct(value.Elem().Interface())
	case reflect.Struct:
		return d.validateStruct(obj)
	case reflect.Slice, reflect.Array:
		count := value.Len()
		validateRet := make(SliceValidationError, 0)
		for i := 0; i < count; i++ {
			if err := d.validateStruct(value.Index(i).Interface()); err != nil {
				validateRet = append(validateRet, err)
			}
		}
		if len(validateRet) == 0 {
			return nil
		}
		return validateRet
	default:
		return nil
	}
}

func (d *defaultValidator) validateStruct(obj any) error {
	d.lazyInit()
	return d.validate.Struct(obj)
}

func (d *defaultValidator) lazyInit() {
	d.one.Do(func() {
		d.validate = validator.New()
	})
}

func (c *Context) DealJson(data any) error {
	body := c.R.Body
	if c.R == nil || body == nil {
		return errors.New("invalid request")
	}
	decoder := json.NewDecoder(body)
	if c.DisallowUnknownFields {
		decoder.DisallowUnknownFields()
	}
	if c.IsValidate {
		err := validateRequireParam(data, decoder)
		if err != nil {
			return err
		}
	} else {
		err := decoder.Decode(data)
		if err != nil {
			return err
		}
	}
	return validate(data)
}

func validate(obj any) error {
	return Validator.ValidateStruct(obj)
}

7. 多种类型参数接收-绑定器实现

在实际中，我们需要处理json参数，xml参数或者其他参数，同样，我们将其行为抽象为接口，赋予其不同实现，这样代码更加优雅，扩展维护更加方便

import "net/http"

type Binding interface {
	Name() string
	Bind(*http.Request, any) error
}

7.1 JSON绑定器

package binding

import "net/http"

type Binding interface {
	Name() string
	Bind(*http.Request, any) error
}

var JSON = jsonBinding{}

package bind

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"reflect"
)

type jsonBinding struct {
	DisallowUnknownFields bool
	IsValidate            bool
}

func (b jsonBinding) Name() string {
	return "json"
}

func (b jsonBinding) Bind(r *http.Request, obj any) error {
	body := r.Body
	//post 传参的内容 是放在body中的
	if body == nil {
		return errors.New("invalid request")
	}
	decoder := json.NewDecoder(body)
	if b.DisallowUnknownFields {
		decoder.DisallowUnknownFields()
	}
	if b.IsValidate {
		err := validateParam(obj, decoder)
		if err != nil {
			return err
		}
	} else {
		err := decoder.Decode(obj)
		if err != nil {
			return err
		}

	}
	return validate(obj)
}

func validateParam(obj any, decoder *json.Decoder) error {

	//反射
	valueOf := reflect.ValueOf(obj)
	//判断其是否为指针类型
	if valueOf.Kind() != reflect.Pointer {
		return errors.New("This argument must have a pointer type")
	}
	elem := valueOf.Elem().Interface()
	of := reflect.ValueOf(elem)

	switch of.Kind() {
	case reflect.Struct:
		return checkParam(of, obj, decoder)
	case reflect.Slice, reflect.Array:
		elem := of.Type().Elem()
		if elem.Kind() == reflect.Struct {
			return checkParamSlice(elem, obj, decoder)
		}
	default:
		_ = decoder.Decode(obj)
	}
	return nil
}

func checkParamSlice(of reflect.Type, obj any, decoder *json.Decoder) error {
	mapValue := make([]map[string]interface{}, 0)
	_ = decoder.Decode(&mapValue)
	for i := 0; i < of.NumField(); i++ {
		field := of.Field(i)
		name := field.Name
		jsonName := field.Tag.Get("json")
		if jsonName != "" {
			name = jsonName
		}
		required := field.Tag.Get("msgo")
		for _, v := range mapValue {
			value := v[name]
			if value == nil && required == "required" {
				return errors.New(fmt.Sprintf("filed [%s] is not exist,because [%s] is required", jsonName, jsonName))
			}
		}
	}
	b, _ := json.Marshal(mapValue)
	_ = json.Unmarshal(b, obj)
	return nil
}

func checkParam(of reflect.Value, obj any, decoder *json.Decoder) error {
	//解析为map，然后根据map中的key 进行比对
	//判断类型 结构体 才能解析为map
	mapValue := make(map[string]interface{})
	_ = decoder.Decode(&mapValue)
	for i := 0; i < of.NumField(); i++ {
		field := of.Type().Field(i)
		name := field.Name
		jsonName := field.Tag.Get("json")
		if jsonName != "" {
			name = jsonName
		}
		required := field.Tag.Get("msgo")
		value := mapValue[name]
		if value == nil && required == "required" {
			return errors.New(fmt.Sprintf("filed [%s] is not exist,because [%s] is required", jsonName, jsonName))
		}
	}
	b, _ := json.Marshal(mapValue)
	_ = json.Unmarshal(b, obj)
	return nil
}

func (c *Context) BindJson(obj any) error {
	jsonBinding := binding.JSON
	jsonBinding.DisallowUnknownFields = c.DisallowUnknownFields
	jsonBinding.IsValidate = c.IsValidate
	return c.MustBindWith(obj, jsonBinding)
}

func (c *Context) MustBindWith(obj any, bind bind.Binding) error {
	if err := c.ShouldBind(obj, bind); err != nil {
		c.W.WriteHeader(http.StatusBadRequest)
		return err
	}
	return nil
}

func (c *Context) ShouldBind(obj any, bind bind.Binding) error {
	return bind.Bind(c.R, obj)
}

7.2 XML绑定器

func (c *Context) BindXML(obj any) error {
	return c.MustBindWith(obj, binding.XML)
}

package bind

import (
	"encoding/xml"
	"net/http"
)

type xmlBinding struct {
}

func (b xmlBinding) Name() string {
	return "mxl"
}

func (b xmlBinding) Bind(r *http.Request, obj any) error {
	if r.Body == nil {
		return nil
	}
	decoder := xml.NewDecoder(r.Body)
	if err := decoder.Decode(obj); err != nil {
		return err
	}
	return validate(obj)
}

type User struct {
	Name      string   `xml:"name" json:"name" msgo:"required"`
	Age       int      `xml:"age" json:"age" validate:"required,max=50,min=18"`
	Addresses []string `json:"addresses"`
}

<User>
<name>张三</name>
<age>20</age>
</User>

g.Post("/xmlParam", func(ctx *msgo.Context) {
		user := &User{}
		//user := User{}
		err := ctx.BindXML(user)
		if err == nil {
			ctx.JSON(http.StatusOK, user)
		} else {
			log.Println(err)
		}
	})