package httpclent import ( "context" "errors" "fmt" "io" "io/ioutil" "net/http" "net/url" "reflect" "strings" "time" "git.shuncheng.lu/bigthing/gocommon/pkg/conf" "git.shuncheng.lu/bigthing/gocommon/pkg/internal/json" "git.shuncheng.lu/bigthing/gocommon/pkg/logger" "git.shuncheng.lu/bigthing/gocommon/pkg/net/qconf" "git.shuncheng.lu/bigthing/gocommon/pkg/trace" ) // 请求+抓取异常+解析响应体(比较适合自动解析响应体的,适用于POST/GET/.. 获取响应体为Json的) func HttpRequestAndDecode(ctx context.Context, serverName string, path string, params interface{}, result interface{}, options ...Option) error { // 校验绑定的响应类型 if err := validateReceiver(result); err != nil { // result 只支持ptr return err } // 发送请求 response, err := DefaultHttpRequest(ctx, serverName, path, params, options...) if err != nil { logger.Errorc(ctx, "[HttpRequest] find err,err=%v", err) return err } // 检测响应体 if err := CheckResponseBodyHasError(response); err != nil { // 这里不记录 error 日志,根据业务需求记录 return err } // 绑定响应数据 err = json.Unmarshal(response, result) if err != nil { logger.Errorc(ctx, "[HttpRequest] json decode http response err: %s", err) return fmtError("[HttpRequest] json decode http response err: %s", err) } return nil } // 主要是适用于 对于请求参数无需自己绑定/默认校验不符合标准规则/请求参数可能不是json可能是文件的请求 func DefaultHttpRequest(ctx context.Context, serverName string, path string, params interface{}, options ...Option) ([]byte, error) { beginTime := time.Now() ctx = context.WithValue(ctx, _ServerName, serverName) // clone,不影响原来的ctx,并发安全 ops := LoadDefaultOp(options...) initDefaultHttpRequestOp(ops) defer func() { FreeOp(ops) }() result, err := HttpRequest(ctx, func() (*http.Request, error) { request, err := newProxyRequest(ctx, serverName, path, params, ops) if err != nil { return nil, err } logger.Infoc(ctx, "[HttpRequest] start,server_name=%s,url=%s,params=%+v", serverName, request.URL, params) return request, nil }, ops) if err != nil { return nil, err } logger.Infoc(ctx, "[HttpRequest] end,server_name=%s,path=%s,spend=%fs,result=%s", serverName, path, time.Now().Sub(beginTime).Seconds(), result) return result, err } // 内部代理的http的request func newProxyRequest(ctx context.Context, serverName string, requestPath string, requestParam interface{}, op *Options) (*http.Request, error) { if op == nil { return nil, errors.New("[newProxyRequestV2] find error") } // 处理请求URL _url, err := handlerRequestUrl(ctx, serverName, requestPath, op) if err != nil { return nil, err } // json请求参数 reader, err := handlerRequestParams(ctx, requestParam, op) if err != nil { return nil, err } // 创建request, 升级api,目前线上编译版本高于1.13,支持传递ctx request, err := http.NewRequestWithContext(ctx, string(op.HttpMethod), _url.String(), reader) if err != nil { return nil, fmtError("[NewRequest] err,err=%+v,url=%s,params=%+v", err, _url, requestParam) } // 处理请求 postHandlerRequest(ctx, request, op) return request, nil } // new一个普通的request func NewHttpRequest(method, url string) (*http.Request, error) { return http.NewRequest(method, url, nil) } // 主要是适用于自定义的更强的 // no cached cookie func HttpRequest(ctx context.Context, request NewHttpRequestFunc, option *Options) ([]byte, error) { if option.RetryMaxNum == 0 { option.RetryMaxNum = defaultRetryNum } if option.TimeOut == 0 { option.TimeOut = defaultTimeOut1s } if option.RetryIdleTime == 0 { option.RetryIdleTime = defaultRetryIdleTime } client := http.Client{ Transport: http.DefaultTransport, Timeout: option.TimeOut, } var retryNum = 0 Retry: response, err := httpDo(ctx, &client, request) if err != nil { if retryNum < option.RetryMaxNum { retryNum++ logger.Warnc(ctx, "[HttpRequest] request %s err,cur_retry=%d,max_retry=%d,err=%v", getServerName(ctx), retryNum, option.RetryMaxNum, err) time.Sleep(option.RetryIdleTime) goto Retry } else { if response != nil && response.Body != nil { response.Body.Close() } return nil, err } } defer response.Body.Close() if option.HandlerResponse != nil { if err := option.HandlerResponse(ctx, response); err != nil { return nil, err } } body, err := ioutil.ReadAll(response.Body) if err != nil { return nil, err } return body, nil } func httpDo(ctx context.Context, client *http.Client, requestFunc NewHttpRequestFunc) (*http.Response, error) { request, err := requestFunc() if err != nil { return nil, err } beginTime := time.Now() response, err := trace.HttpClientTrace(ctx, client, request, getServerName(ctx)) // 打印第三方调用日志 addThirdLog(ctx, request, response, beginTime) return response, err } func addThirdLog(ctx context.Context, request *http.Request, response *http.Response, beginTime time.Time) { // 检查 第三方日志开关 是否关闭 if !conf.GetThirdLogWitch() { return } // 打印第三方调用日志 if response != nil { logger.ThirdLog(request.URL.String(), response.StatusCode, time.Now().Sub(beginTime).Milliseconds(), response.ContentLength, logger.GetTraceId(ctx)) } else { //response 为 nil, err 一定有值 // 互联网信息服务扩展状态码, 449 Retry With 重新发送 logger.ThirdLog(request.URL.String(), 449, time.Now().Sub(beginTime).Milliseconds(), 0, logger.GetTraceId(ctx)) } } func initDefaultHttpRequestOp(op *Options) { if op.HandlerRequestHeader == nil { op.HandlerRequestHeader = DefaultHandlerRequestHandler } if op.HandlerRequestParams == nil { op.HandlerRequestParams = DefaultHandlerRequestParams } if op.EncodeRequestBody == nil { op.EncodeRequestBody = DefaultEncodeRequestBody } if op.ServerHostUrl == nil { op.ServerHostUrl = DefaultServerHost } if op.Scheme == "" { op.Scheme = defaultScheme } if op.HttpMethod == "" { op.HttpMethod = defaultMethod } } func handlerRequestUrl(ctx context.Context, host string, requestPath string, op *Options) (*url.URL, error) { if op.ServerHostUrl != nil { var err error if host, err = op.ServerHostUrl(ctx, host); err != nil { return nil, fmtError("[handlerRequestUrl] err,err=%+v,server_name=%s", err, host) } } _url := url.URL{ Scheme: string(op.Scheme), Host: host, Path: requestPath, } // 处理请求url if op.HandlerRequestUrl != nil { if err := op.HandlerRequestUrl(ctx, &_url); err != nil { return nil, err } } return &_url, nil } func postHandlerRequest(ctx context.Context, req *http.Request, op *Options) { // 请求头 if op.HandlerRequestHeader != nil { op.HandlerRequestHeader(ctx, req.Header) } // 请求cookie if op.AddRequestCookies != nil { cookies := op.AddRequestCookies(ctx, req.Cookies()) for index, _ := range cookies { req.AddCookie(cookies[index]) } } } // 处理请求参数 func handlerRequestParams(ctx context.Context, req interface{}, op *Options) (io.Reader, error) { if op.HandlerRequestParams != nil { req = op.HandlerRequestParams(ctx, req) } if op.EncodeRequestBody != nil { return op.EncodeRequestBody(ctx, req) } return nil, nil } func GetServerUrl(serverName string) (string, error) { host := conf.GetString(serverName, "host", "") //开发环境使用的是ip地址,无法使用qconf if strings.Compare(conf.GetEnv(), conf.EnvDebug) == 0 { return host, nil } host, err := qconf.GetHost(host, qconf.GetQconfIdc()) if err != nil { return "", err } return host, nil } func fmtError(format string, v ...interface{}) error { return errors.New(fmt.Sprintf(format, v...)) } // 校验需要绑定的参数,只能指针类型! func validateReceiver(result interface{}) error { if result == nil { return validateNilError } else { kind := reflect.TypeOf(result).Kind() if kind != reflect.Ptr { return validateTypeError } return nil } } func getServerName(ctx context.Context) string { s, _ := ctx.Value(_ServerName).(string) return s } type HttpRequestInfo struct { ServiceName string Path string Method string Params interface{} } // 兼容老版本! func NewProxyRequest(requestInfo *HttpRequestInfo, ctx context.Context, op *Options) (*http.Request, error) { if requestInfo.Method != "" { op.HttpMethod = HttpMethod(requestInfo.Method) } return newProxyRequest(ctx, requestInfo.ServiceName, requestInfo.Path, requestInfo.Params, op) }