Go Standard Lib

fmt

Format specifiers:

%t: boolean
%c: character
%U: unicode code point
%s: strings or []bytes
%q: quoted string
%#q: quoted string with backquotes
%b: bit representation
%d: integers (base 10)
%o: octal (base 8) notation
%x/%X: hexadecimal (base 16) notation
%f: floats
%g: complex
%e: scientific notation
%p: pointers (hexadecimal address with prefix 0x)
%v: default format, when String() exists, this is used.
%+v: gives a complete output of the instance with its fields
%#v: gives us a complete output of the instance with its fields and qualified type name
%T gives us the complete type specification
%%: if you want to print a literal % sign

type user struct {
    name string
}

func main() {
    u := user{"tang"}
    //Printf 格式化输出
    fmt.Printf("%+v\n", u)       //格式化输出结构
    fmt.Printf("%#v\n", u)       //输出值的 Go 语言表示方法
    fmt.Printf("%T\n", u)        //输出值的类型的 Go 语言表示
    fmt.Printf("%t\n", true)     //输出值的 true 或 false
    fmt.Printf("%b\n", 1024)     //二进制表示
    fmt.Printf("%c\n", 11111111) //数值对应的 Unicode 编码字符
    fmt.Printf("%d\n", 10)       //十进制表示
    fmt.Printf("%o\n", 8)        //八进制表示
    fmt.Printf("%q\n", 22)       //转化为十六进制并附上单引号
    fmt.Printf("%x\n", 1223)     //十六进制表示，用a-f表示
    fmt.Printf("%X\n", 1223)     //十六进制表示，用A-F表示
    fmt.Printf("%U\n", 1233)     //Unicode表示
    fmt.Printf("%b\n", 12.34)    //无小数部分，两位指数的科学计数法6946802425218990p-49
    fmt.Printf("%e\n", 12.345)   //科学计数法，e表示
    fmt.Printf("%E\n", 12.34455) //科学计数法，E表示
    fmt.Printf("%f\n", 12.3456)  //有小数部分，无指数部分
    fmt.Printf("%g\n", 12.3456)  //根据实际情况采用%e或%f输出
    fmt.Printf("%G\n", 12.3456)  //根据实际情况采用%E或%f输出
    fmt.Printf("%s\n", "wqdew")  //直接输出字符串或者[]byte
    fmt.Printf("%q\n", "dedede") //双引号括起来的字符串
    fmt.Printf("%x\n", "abczxc") //每个字节用两字节十六进制表示，a-f表示
    fmt.Printf("%X\n", "asdzxc") //每个字节用两字节十六进制表示，A-F表示
    fmt.Printf("%p\n", 0x123)    //0x开头的十六进制数表示
}

io

studygolang - io — 基本的 IO 接口

Reader/Writer

// interface definition
type Reader interface {
    Read(p []byte) (n int, err error)
}

type Writer interface {
    Write(p []byte) (n int, err error)
}

标准库中实现了 io.Reader 或 io.Writer 接口的类型

os.File 同时实现了 io.Reader 和 io.Writer
strings.Reader 实现了 io.Reader
bufio.Reader/Writer 分别实现了 io.Reader 和 io.Writer
bytes.Buffer 同时实现了 io.Reader 和 io.Writer
bytes.Reader 实现了 io.Reader
compress/gzip.Reader/Writer 分别实现了 io.Reader 和 io.Writer
crypto/cipher.StreamReader/StreamWriter 分别实现了 io.Reader 和 io.Writer
crypto/tls.Conn 同时实现了 io.Reader 和 io.Writer
encoding/csv.Reader/Writer 分别实现了 io.Reader 和 io.Writer
mime/multipart.Part 实现了 io.Reader
net/conn 分别实现了 io.Reader 和 io.Writer(Conn接口定义了Read/Write)

ReadFrom/WriteTo

file, _ := os.Open("writeAt.txt")
if err != nil {
    panic(err)
}
defer file.Close()
writer := bufio.NewWriter(os.Stdout)
writer.ReadFrom(file)
writer.Flush()

reader := bytes.NewReader([]byte("hello world"))
reader.WriteTo(os.Stdout)

Closer

// interface definition
type Closer interface {
    Close() error
}

ReadAll

原 ioutil.ReadAll

fileReader, _ := os.Open(fileName)
content, _ := io.ReadAll(fileReader)

bufio

Reader

io.Reader/io.Writer with buffer

// read until seperator
reader := bufio.NewReader(strings.NewReader("hello \nworld"))
line, _ := reader.ReadSlice('\n')
fmt.Printf("the line:%s\n", line)

// read string
s, err := reader.ReadString()

// read line
line, isPrefix, err := reader.ReadLine()

// read bytes
line, err := reader.ReadBytes('\n')
line = bytes.TrimRight(line, "\r\n")

Scanner

file, _ := os.Open("data/iris.csv")
scanner := bufio.NewScanner(file)
for scanner.Scan() {
    fmt.Println(scanner.Text())
}

Writer

使用 bufio.Writer 时，在所有的 Write 操作完成之后，应该调用 Flush 方法使得缓存都写入 io.Writer 对象中。

writer := bufio.NewWriter(os.Stdout)
writer.WriteString("hello")
writer.Flush()

bytes

Reader

x := []byte("hello world")

r1 := bytes.NewReader(x)
d1 := make([]byte,len(x))
n, _ := r1.Read(d1)
fmt.Println(n,string(d1))

Buffer

// init empty buffer
buf := new(bytes.Buffer)
// equivalent to
buf := &bytes.Buffer{}

// init buffer from bytes
buf := bytes.NewBuffer([]byte("Hello World"))

// init buffer from string
buf := bytes.NewBufferString("Hello World")

// get bytes from bytes.Buffer
b := buf.Bytes()

os

Exit

用于退出程序并返回一个整型返回值

调用后会立即退出，一般错误退出不要使用 os.Exit ，可能会来不及打印日志就退出

func main() {
    defer fmt.Println("!")
    os.Exit(3)
}

以上 defer 内容将无法打印出

exec

func main() {
  cmd := exec.Command("cal", "15", "2012")
  var stdout, stderr bytes.Buffer
  cmd.Stdout = &stdout
  cmd.Stderr = &stderr
  err := cmd.Run()
  if err != nil {
    log.Fatalf("cmd.Run() failed: %v\n", err)
  }

  fmt.Printf("output:\n%s\nerror:\n%s\n", stdout.String(), stderr.String())
}

ReadFile/WriteFile

原 ioutil.ReadFile/iotuil.WriteFile

fileName := "data/iris.csv"
read, _ := os.ReadFile(fileName)

sync

Go Course - Sync Package

useful primitives:

WaitGroup
Mutex
RWMutex
- StackOverflow - How to use RWMutex
Cond
Once
Pool
Map
Atomic

Wait Group

package main

import (
    "fmt"
    "sync"
)

func work() {
    fmt.Println("working...")
}

func main() {
    var wg sync.WaitGroup

    wg.Add(1)
    go func() {
        defer wg.Done()
        work()
    }()

    wg.Wait()
}

Mutex

package main

import (
    "fmt"
    "sync"
)

type Counter struct {
    m     sync.Mutex
    value int
}

func (c *Counter) Update(n int, wg *sync.WaitGroup) {
    c.m.Lock()
    defer wg.Done()
    fmt.Printf("Adding %d to %d\n", n, c.value)
    c.value += n
    c.m.Unlock()
}

func main() {
    var wg sync.WaitGroup

    c := Counter{}

    wg.Add(4)

    go c.Update(10, &wg)
    go c.Update(-5, &wg)
    go c.Update(25, &wg)
    go c.Update(19, &wg)

    wg.Wait()
}

Once

package main

import (
    "fmt"
    "sync"
)

func main() {
    var count int

    increment := func() {
        count++
    }

    var once sync.Once

    var increments sync.WaitGroup
    increments.Add(100)

    for i := 0; i < 100; i++ {
        go func() {
            defer increments.Done()
            once.Do(increment)
        }()
    }

    increments.Wait()
    fmt.Printf("Count is %d\n", count)
}

Map

context

Practical Go Lessons - Context

Usages:

Cancellation propagation
Transmission of request-scoped data along with the call stack
Set deadlines and timeouts

The Context interface

type Context interface {
    Deadline() (deadline time.Time, ok bool)
    Done() <-chan struct{}
    Err() error
    Value(key interface{}) interface{}
}

Add context to function/methods

// Go idiom: the context is usually the first argument
func foo1(ctx context.Context, a int) {
    // ...
}

// Go idiom: defer cancel(), avoid goroutine leak
ctx, cancel = context.WithCancel(ctx)
defer cancel()

To derive a context, you can use the following functions

WithCancel
WithTimeout
WithDeadline
WithValue

// Calling `WithCancel` will give us a way to cancel an operation
ctx, cancel := context.WithCancel(context.Background())

// Add timeout for an operation
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)

// Add deadline
deadline := time.Date(2021, 12, 12, 3, 30, 30, 30, time.UTC)
ctx, cancel := context.WithDeadline(context.Background(), deadline)

// call cancel function, and the cancellation signal will be propagated to the child context
cancel()

// carry value using WithValue
type key int

const (
    requestID key = iota
    jwt
)

ctx := context.WithValue(req.Context(), requestID, uuid.NewV4().String())
ctx = context.WithValue(ctx, jwt, req.Header.Get("Authorization"))

reqID, ok := ctx.Value(requestID).(string)
if !ok {
    return false, fmt.Errorf("requestID in context does not have the expected type")
}

net

TCP server

import "net"

listener, err := net.Listen("tcp", ":12358")
if err != nil {
    fmt.Println("Error listening", err.Error())
    return //终止程序
}
// 监听并接受来自客户端的连接
for {
    conn, err := listener.Accept()
    if err != nil {
        fmt.Println("Error accepting", err.Error())
        return // 终止程序
    }
    go handleConn(conn)
}

TCP client

conn, err := net.Dial("tcp", "192.168.3.3:12358")
if err != nil {
    fmt.Println("Error dialing ", err.Error())
    return
}
fmt.Println("Connected")
defer conn.Close()

http

client

Default http client

import "net/http"
import "bytes"

// get request
resp, err := http.Get("https://example.com")
if err != nil {
    // Handle the error
}
respBody := resp.Body
defer respBody.Close() // when you're using the http.Client to make requests, close the response body after reading from it
// Read from the response body
// ...
body, err := io.ReadAll(resp.Body)
if err != nil {
    // ...
}
fmt.Println(string(body))

Customizing http client

jsonBody := []byte(`{"client_message": "hello, server!"}`)
bodyReader := bytes.NewReader(jsonBody)

// post request
req, err := http.NewRequest(http.MethodPost, url, bodyReader)
if err != nil {
    // ...
}
client := &http.Client{
    Timeout: 10 * time.Second,
}
resp, err := client.Do(req)
if err != nil {
    // ...
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
	// ...
}
fmt.Println(string(body))

How to Use the HTTP Client in GO To Enhance Performance

Don’t use the default HTTP client, always specify the timeout in http.Client according to your use case

httpClient := &http.Client{
  Timeout: time.Second * 10,
}

Don’t use Default Transport and increase MaxIdleConnsPerHost

transport := &http.Transport{
    MaxIdleConns:        10,     // 最大的空闲连接数
    MaxConnsPerHost:     10,
    MaxIdleConnsPerHost: 10,
    IdleConnTimeout:     30,     // 空闲连接超时时间（秒）
}

httpClient = &http.Client{
  Timeout:   10 * time.Second,
  Transport: transport,
}

server

import (
	"fmt"
    "net/http"
)

func pingHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Printf("%+v\n", r.URL)
	w.Write([]byte("pong"))
}

func postDataHandler(w http.ResponseWriter, r *http.Request) {
	// limit http method
    if r.Method != http.MethodPost {
		http.Error(w, "Method not allowed.", http.StatusMethodNotAllowed)
		return
	}
    
    // parse request posted data
	reqData := make(map[string]interface{})
	err := json.NewDecoder(r.Body).Decode(&reqData)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	defer r.Body.Close() // request.Body should be close after read

	for k, v := range reqData {
		log.Printf("req %s: %v", k, v)
	}

    // set response
	w.Header().Set("Content-Type", "application/json")
	resp := map[string]interface{}{
		"Code":    0,
		"Message": "hello world",
	}
	err = json.NewEncoder(w).Encode(resp)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

func main() {
	http.HandleFunc("/ping", pingHandler)
	http.HandleFunc("/data", postDataHandler)
	http.ListenAndServe(":18080", nil)
}

math

rand

import "math/rand"

// 生成[0, 100)的随机整数
rand.Intn(100)

// 生成[0, 1)的随机浮点数
rand.Float64()

time

ticker

import "time"

// or use time.NewTicker().C
ticker := time.NewTicker(2 * time.Second)
defer ticker.Stop()
for range ticker.C {
    doSomething()
}

// tick immediately
for ; true; <- ticker.C {
    fmt.Printf("%s\n", "hello")
}

// for-select loop
interrupt := make(chan os.Signal, 1)
signal.Notify(interrupt, os.Interrupt)

ticker := time.NewTicker(2 * time.Second)
defer ticker.Stop()

loop:
for {
    select {
        case <- ticker.C: 
            f()
        case <- interrupt:
            break loop
    }
}

// use time.Tick
// NOT RECOMMANDED, the underlying Ticker cannot be recovered by the garbage collector; it "leaks"
for now := range time.Tick(2 * time.Second) {
	fmt.Println(now)
}

timer

// run in 5 seconds later
timer := time.NewTimer(5 * time.Second)
<- timer.C
doSomething()

log

import "log"

log.Println("hello world")

// log to file
file, err := os.OpenFile("logs.log", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
    log.Fatal(err)
}
log.SetOutput(file)
log.Println("hello wolrd")

// logger
logger = log.New(file, "MODULE1", log.Ldate|log.Ltime|log.Lshortfile) // set output file, prefix, and format
logger.Println("hello wolrd")

// log to file and stdout
import (
    "io"
    "os"
)
mw := io.MultiWriter(os.Stdout, file)
logger = log.New(mw, "MODULE1", log.Ldate|log.Ltime|log.Lshortfile)

encoding

json

struct -> JSON str

type Person struct {
	Name string
	Age  int
}

person := Person{Name: "Tom", Age: 12}
marshaled, err := json.Marshal(person)
if err != nil {
    fmt.Printf("Error marsaling %s\n", err)
}
jsonStr := string(marshaled)

JSON str -> struct

person := Person{}
if err := json.Unmarshal([]byte(jsonStr), &person); err != nil {
    fmt.Printf("Error unmarsaling %s\n", err)
}

map -> JSON str

m := map[string]interface{}{
    "Name": "Tom",
    "Age": 12,
}
marshaled, err := json.Marshal(m)
if err != nil {
    fmt.Printf("Error marsaling %s\n", err)
}
jsonStr := string(marshaled)

JSON str -> map

m := make(map[string]interface{})
if err := json.Unmarshal([]byte(jsonStr), &m); err != nil {
    fmt.Printf("Error unmarsaling %s\n", err)
}

gob

知乎 - Go语言二进制协议gob介绍

encode in gob

buf := new(bytes.Buffer)
enc := gob.NewEncoder(buf)
err := enc.Encode(data)
if err != nil {
    fmt.Println("gob encode failed, err:", err)
}

decode from gob

buf := bytes.NewBuffer(b)
dec := gob.NewDecoder(buf)
err = dec.Decode(&decoded)
if err != nil {
	fmt.Println("gob decode failed, err", err)
}

runtime

import "runtime"

// To exit goroutine prematurely
runtime.Goexit()

// Yields the processor, allowing other goroutines to run
runtime.Gosched()

// Runs a garbage collection and blocks the caller until the garbage collection is complete.
runtime.GC()

// Returns the number of logical CPUs
runtime.NumCPU()

// Gives detailed picture on how memory is being allocated, used, freed, and GC'ed
runtime.ReadMemStats()

// Count the number of goroutines
runtime.NumGoroutine()

pprof

func main() {
    f, _ := os.OpenFile("cpu.profile", os.O_CREATE|os.O_RDWR, 0644)
    defer f.Close()
    pprof.StartCPUProfile(f)
    defer pprof.StopCPUProfile()

    n := 10
    for i := 1; i <= 5; i++ {
        fmt.Printf("fib(%d)=%d\n", n, fib(n))
        n += 3 * i
    }
}

执行 go run main.go 后会生成一个 cpu.profile 文件。这个文件记录了程序的运行状态。使用 go tool pprof cpu.profile 命令分析这个文件。

查看火焰图

go tool pprof -http :8080 cpu.profile

内存 profiling

func main() {
    f, _ := os.OpenFile("memory.profile", os.O_CREATE|os.O_RDWR, 0644)
    defer f.Close()
    runtime.GC()
    if err := pprof.WriteHeapProfile(f); err != nil {
         log.Fatal("could not write memory profile: ", err)
    }

    n := 10
    for i := 1; i <= 5; i++ {
        fmt.Printf("fib(%d)=%d\n", n, fib(n))
        n += 3 * i
    }
}

net/http/pprof

import (
    _ "net/http/pprof"
)

func NewProfileHttpServer(addr string) {
    // 在一个新的 goroutine 中调用http.ListenAndServe()在某个端口启动一个默认的 HTTP  
    go func() {
        log.Fatalln(http.ListenAndServe(addr, nil))
    }()
}

使用命令 go run main.go 启动服务器可以用浏览器打开 http://localhost:9999/debug/pprof/，或者远程获取 profile 文件

go tool pprof -http :8080 localhost:9999/debug/pprof/profile?seconds=120

reflect

解析未知类型的变量

import (
	"fmt"
	"reflect"
)

type MyVar interface {
    Print() string
}

type Var1 struct {
    Name string
}

type Var2 struct {
    Name string
    Age int
}

func (v Var1) Print() string {
    return v.Name
}

func (v Var2) Print() string {
    return fmt.Sprintf("%s:%d", v.Name, v.Age)
}

func main() {
    v1 := Var1{Name: "v1"}
    v2 := Var2{Name: "v2", Age: 10}
    vs := []MyVar{v1, v2}
    for _, v := range vs {
        // 得到接口值的动态类型
        t := reflect.TypeOf(v)
        // 得到具体值
        value := reflect.ValueOf(v)
		fmt.Printf("%s: %+v\n", t.Name(), value)
    }
}

reflect.TypeOf 返回一个 reflect.Type 对象，它总是返回具体类型

reflect.ValueOf 返回一个 reflect.Value 对象，它的种类是有限的，只有少数几种，因此可以根据情况分别处理

v := reflect.ValueOf(variable)
siwtch v.Kind() {
case reflect.Invalde:
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
case reflect.Bool:
case reflect.String:
case reflect.Chan, reflect.Func, reflect.Ptr, reflect.Slice, reflect.Map:
case reflect.Array, reflect.Struct, reflect.Interface:
default:
}

也可以使用 reflect.Value 来设置值

elem := reflect.ValueOf(&v1.Name).Elem() // 获取 v1.Name 的地址
elem.Set(reflect.ValueOf(v2.Name))       // 将 v2.Name 的值赋给 v1.Name
fmt.Printf("%+v\n", v1)

反射常用示例

// 判断某值是否是 Slice
v := reflect.ValueOf(x)
if v.Kind() == reflect.Slice {
    fmt.Println("is slice")
} else {
    fmt.Println("is not slice")
}

// 根据字段名获取值
fieldValueOf := reflect.ValueOf(x).FieldByName("Name")

// 根据方法名调用方法
method := reflect.ValueOf(x).MethodByName("Show")
values := make([]reflect.Value, 0)
method.Call(values)

// 获取某个类的所有方法
v := reflect.ValueOf(x)
t := reflect.TypeOf(x)
for i := 0; i < v.NumMethod(); i++ {
    methodType := v.Method(i).Type()
    fmt.Printf("func (%s) %s%s\n", t, t.Method(i).Name, strings.TrimPrefix(methodType.String(), "func"))
}

fmt#

Format specifiers:#

io#

Reader/Writer#

ReadFrom/WriteTo#

Closer#

ReadAll#

bufio#

Reader#

Scanner#

Writer#

bytes#

Reader#

Buffer#

os#

Exit#

exec#

ReadFile/WriteFile#

sync#

Wait Group#

Mutex#

Once#

Map#

context#

net#

TCP server#

TCP client#

http#

client#

Default http client#

Customizing http client#

server#

math#

rand#

time#

ticker#

timer#

log#

encoding#

json#

gob#

runtime#

pprof#

net/http/pprof#

reflect#

fmt

Format specifiers:

io

Reader/Writer

ReadFrom/WriteTo

Closer

ReadAll

bufio

Reader

Scanner

Writer

bytes

Reader

Buffer

os

Exit

exec

ReadFile/WriteFile

sync

Wait Group

Mutex

Once

Map

context

net

TCP server

TCP client

http

client

Default http client

Customizing http client

server

math

rand

time

ticker

timer

log

encoding

json

gob

runtime

pprof

net/http/pprof

reflect