Utils
godetenv
读取同目录下 .env
文件
name=Tom
age=18
example
import "github.com/joho/godotenv"
func main() {
err := godotenv.Load()
if err != nil {
log.Fatal(err)
}
fmt.Println("name: ", os.Getenv("name"))
fmt.Println("age: ", os.Getenv("age"))
}
// 也可以加载其他文件
// 如果多个文件出现同一个 key,先出现的优先后出现的不生效
godotenv.Load("common", "dev.env")
也可以加载 YAML 文件
name: awesome web
version: 0.0.1
# os.Getenv("name")
一般有多个环境文件:.env.dev
、.env.test
、env.production
,可以用以下代码切换
env := os.Getenv("APP_ENV")
if env == "" {
env = "dev"
}
err := godotenv.Load(".env." + env)
if err != nil {
log.Fatal(err)
}
err = godotenv.Load()
if err != nil {
log.Fatal(err)
}
fmt.Println("name: ", os.Getenv("name"))
fmt.Println("version: ", os.Getenv("version"))
fmt.Println("database: ", os.Getenv("database"))
CLI
flag
用于解析命令行参数
package main
import (
"fmt"
"flag"
)
var (
intflag int
boolflag bool
stringflag string
)
func main() {
flag.IntVar(&intflag, "intflag", 0, "int flag value")
flag.BoolVar(&boolflag, "boolflag", false, "bool flag value")
flag.StringVar(&stringflag, "stringflag", "default", "string flag value")
flag.Parse()
fmt.Println("int flag:", intflag)
fmt.Println("bool flag:", boolflag)
fmt.Println("string flag:", stringflag)
}
run in shell
go run main.go -h
go run main.go -stringflag hello -intflag 1 -boolflag
支持格式
./main -isbool
./main -flag=x
./main -flag x
./main --isbool
./main --flag=x
./main --flag x
# 停止解析
./main -- -flag=x
./main noflag -flag=x
Pflag
用于替代 flag,解析符合 POSIX 标准格式的参数
import "github.com/spf13/pflag"
// declares an integer flag, -flagname, stored in the pointer ip, with type *int.
var ip *int = pflag.Int("flagname", 1234, "help message for flagname")
// with one-letter shorthands
var ip = pflag.IntP("flagname", "f", 1234, "help message")
// or bind the flag to a variable using the Var() functions
var flagvar int
pflag.IntVar(&flagvar, "flagname", 1234, "help message for flagname")
// parse the command line
pflag.Parse()
Go-flags
用于解析命令行参数,功能更丰富,使用 tag 定义参数
import "github.com/jessevdk/go-flags"
type Option struct {
Verbose []bool `short:"v" long:"verbose" description:"Show verbose debug message"`
Required string `short:"r" long:"required" required:"true"`
Default string `short:"d" long:"default" default:"default"`
}
func main() {
var opt Option
flags.Parse(&opt)
fmt.Println(opt.Verbose)
}
选项分组
type Option struct {
Basic GroupBasicOption `description:"basic type" group:"basic"`
Slice GroupSliceOption `description:"slice of basic type" group:"slice"`
}
type GroupBasicOption struct {
IntFlag int `short:"i" long:"intflag" description:"int flag"`
BoolFlag bool `short:"b" long:"boolflag" description:"bool flag"`
FloatFlag float64 `short:"f" long:"floatflag" description:"float flag"`
StringFlag string `short:"s" long:"stringflag" description:"string flag"`
}
type GroupSliceOption struct {
IntSlice int `long:"intslice" description:"int slice"`
BoolSlice bool `long:"boolslice" description:"bool slice"`
FloatSlice float64 `long:"floatslice" description:"float slice"`
StringSlice string `long:"stringslice" description:"string slice"`
}
func main() {
var opt Option
p := flags.NewParser(&opt, flags.Default)
_, err := p.ParseArgs(os.Args[1:])
if err != nil {
log.Fatal("Parse error:", err)
}
basicGroup := p.Command.Group.Find("basic")
for _, option := range basicGroup.Options() {
fmt.Printf("name:%s value:%v\n", option.LongNameWithNamespace(), option.Value())
}
sliceGroup := p.Command.Group.Find("slice")
for _, option := range sliceGroup.Options() {
fmt.Printf("name:%s value:%v\n", option.LongNameWithNamespace(), option.Value())
}
}
子命令
子命令必须实现go-flags
定义的Commander
接口:
type Commander interface {
Execute(args []string) error
}
type MathCommand struct {
Op string `long:"op" description:"operation to execute"`
Args []string
Result int64
}
func (this *MathCommand) Execute(args []string) error {
if this.Op != "+" && this.Op != "-" && this.Op != "x" && this.Op != "/" {
return errors.New("invalid op")
}
for _, arg := range args {
num, err := strconv.ParseInt(arg, 10, 64)
if err != nil {
return err
}
this.Result += num
}
this.Args = args
return nil
}
type Option struct {
Math MathCommand `command:"math"`
}
func main() {
var opt Option
_, err := flags.Parse(&opt)
if err != nil {
log.Fatal(err)
}
fmt.Printf("The result of %s is %d", strings.Join(opt.Math.Args, opt.Math.Op), opt.Math.Result)
}
Viper
用于解析配置
import "github.com/spf13/viper"
viper.SetConfigName("config") // 设置文件名,不要带后缀
viper.SetConfigType("toml") // 配置类型
viper.AddConfigPath(".", "/etc/conf") //配置搜索路径
// 读取配置文件
err := viper.ReadInConfig()
if err != nil {
log.Fatal("read config failed: %v", err)
}
// 设置默认值
viper.SetDefault("redis.port", 6381)
// 获取配置
fmt.Println(viper.Get("app_name")) // return interface{}
fmt.Println(viper.Get("log_level"))
fmt.Println("redis ip: ", viper.GetString("redis.ip")) // get string
fmt.Println("redis port: ", viper.GetInt("redis.port")) // get int
fmt.Println("protocols: ", viper.GetStringSlice("server.protocols")) // get string slice
fmt.Println("timeout: ", viper.GetDuration("server.timeout")) // get duration
fmt.Println("mysql settings: ", viper.GetStringMap("mysql")) // return map[string]interface{}
fmt.Println("redis settings: ", viper.GetStringMapString("redis")) // return map[string]string
fmt.Println("all settings: ", viper.AllSettings()) // get all settings in map[string]interface{}
// check variable is set or not
if viper.IsSet("redis.port") {
fmt.Println("redis.port is set")
} else {
fmt.Println("redis.port is not set")
}
// Unmarshal
type MySQLConfig struct {
IP string
Port int
User string
Password string
Database string
}
var c MySQLConfig
viper.Unmarshal(&c)
// 命令行选项
pflag.Int("redis.port", 8381, "Redis port to connect") // viper 使用 pflag 库来解析选项
viper.BindPFlags(pflag.CommandLine) // 绑定选项到配置中
// 绑定环境变量
viper.AutomaticEnv() // 绑定全部环境变量
viper.BindEnv("go.path", "GOPATH") // 单独绑定环境变量
fmt.Println("GOPATH: ", viper.Get("GOPATH"))
保存配置
viper.Set("log_level", "DEBUG")
err := viper.SafeWriteConfig() // 如果配置文件存在,则不覆盖
if err != nil {
log.Fatal("write config failed: ", err)
}
监听文件修改
viper.WatchConfig()
fmt.Println("redis port before sleep: ", viper.Get("redis.port"))
time.Sleep(time.Second * 10) // 在这期间修改文件配置,下次会读到新值
fmt.Println("redis port after sleep: ", viper.Get("redis.port"))
cobra
用于构建命令行程序
子命令放在 cmd/your_command.go
下
import "github.com/spf13/cobra"
var rootCmd = &cobra.Command {
Use: "hugo",
Short: "Hugo is a very fast static site generator",
Long: `A Fast and Flexible Static Site Generator built with
love by spf13 and friends in Go.
Complete documentation is available at https://gohugo.io/documentation/`,
Run: func(cmd *cobra.Command, args []string) {
// Do Stuff Here
},
}
func Execute() {
if err := rootCmd.Execute(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
cli
用于构建命令行程序,相对于 cobra 更简洁
import (
"github.com/urfave/cli/v2" // imports as package "cli"
)
func main() {
app := &cli.App{
Name: "boom",
Usage: "make an explosive entrance",
Action: func(*cli.Context) error {
fmt.Println("boom! I say!")
return nil
},
}
if err := app.Run(os.Args); err != nil {
log.Fatal(err)
}
}
with options
func main() {
app := &cli.App{
Flags: []cli.Flag{
&cli.StringFlag{
Name: "lang",
Value: "english",
Usage: "language for the greeting",
},
},
Action: func(c *cli.Context) error {
name := "world"
if c.NArg() > 0 {
name = c.Args().Get(0)
}
if c.String("lang") == "ch" {
fmt.Println("你好", name)
} else {
fmt.Println("hello", name)
}
return nil
},
}
err := app.Run(os.Args)
if err != nil {
log.Fatal(err)
}
}
with sub command
func main() {
app := &cli.App{
Commands: []*cli.Command{
{
Name: "add",
Aliases: []string{"a"},
Usage: "add a task to the list",
Action: func(c *cli.Context) error {
fmt.Println("added task: ", c.Args().First())
return nil
},
},
{
Name: "complete",
Aliases: []string{"c"},
Usage: "complete a task on the list",
Action: func(c *cli.Context) error {
fmt.Println("completed task: ", c.Args().First())
return nil
},
},
{
Name: "template",
Aliases: []string{"t"},
Usage: "options for task templates",
Subcommands: []*cli.Command{
{
Name: "add",
Usage: "add a new template",
Action: func(c *cli.Context) error {
fmt.Println("new task template: ", c.Args().First())
return nil
},
},
{
Name: "remove",
Usage: "remove an existing template",
Action: func(c *cli.Context) error {
fmt.Println("removed task template: ", c.Args().First())
return nil
},
},
},
},
},
}
err := app.Run(os.Args)
if err != nil {
log.Fatal(err)
}
}
bubbletea
Network
Gin
bindings
bind query
type Request struct {
Size int64 `form:"size"`
Page int64 `form:"page"`
}
request
curl "http://localhost:8080/api/test?size=1&page=1"
bind query with validation
type Request struct {
Size int64 `form:"size" binding:"required,min=1,max=100"`
Page int64 `form:"page" binding:"required,min=1`
Sort string `form:"sort,default=ts"`
Order string `form:"order,default=desc" binding:"oneof=desc asc"`
}
bind query array parameters
type testBindRequest struct {
Values []string `form:"value"`
}
request
curl "http://localhost:8080/api/test?value=a&value=b&value=c"
bind JSON
type testBindRequest struct {
Size int64 `json:"size" binding:"required,min=1,max=100"`
Page int64 `json:"page" binding:"required,min=1"`
Sort string `json:"sort"`
Order string `json:"order" binding:"oneof=desc asc"`
Filters []string `json:"filter"`
}
request
curl -XPOST -H 'Content-Type: application/json' -d '{"size":1,"page":1,"filter":["a:1","b:2"]' "http://localhost:8080/api/test"
JSON parameters do not support default value, can use Github - go-defaults
type Request struct {
Size int64 `json:"size" binding:"min=1,max=100" default:"1"`
Page int64 `json:"page" binding:"min=1" default:"1"`
Sort string `json:"sort" default:"ts"`
Order string `json:"order" binding:"oneof=desc asc" default:"desc"`
Filters []Filter `json:"filters"`
}
var req Request
defaults.SetDefaults(&req)
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{
"error": err.Error(),
})
}
bind path parameter
// router
r.GET("/name/:name", s.GetPerson)
// get param
name := c.Param("name")
grouping
func main() {
router := gin.Default()
// Simple group: v1
v1 := router.Group("/v1")
{
v1.POST("/login", loginEndpoint)
v1.POST("/submit", submitEndpoint)
v1.POST("/read", readEndpoint)
}
// Simple group: v2
v2 := router.Group("/v2")
{
v2.POST("/login", loginEndpoint)
// nested group
v21 := v2.Group("/v21")
{
v21.POST("/login", loginEndpoint)
v21.POST("/submit", submitEndpoint)
v21.POST("/read", readEndpoint)
}
}
router.Run(":8080")
}
cors
Github - CORS gin’s middleware
package main
import (
"time"
"github.com/gin-contrib/cors"
"github.com/gin-gonic/gin"
)
func main() {
router := gin.Default()
// CORS for https://foo.com and https://github.com origins, allowing:
// - PUT and PATCH methods
// - Origin header
// - Credentials share
// - Preflight requests cached for 12 hours
router.Use(cors.New(cors.Config{
AllowOrigins: []string{"https://foo.com"},
AllowMethods: []string{"PUT", "PATCH"},
AllowHeaders: []string{"Origin"},
ExposeHeaders: []string{"Content-Length"},
AllowCredentials: true,
AllowOriginFunc: func(origin string) bool {
return origin == "https://github.com"
},
MaxAge: 12 * time.Hour,
}))
router.Run()
}
middlewares
func main() {
// Creates a router without any middleware by default
r := gin.New()
// Global middleware
// Logger middleware will write the logs to gin.DefaultWriter even if you set with GIN_MODE=release.
// By default gin.DefaultWriter = os.Stdout
r.Use(gin.Logger())
// Recovery middleware recovers from any panics and writes a 500 if there was one.
r.Use(gin.Recovery())
// Per route middleware, you can add as many as you desire.
r.GET("/benchmark", MyBenchLogger(), benchEndpoint)
// Authorization group
// authorized := r.Group("/", AuthRequired())
// exactly the same as:
authorized := r.Group("/")
// per group middleware! in this case we use the custom created
// AuthRequired() middleware just in the "authorized" group.
authorized.Use(AuthRequired())
{
authorized.POST("/login", loginEndpoint)
authorized.POST("/submit", submitEndpoint)
authorized.POST("/read", readEndpoint)
// nested group
testing := authorized.Group("testing")
testing.GET("/analytics", analyticsEndpoint)
}
// Listen and serve on 0.0.0.0:8080
r.Run(":8080")
}
func Logger() gin.HandlerFunc {
// Do some initialization logic here
// When middleware gets loaded into request chain, whatever you define before the return statement will be executed only once
return func(c *gin.Context) {
t := time.Now()
// Set example variable
c.Set("example", "12345")
// before request
c.Next()
// after request
latency := time.Since(t)
log.Print(latency)
// access the status we are sending
status := c.Writer.Status()
log.Println(status)
}
}
func main() {
r := gin.New()
r.Use(Logger())
r.GET("/test", func(c *gin.Context) {
example := c.MustGet("example").(string)
// it would print: "12345"
log.Println(example)
})
// Listen and serve on 0.0.0.0:8080
r.Run(":8080")
}
writer
// 当需要返回简单的文本字符串时
c.String(http.StatusOK, "OK")
// 当需要返回 JSON 格式的数据时
c.JSON(http.StatusOK, gin.H{"message": "OK"})
// 需要更精细地控制响应内容时
c.Writer.WriteHeader(http.StatusOK)
c.Writer.Write([]byte("OK"))
// 返回任意类型的响应数据,包括二进制数据
c.Data(http.StatusOK, "text/plain", []byte("OK"))
DataBase
sqlx
package main
import (
"database/sql"
"fmt"
"log"
_ "github.com/lib/pq"
"github.com/jmoiron/sqlx"
)
var schema = `
CREATE TABLE person (
first_name text,
last_name text,
email text
);
CREATE TABLE place (
country text,
city text NULL,
telcode integer
)`
type Person struct {
FirstName string `db:"first_name"`
LastName string `db:"last_name"`
Email string
}
type Place struct {
Country string
City sql.NullString
TelCode int
}
func main() {
// this Pings the database trying to connect
// use sqlx.Open() for sql.Open() semantics
db, err := sqlx.Connect("postgres", "user=foo dbname=bar sslmode=disable")
if err != nil {
log.Fatalln(err)
}
// exec the schema or fail; multi-statement Exec behavior varies between
// database drivers; pq will exec them all, sqlite3 won't, ymmv
db.MustExec(schema)
tx := db.MustBegin()
tx.MustExec("INSERT INTO person (first_name, last_name, email) VALUES ($1, $2, $3)", "Jason", "Moiron", "jmoiron@jmoiron.net")
tx.MustExec("INSERT INTO person (first_name, last_name, email) VALUES ($1, $2, $3)", "John", "Doe", "johndoeDNE@gmail.net")
tx.MustExec("INSERT INTO place (country, city, telcode) VALUES ($1, $2, $3)", "United States", "New York", "1")
tx.MustExec("INSERT INTO place (country, telcode) VALUES ($1, $2)", "Hong Kong", "852")
tx.MustExec("INSERT INTO place (country, telcode) VALUES ($1, $2)", "Singapore", "65")
// Named queries can use structs, so if you have an existing struct (i.e. person := &Person{}) that you have populated, you can pass it in as &person
tx.NamedExec("INSERT INTO person (first_name, last_name, email) VALUES (:first_name, :last_name, :email)", &Person{"Jane", "Citizen", "jane.citzen@example.com"})
tx.Commit()
// Query the database, storing results in a []Person (wrapped in []interface{})
people := []Person{}
db.Select(&people, "SELECT * FROM person ORDER BY first_name ASC")
jason, john := people[0], people[1]
fmt.Printf("%#v\n%#v", jason, john)
// Person{FirstName:"Jason", LastName:"Moiron", Email:"jmoiron@jmoiron.net"}
// Person{FirstName:"John", LastName:"Doe", Email:"johndoeDNE@gmail.net"}
// You can also get a single result, a la QueryRow
jason = Person{}
err = db.Get(&jason, "SELECT * FROM person WHERE first_name=$1", "Jason")
fmt.Printf("%#v\n", jason)
// Person{FirstName:"Jason", LastName:"Moiron", Email:"jmoiron@jmoiron.net"}
// if you have null fields and use SELECT *, you must use sql.Null* in your struct
places := []Place{}
err = db.Select(&places, "SELECT * FROM place ORDER BY telcode ASC")
if err != nil {
fmt.Println(err)
return
}
usa, singsing, honkers := places[0], places[1], places[2]
fmt.Printf("%#v\n%#v\n%#v\n", usa, singsing, honkers)
// Place{Country:"United States", City:sql.NullString{String:"New York", Valid:true}, TelCode:1}
// Place{Country:"Singapore", City:sql.NullString{String:"", Valid:false}, TelCode:65}
// Place{Country:"Hong Kong", City:sql.NullString{String:"", Valid:false}, TelCode:852}
// Loop through rows using only one struct
place := Place{}
rows, err := db.Queryx("SELECT * FROM place")
for rows.Next() {
err := rows.StructScan(&place)
if err != nil {
log.Fatalln(err)
}
fmt.Printf("%#v\n", place)
}
// Place{Country:"United States", City:sql.NullString{String:"New York", Valid:true}, TelCode:1}
// Place{Country:"Hong Kong", City:sql.NullString{String:"", Valid:false}, TelCode:852}
// Place{Country:"Singapore", City:sql.NullString{String:"", Valid:false}, TelCode:65}
// Named queries, using `:name` as the bindvar. Automatic bindvar support
// which takes into account the dbtype based on the driverName on sqlx.Open/Connect
_, err = db.NamedExec(`INSERT INTO person (first_name,last_name,email) VALUES (:first,:last,:email)`,
map[string]interface{}{
"first": "Bin",
"last": "Smuth",
"email": "bensmith@allblacks.nz",
})
// Selects Mr. Smith from the database
rows, err = db.NamedQuery(`SELECT * FROM person WHERE first_name=:fn`, map[string]interface{}{"fn": "Bin"})
// Named queries can also use structs. Their bind names follow the same rules
// as the name -> db mapping, so struct fields are lowercased and the `db` tag
// is taken into consideration.
rows, err = db.NamedQuery(`SELECT * FROM person WHERE first_name=:first_name`, jason)
// batch insert
// batch insert with structs
personStructs := []Person{
{FirstName: "Ardie", LastName: "Savea", Email: "asavea@ab.co.nz"},
{FirstName: "Sonny Bill", LastName: "Williams", Email: "sbw@ab.co.nz"},
{FirstName: "Ngani", LastName: "Laumape", Email: "nlaumape@ab.co.nz"},
}
_, err = db.NamedExec(`INSERT INTO person (first_name, last_name, email)
VALUES (:first_name, :last_name, :email)`, personStructs)
// batch insert with maps
personMaps := []map[string]interface{}{
{"first_name": "Ardie", "last_name": "Savea", "email": "asavea@ab.co.nz"},
{"first_name": "Sonny Bill", "last_name": "Williams", "email": "sbw@ab.co.nz"},
{"first_name": "Ngani", "last_name": "Laumape", "email": "nlaumape@ab.co.nz"},
}
_, err = db.NamedExec(`INSERT INTO person (first_name, last_name, email)
VALUES (:first_name, :last_name, :email)`, personMaps)
}
Gorm
Others
- xorm
- pgx
- sqlc
- goose
Data
NSQ
Concepts
- topic: a topic is a distinct stream of messages
- channel: an independent queue for a topic
- messages are pushed to consumers
- messages are dilivered at least once
- messages received are un-ordered
HTTP API
https://nsq.io/components/nsqd.html
# 查看node
curl "http://${host}:4161/nodes"
# 查看topic列表
curl "http://${host}:4161/topics"
# 查看producer列表
curl "http://${host}:4161/lookup?topic=${topic_name}"
# 查看channels
curl "http://${host}:4161/channels?topic=${topic_name}"
Utils
- nsqd:
- default port: 4151
- nsqlookupd: discovery service
- nsqlookupd instances are independent, need no coordination
- default port: 4161
- nsqadmin: web interface to administrate and introspect
- default port: 4171
- nsq_to_http: transport over HTTP
- nsq_tail: consume and writes to stdout
- nsq_to_file: persists to disk
- nsq_to_nsq: consume and re-publish to destination nsqd
- to_nsq: read stdin and publish to nsqd
- nsq_stat: display aggregate stats
# nsqadmin
bin/nsqadmin -lookupd-http-address="${host}:4161"
# tail
bin/nsq_tail -lookupd-http-address="${host}:4161" -topic=${topic_name}
# to file
bin/nsq_to_file -lookupd-http-address="${host}:4161" -topic=${topic_name} --output-dir=output
python
Install
pip install pynsq
Consumer
import nsq
# Callback for each message received
def process_message(message):
print('Received message:', message.body)
message.finish()
# Create a reader instance
reader = nsq.Reader(
message_handler=process_message,
lookupd_http_addresses=['http://127.0.0.1:4161'],
topic='example_topic',
channel='example_channel',
lookupd_poll_interval=15
)
# Start the IOLoop to handle asynchronous operations
nsq.run()
Producer
# Function to handle the response from nsq
def finish_pub(conn, data):
print('Published:', data)
# Create a producer instance
writer = nsq.Writer(['127.0.0.1:4150'])
# Publish a message to the 'example_topic' topic
topic = 'example_topic'
message = 'Hello NSQ!'
writer.pub(topic, message, finish_pub)
# Start the IOLoop to handle asynchronous operations
nsq.run()
go-elasticsearch
初始化
The Go client for Elasticsearch: Configuration and customization
// import
import (
elasticsearch7 "github.com/elastic/go-elasticsearch/v7"
)
// init client
es, err := elasticsearch.NewDefaultClient()
if err != nil {
log.Fatalf("Error creating the client: %s", err)
}
插入数据
查询数据
解析结果
https://github.com/elastic/go-elasticsearch/tree/main/_examples/encoding
泛型
samber/lo
import "github.com/samber/lo"
// map
lo.Map([]int64{1, 2, 3, 4}, func(x int64, index int) string {
return strconv.FormatInt(x, 10)
})
// []string{"1", "2", "3", "4"}