灰度发布

2018/5/20 posted in  OPS LUA OpenResty
CREATE TABLE `gray` (
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `project` varchar(45) NOT NULL,
  `shopId` text,
  `status` tinyint(1) unsigned NOT NULL DEFAULT '1',
  `created_at` timestamp NULL DEFAULT NULL,
  `updated_at` timestamp NULL DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4;
INSERT INTO `cpx_prod`.`gray` (`id`, `project`, `shopId`, `status`, `created_at`, `updated_at`) VALUES ('2', 'kufang', '1804', '1', '2018-05-28 17:04:25', '2018-05-28 17:04:25');

灰度开发方案

  1. 设计灰度表结构和存储位置
  2. 设计redis键,选择灰度发布使用的redis服务器
  3. 在ops平台上开发店铺灰度灰度状态管理页面
  4. 使用ops现有的指定发布功能发布灰度代码
  5. nginx lua根据店铺的灰度状态走不同的项目路径

灰度缓存更新

  1. ops灰度系统设置了店铺灰度后更新灰度缓存
  2. lua检测到redis里没有缓存key,从mysql中读取店铺灰度数据,缓存到redis中,redis使用一个key保存所有灰度店铺的id,使用json序列化

灰度代码发布

  1. 灰度代码发布复用ops上的指定发布

注意事项

​ 数据库改表(ddl)暂不支持灰度(或人工审核)

​ 支持灰度的就代码、新表、新库

目标

​ 降低上新功能的影响范围

usermod -a -G deploy yangsheng
ngx.say(cjson.encode(cjson.encode(v)))
ngx.say(type(v))
ngx.say(cjson.encode(v["shopId"]))
ngx.say("shopIds: ", type(shopIds))
ngx.say("jsoned shopIds: ", cjson.encode(shopIds))
ngx.say("hasValue: ", cjson.encode(inArray(shopIds, 1804)))
ngx.say("hasValue1: ", cjson.encode(inArray(shopIds, 1703)))
ngx.say("hasValue2: ", cjson.encode(inArray(shopIds, 1999)))
ngx.say("result: ", cjson.encode(res))
local shopId = 1804;
local project = "kufang"
local cjson = require "cjson"
local mysql = require "resty.mysql"
local redis = require "resty.redis"

-- 灰度
local GrayRouter = {
    project, -- 当前项目
    shopId, -- 当前店铺id
    grayShopIds = {}, -- 当前项目所有的灰度店铺id
    redisGrayKey, -- redis缓存键
    dbConfig = {
        host = "172.16.123.1",
        port = 3306,
        database = "cpx_prod",
        user = "root",
        password = "root",
        charset = "utf8",
        max_packet_size = 1024 * 1024,
    },
    redisHost = "172.16.123.1",
    redisPort = 6379,
    dbConnect, -- mysql 连接
    redisConnect, -- redis 连接
}

function GrayRouter:new(project, shopId)
    o = {}
    setmetatable(o, self)
    self.__index = self
    self.project = project
    self.shopId = shopId
    return o
end

function GrayRouter:setupRedisGrayKey(project)
    self.redisGrayKey = string.format("%s_shop_in_gray", project);
    return self.redisGrayKey
end

function GrayRouter:connectRedis()
    local red = redis:new()
    red:set_timeout(1000) -- 1 sec
    local ok, err = red:connect(self.redisHost, self.redisPort)
    if not ok then
        ngx.log(ngx.ERR, "failed to connect: ", err)
        return false
    end

    -- local res, err = red:auth("foobared")
    -- if not res then
    --     ngx.log(ngx.ERR, "failed to authenticate: ", err)
    --     return
    -- end
    ngx.log(ngx.DEBUG, "connected to redis.")
    self.redisConnect = red
    return self.redisConnect
end

function GrayRouter:connectMysql()
    ngx.log(ngx.DEBUG, "connectMysql")
    local db, err = mysql:new()
    if not db then
        ngx.log(ngx.ERR, "failed to instantiate mysql: ", err)
        return
    end

    db:set_timeout(1000) -- 1 sec

    local ok, err, errcode, sqlstate = db:connect(self.dbConfig)

    if not ok then
        ngx.log(ngx.ERR, "failed to connect: ", err, ": ", errcode, " ", sqlstate)
        return
    end

    ngx.log(ngx.DEBUG, "connected to mysql.")
    self.dbConnect = db

    return self.dbConnect
end

function GrayRouter:getGrayShopIds()
    local grayShopIds = {}

    -- 先从redis中取数据
    if self.redisConnect:exists(self.redisGrayKey) == 1 then
        ngx.say("read redis");
        local grayShopIds = self.redisConnect:get(self.redisGrayKey);
        grayShopIds = cjson.decode(grayShopIds);
    else
        -- 连接mysql
        if not self:connectMysql() then
            self:inProductionContext()
        end

        local res, err, errcode, sqlstate = self.dbConnect:query(string.format("SELECT * FROM gray WHERE project='%s' AND status=1", self.project))
        if not res then
            ngx.log(ngx.ERR, "bad result: ", err, ": ", errcode, ": ", sqlstate, ".")
            return
        end

        for k, v in pairs(res) do
            grayShopIds[k] = v["shopId"]
        end
        self.redisConnect:set(self.redisGrayKey, cjson.encode(grayShopIds))
        self:keepMysqlAlive()
    end

    self.grayShopIds = grayShopIds
    return self.grayShopIds
end

function GrayRouter:keepMysqlAlive()
    -- put it into the connection pool of size 100,
    -- with 10 seconds max idle timeout
    local ok, err = self.dbConnect:set_keepalive(10000, 100)
    if not ok then
        ngx.log(ngx.ERR, "failed to set keepalive: ", err)
        return
    end
end

-- nginx进入生产环境执行
function GrayRouter:inProductionContext()
    ngx.log(ngx.DEBUG, "in production context")
    -- ngx.exec("@normal")
end

-- nginx进入灰度环境执行
function GrayRouter:inGrayContext()
    ngx.log(ngx.DEBUG, "in gray context")
    -- ngx.exec("@gray")
end

function GrayRouter:inGray()
    for index, value in ipairs(self.grayShopIds) do
        if value == self.shopId then
            return true
        end
    end
    return false
end

-- 开始执行灰度逻辑
function GrayRouter:bootstrap()
    -- 检查参数
    if not self.project then
        self:inProductionContext()
    end
    if not self.shopId then
        self:inProductionContext()
    end
    -- 设置redis缓存key
    if not self:setupRedisGrayKey(self.project) then
        self:inProductionContext()
    end
    -- 连接redis
    if not self:connectRedis() then
        self:inProductionContext()
    end
    -- 从数据库或中获取灰度的店铺id
    if not self:getGrayShopIds() then
        self:inProductionContext()
    end
    -- 判断店铺是否在灰度名单中
    if self:inGray() then
        self:inGrayContext()
    else
        self:inProductionContext()
    end
end

gray = GrayRouter:new(project, shopId);
gray:bootstrap()