1、简介
kdop共享库与jenkins类似，但也不完全一样，目前kdop只能动态load一个groovy脚本。共享库主要解决多条流水线重复写相同的脚本，降低维护成本同时提高流水线稳定性。共享库存储在git上，koca-devops仓库ci分支下kdop目录
2、Git相关操作库
/**
* git克隆
* remoteAddr：仓库地址 如：http://ip:port/KOCA/koca-devops.git
* branch：git分支 如：refs/heads/develop
* credentialsId：git用户凭证
* dirName：git克隆在dirName目录下
*/
def gitClone(String remoteAddr, String branch, String credentialsId,String dirName) {
dir(dirName){
checkout([$class: 'GitSCM',
branches: [[name: (branch)]], doGenerateSubmoduleConfigurations: false,
extensions: [], submoduleCfg: [],
userRemoteConfigs: [[credentialsId: (credentialsId), url: (remoteAddr)]]])
}
}
return this
3、数据库相关操作库
/**
* 提供指定类型的增量数据脚本
* sqlPathList: sql路径列表
* dbType：数据库类型
* 返回：增量sql脚本的路径
*/
def getUpdateSql(def sqlPathList,String dbType) {
def locations = ''
if (sqlPathList != null) {
for (path in sqlPathList) {
def singlePath = path
def fullPath = "filesystem:$WORKSPACE/$singlePath,"
locations += fullPath
}
}
println(locations)
return locations
}
/**
* 提供指定类型的全量数据脚本
* sqlPathList: sql路径列表
* dbType：数据库类型
* 返回全量脚本的路径
*/
def getAllSql(def paths,String dbType) {
sh """
mkdir -p $WORKSPACE/${dbType}/all-sql
mv -f $WORKSPACE/${dbType}/all-sql $WORKSPACE/${dbType}/${env.BUILD_NUMBER}
mkdir -p $WORKSPACE/${dbType}/all-sql
"""
if (paths != null) {
for (path in paths) {
def singlePath = path
sh """
for file in ` find $WORKSPACE/$singlePath -type f -name '*.sql' | grep 'ALL' `
do
echo Y |cp -f \$file $WORKSPACE/${dbType}/all-sql
done
"""
}
}
sh """
cd $WORKSPACE/${dbType}/all-sql
for file in `ls `
do
mv \$file V0\${file#ALL}
done
"""
return "filesystem:$WORKSPACE/${dbType}/all-sql"
}
/*
清空数据库
dbConnectMsg：数据库的连接信息
*/
def flywayCleanDb(def dbConnectMsg) {
sh "flyway -driver=$dbConnectMsg.jdbcDriver -url='$dbConnectMsg.jdbcUrl' -user=$dbConnectMsg.jdbcUsername -password=$dbConnectMsg.jdbcPassword clean"
}
/*
增量更新数据库
dbConnectMsg：数据库连接信息
fromDatabase：包含数据库脚本与数据库类型
*/
def flywayUpdateDb(def dbConnectMsg, def fromDatabase) {
def sqlPath = getUpdateSql(fromDatabase.sqlPath,fromDatabase.type)
sh "flyway -locations=$sqlPath -driver=$dbConnectMsg.jdbcDriver -url='$dbConnectMsg.jdbcUrl' -user=$dbConnectMsg.jdbcUsername -password=$dbConnectMsg.jdbcPassword -validateOnMigrate='false' migrate"
}
/*
增量更新数据库,支持无序迁移
dbConnectMsg：数据库连接信息
fromDatabase：包含数据库脚本与数据库类型
*/
def flywayUpdateDbOutoforder(def dbConnectMsg, def fromDatabase) {
def sqlPath = getUpdateSql(fromDatabase.sqlPath,fromDatabase.type)
sh "flyway -locations=$sqlPath -driver=$dbConnectMsg.jdbcDriver -url='$dbConnectMsg.jdbcUrl' -user=$dbConnectMsg.jdbcUsername -password=$dbConnectMsg.jdbcPassword -validateOnMigrate='false' -outOfOrder='true' migrate"
}
/*
全量更新数据库
dbConnectMsg：数据库连接信息
fromDatabase：包含数据库脚本与数据库类型
*/
def flywayAllDb(def dbConnectMsg, def fromDatabase) {
def sqlPath = getAllSql(fromDatabase.sqlPath,fromDatabase.type)
sh "flyway -locations=$sqlPath -driver=$dbConnectMsg.jdbcDriver -url='$dbConnectMsg.jdbcUrl' -user=$dbConnectMsg.jdbcUsername -password=$dbConnectMsg.jdbcPassword -validateOnMigrate='false' migrate"
}
return this
4、KOCA标准门户相关操作库
def kocaGit
// 项目对象
class Project {
// 项目名
def name
// 服务列表
def servers
// web模块
def web
// 数据库模块
def fromDatabases
}
// 服务对象
class Server {
String name
String port
def args
}
// web对象
class Web {
def archetypeAndUis
def args
def modules
}
class ArchetypeAndUi {
String env
def archetype
String uiVersion
}
// 数据库连接对象
class DbConnectMsg{
String jdbcUrl
String jdbcUsername
String jdbcPassword
String jdbcDriver
}
// 数据库对象
class FromDatabase {
String type
def jdbc
def sqlPath
}
/*
读取yaml文件
filePath : project.yaml文件路径
**/
def readYaml(String filePath) {
def data = readYaml file: "$filePath"
return data
}
/*
初始化Project清单
yamlPath : project.yaml文件路径
**/
def initProject(String yamlPath){
Project project = new Project()
def data = readYaml(yamlPath)
// 获取项目名
project.name = data.project.name
// 获取服务列表
def servers = new ArrayList()
for(item in data.project.server) {
def server = new Server()
server.name = item.name
server.port = item.port
server.args = item.args
servers.add(server)
}
project.servers = servers
// 获取web模块
Web web = new Web()
web.archetypeAndUis = data.project.web.archetypeAndUi
web.args = data.project.web.args
web.modules = data.project.web.modules
project.web = web
// 获取数据库模块
def fromDatabases = new ArrayList()
for(item in data.project.fromDatabase) {
FromDatabase fromDatabase = new FromDatabase()
fromDatabase.type = item.type
fromDatabase.jdbc = item.jdbc
fromDatabase.sqlPath = item.sqlPath
fromDatabases.add(fromDatabase)
}
project.fromDatabases = fromDatabases
return project
}
// 获取数据库驱动
String switchJdbcDriver(String type) {
if (type == 'oracle') {
return 'oracle.jdbc.OracleDriver'
}else if (type == 'mysql') {
return 'com.mysql.cj.jdbc.Driver'
}else if (type == 'tdsql') {
return 'com.mysql.cj.jdbc.Driver'
}else if (type == 'mssql') {
return 'com.microsoft.sqlserver.jdbc.SQLServerDriver'
}else if (type == 'dm') {
return 'dm.jdbc.driver.DmDriver'
}
return ''
}
/*
获取指定环境的数据库链接信息
fromDatabase ： 指定类型的数据库信息
defaultBranch： 环境(开发、测试、演示等)
**/
def getDbConnectMsg(FromDatabase fromDatabase, String defaultBranch) {
DbConnectMsg dbConnectMsg = new DbConnectMsg()
dbConnectMsg.jdbcDriver = switchJdbcDriver(fromDatabase.type)
dbConnectMsg.jdbcUrl = fromDatabase.jdbc."${defaultBranch}".jdbcUrl
dbConnectMsg.jdbcUsername = fromDatabase.jdbc."${defaultBranch}".jdbcUsername
dbConnectMsg.jdbcPassword = fromDatabase.jdbc."${defaultBranch}".jdbcPassword
return dbConnectMsg
}
/**
获取指定数据库类型的FromDatabase对象
project ：project.yaml配置文件对象
dbType： 数据库类型
*/
def getFromDatabase(def project,String dbType) {
for(fromDatabase in project.fromDatabases) {
if (fromDatabase.type == dbType) {
return fromDatabase
}
}
}
/**
编译后台
path:后台pom.xml的路径
op: install packge 等
skip:是否跳过测试
*/
def compileServer(String path, String op,Boolean skip){
def testSkip
if (skip) {
testSkip = "-Dmaven.test.skip=true"
} else {
testSkip = ""
}
sh "cd $path && mvn clean $op $testSkip"
}
/**
编译bootapp
servers：bootapp列表
*/
def compileBootApps(def servers) {
for(serve in servers) {
def name = serve.name
compileServer(name, "package",true)
}
}
/*
启动服务
dbConnectMsg： 数据库的连接信息
server： 服务的信息
appName： 项目名
**/
def startServer(DbConnectMsg dbConnectMsg, Server server, String appName) {
sh """
filename=`ls $server.name/target/|grep bin`
tar -zxvf $server.name/target/\$filename -C /opt/application/$appName/
foldername=\${filename%-bin.tar.gz}
JENKINS_NODE_COOKIE=dontKillMe && /opt/application/$appName/\$foldername/bin/startup.sh \
--server.port=$server.port \
--koca.jdbc.default-data-source-id=default \
--koca.jdbc.data-sources[0].id=default \
--koca.jdbc.data-sources[0].pool.url='$dbConnectMsg.jdbcUrl' \
--koca.jdbc.data-sources[0].pool.username=$dbConnectMsg.jdbcUsername \
--koca.jdbc.data-sources[0].pool.password=$dbConnectMsg.jdbcPassword \
$server.args
sleep 10
"""
}
/**
启动bootapp
project ： project.yaml配置对象
dbType： 数据库类型mysql/oracle/mssql
env：环境develop/test/master
*/
def startBootApps(Project project, String dbType, String env) {
FromDatabase fromDatabase = getFromDatabase(project, dbType)
DbConnectMsg dbConnectMsg = getDbConnectMsg(fromDatabase, env)
def appName = project.name
sh """
mkdir -p /opt/application/$appName
ps -ef|grep java | grep koca|grep $appName| awk '{print \$2}'|xargs -r kill -9
rm -rf /opt/application/$appName/*
"""
for(service in project.servers) {
def arg
for(i in service.args) {
if (i.name == env) {
arg = i.value
break
} else {
arg = ''
}
}
service.args = arg
startServer(dbConnectMsg, service, appName)
}
}
// 清除npm缓存
def cleanNpmCache() {
sh "npm cache clean --force"
}
/*
获取环境的中文名
env：develop/test/master
**/
def getEnvName(String env) {
def envName
if (env == "dev") {
envName = "开发环境"
}
else if (env == "test") {
envName = "测试环境"
}
else {
envName = "演示环境"
}
return envName
}
/*
获取前端的参数
web: web对象
env: 环境
**/
def setWebArgs(Web web,String env) {
def envName = getEnvName(env)
def astr = "--VUE_APP_ENV_MARK:'$envName' "
for (a in web.args) {
astr += "--'" + a + "' "
}
return astr
}
/*
前端模块代码clone
modulesList： 前端组件模块清单
env：环境
**/
def frontModuleCodeClone(def modulesList, String env) {
kocaGit = load 'koca-devops/kdop/kocaGit.groovy'
def keys = modulesList.keySet()
for(key in keys) {
def item = modulesList[key]
def url = item["url"]
kocaGit.gitClone(url,"refs/heads/${env}",'gitlabuser',"$key")
}
}
/*
前端模块代码编译打包
m ：要构建的组件名
npmRegistry：本地的Npm仓库
key：内置的组件列表名
item：组件路径
version：vue版本
**/
def pack(String m,String npmRegistry, String key, String item, String version){
def path
if (version == "v2") {
path = "$npmRegistry/szkingdom.yf.$m"
} else {
path = "$npmRegistry/szkingdom.koca-$m"
}
sh """
mkdir -p $path
"""
dir("$key") {
sh """
cd $item
npm pack
filename=`ls |grep .tgz`
cd -
rm -rf *.tgz
mv -f $item/\$filename "../$path"
"""
}
}
// 前端模块代码本地打包并返回拼接后模块列表字符串
def npmLocalPack(Web web, def modulesList, String npmRegistry, String version, String form) {
def keys = modulesList.keySet()
def mstr = ""
for(m in web.modules) {
def find = false
for(key in keys) {
def items = modulesList[key]
def item = items["$m"]
if(item) {
mstr = mstr + m + ","
find = true
if (form != "npm") {
pack(m,npmRegistry, key, item, version)
}
}
}
if(find == false){
println("application project文件指定了不存在的前端模块 $m ！")
continue
}
}
println("要构建的模块清单: $mstr")
return mstr
}
/*
编译部署逻辑
name：项目名
nodePath：nodejs安装路径
webDeployPath：web包部署的路径
nvmPath：nvm包安装路径
from：依赖从源代码pack还是从npm拉
npmRegistry：本地Npm仓库路径
archetypeAndUi：前端模板及UI组件
mstr：拼接后模块列表字符串
astr: 门户版本信息
version：vue2/vue3
deleteLockFile：是否删除lock文件
changeProduction：修改.env.production文件
*/
def compileAndDeployWeb(String name,String nodePath, String webDeployPath, String nvmPath,String from,String npmRegistry, ArchetypeAndUi archetypeAndUi, String mstr, String astr, String version, Boolean deleteLockFile,String changeProduction) {
def preset
def buildcmd
def preBuildCmd=""
def kcCmd
def nodeVersion
if (version == "v3") {
nodeVersion = 16
preset = 3
// 判断是否删除pnpm-lock.yaml
def rmLockFile =""
if (deleteLockFile) {
rmLockFile = "rm -rf pnpm-lock.yaml &&"
}
// 修改env.production
def items = changeProduction.split(',')
println("items:$items")
for(String item in items) {
if (item.indexOf("=") == -1) {
continue
}
def result = item.split('=')
if (result.size() == 1){
continue
}
String key = result[0]
String value = result[1]
println("key:$key,value:$value")
preBuildCmd = preBuildCmd + "sed -i \"s#$key\\s*=.*#$item#g\" ./.env.production && "
}
preBuildCmd = preBuildCmd+"cat ./.env.production"
println("preBuildCmd:$preBuildCmd")
buildcmd = "$rmLockFile $nodePath/npm install -g pnpm@6.35.1 && $nodePath/pnpm install --no-frozen-lockfile && $nodePath/pnpm build"
} else {
nodeVersion = 14
preset = 2
buildcmd = "$nodePath/npm install --legacy-peer-deps --ignore-scripts && $nodePath/npm rebuild node-sass && $nodePath/npm run build $astr"
}
if (from == "npm") {
kcCmd = " kc create $name -f --archetype=$archetypeAndUi.archetype.name --version $archetypeAndUi.archetype.version --uiversion $archetypeAndUi.uiVersion --modules=$mstr --preset=$preset"
} else {
kcCmd = "kc create $name -f --source=$WORKSPACE/$npmRegistry --archetype=$archetypeAndUi.archetype.name --version $archetypeAndUi.archetype.version --uiversion $archetypeAndUi.uiVersion --modules=$mstr --preset=$preset"
}
sh """
sh /home/npm-group.sh
. $nvmPath/nvm.sh
$nodePath/npm cache clean --force
nvm use $nodeVersion
$nodePath/npm -g install @szkingdom.koca/cli@latest --force
mkdir -p $webDeployPath/$name
cd koca-front
mkdir -p $name
$kcCmd
cd $name
$preBuildCmd
$buildcmd
rm -rf $webDeployPath/$name/*
echo Y | cp -rf dist/* $webDeployPath/$name/
cd -
"""
}
/*
构建前端
project: project.yml配置文件对应的对象
nodePath： nodejs的路径
webDeployPath：web包部署的路径
nvmPath：nvm的安装路径
form：依赖从源代码pack还是从npm拉
env：dev/test/master
version：vue2/vue3
modulesList: 内置的模块与代码路径
deleteLockFile：是否删除lock文件
changeProduction：修改.env.production文件
*/
def compileWebFormSource(Project project, String nodePath, String webDeployPath, String nvmPath,String form,String env, String version, def modulesList,Boolean deleteLockFile, String changeProduction) {
String name = project.name
Web web = project.web
ArchetypeAndUi archetypeAndUi
for(i in web.archetypeAndUis) {
if (i.env == env) {
archetypeAndUi = i
break
}
}
// 本地仓库路径
def npmRegistry = 'koca-front/npm-registry'
sh """
rm -rf koca-front/npm-registry
mkdir -p $npmRegistry
"""
// 前端模块代码clone
frontModuleCodeClone(modulesList,env)
// 前端模块代码本地打包并返回拼接后模块列表字符串
def mstr = npmLocalPack(web, modulesList,npmRegistry,version,form)
def astr = ""
if(version == "v2"){
setWebArgs(web,env)
for(arg in web.args) {
if (arg.indexOf("VUE_APP_VERSION") != -1) {
def argKeyValue = arg.split(':')
def value = argKeyValue[1]
astr = "--VUE_APP_VERSION:'$value'"
}
}
}
// 编译部署web
compileAndDeployWeb(name, nodePath,webDeployPath,nvmPath,form,npmRegistry,archetypeAndUi,mstr,astr,version,deleteLockFile,changeProduction)
}
/*
更新nginx配置
portalFolder：源nginx配置文件路径
nginxPath: nginx安装路径
*/
def reloadNginx(String portalFolder, String nginxPath) {
sh """
cp -rf $portalFolder/nginx-config/* $nginxPath/conf/conf.d/
$nginxPath/sbin/nginx -t
$nginxPath/sbin/nginx -s reload
"""
}
return this