云原生ci/cd 部署¶
1.框架¶
gitea + nginx + argo-eventbus + argo-sensor + argo-workflow + argocd
#由于原生gitea webhook不支持 传入body参数，所以需要nginx转发
用户推送代码到 gitea 触发 gitea webhook --> nginx 接收url，lua脚本转换body参数 --> argo-eventbus 触发argo-sensor --> argo-workflow 运行任务 --> argocd 持续部署
2.gitea部署¶
这里我使用了 hostpath 存储，实际生产环境建议使用 nfs 存储。 密码我是用secret 形式的 后续可以使用vault 存储。
apiVersion: v1
kind: Secret
metadata:
name: gitea-db-secret
namespace: default
type: Opaque
stringData:
username: "postgres"
password: "****"
---
apiVersion: v1
kind: Service
metadata:
name: gitea
spec:
type: NodePort
selector:
app: gitea
ports:
- name: web
port: 3000
targetPort: web
nodePort: 30103
- name: git
port: 22
targetPort: git
nodePort: 30102
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: gitea
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app: gitea
template:
metadata:
labels:
app: gitea
spec:
#nodeSelector:
# kubernetes.io/hostname: "worker03" #The selected node
securityContext:
fsGroup: 1000
initContainers:
- name: init-permissions
image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/library/busybox:latest
command:
- sh
- -c
- chown -R 1000:1000 /data
securityContext:
runAsUser: 0
volumeMounts:
- name: data
mountPath: /data
containers:
- name: gitea
image: docker.io/gitea/gitea:1.21.1
imagePullPolicy: IfNotPresent
ports:
- name: web
containerPort: 3000
- name: git
containerPort: 22
env:
- name: TZ
value: Asia/Shanghai
- name: USER_UID
value: "1000"
- name: USER_GID
value: "1000"
- name: GITEA__database__DB_TYPE
value: postgres
- name: GITEA__database__HOST
value: "192.168.5.121:30284"
- name: GITEA__database__NAME
value: gitea
- name: GITEA__database__USER
valueFrom:
secretKeyRef:
name: gitea-db-secret
key: username
- name: GITEA__database__PASSWD
valueFrom:
secretKeyRef:
name: gitea-db-secret
key: password
- name: GITEA__server__SSH_LISTEN_PORT
value: "22"
- name: GITEA__server__SSH_PORT
value: "30102"
- name: GITEA__server__ROOT_URL
value: "http://192.168.5.121:30103/"
- name: GITEA__webhook__ALLOWED_HOST_LIST
value: "192.168.5.121" # webhook允许的内网地址
volumeMounts:
- name: data
mountPath: /data
readinessProbe:
httpGet:
path: /
port: web
initialDelaySeconds: 15
periodSeconds: 10
livenessProbe:
httpGet:
path: /
port: web
initialDelaySeconds: 30
periodSeconds: 20
volumes:
- name: data
hostPath:
path: /data/gitea
type: DirectoryOrCreate
3.nginx转发配置¶
webhook 填写 http://192.168.5.121:8888/blog?name=xxx&id=xxx 透传给argo-eventbus 的webhook
server {
listen 8888;
server_name _;
access_log /var/log/nginx/gitea-webhook-access.log;
error_log /var/log/nginx/gitea-webhook-error.log;
location /blog {
set $upstream "10.105.78.158:12002";
rewrite_by_lua_block {
ngx.req.read_body()
local body = ngx.req.get_body_data() or "{}"
local args = ngx.req.get_uri_args()
local injected = ""
for k, v in pairs(args) do
injected = injected .. ',"' .. k .. '":"' .. v .. '"'
end
if body == "{}" then
body = string.sub(injected, 2)
body = "{" .. body .. "}"
else
body = string.sub(body, 1, -2) .. injected .. "}"
end
ngx.req.set_body_data(body)
ngx.req.set_header("Content-Length", #body)
ngx.req.set_header("Content-Type", "application/json")
}
proxy_pass http://$upstream;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
4.argo-eventbus部署¶
kubectl create namespace argo-events
kubectl apply -f https://raw.githubusercontent.com/argoproj/argo-events/stable/manifests/install.yaml
kubectl apply -n argo-events -f https://raw.githubusercontent.com/argoproj/argo-events/stable/examples/eventbus/native.yaml
kubectl apply -n argo-events -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/rbac/sensor-rbac.yaml
kubectl apply -n argo-events -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/rbac/workflow-rbac.yaml
5.argo-workflow部署¶
# 添加仓库 并更新仓库
helm repo add argo https://argoproj.github.io/argo-helm
helm repo update argo
# 部署
# -set server.authMode=server 配置切换认证模式，便于免token 登录 UI 界面
helm install argo-workflows argo/argo-workflows -n argo --create-namespace --set server.authMode=server
#镜像如果拉不下来 拉目录下来更新
helm pull argo/argo-workflows --untar --untardir ./argo-workflows-local
helm upgrade argo-workflows ./argo-workflows-local -n default
#如果有values.yaml
helm upgrade argo-workflows ./argo-workflows-local -n default -f ./argo-workflows-local/values.yaml
# 如果你有自己额外的自定义配置文件
helm upgrade argo-workflows ./argo-workflows-local -n default -f ./argo-workflows-local/values.yaml -f ./my-custom-values.yaml
root@root:~/yaml/argo-workflows-local/test# kubectl -n argo get po
NAME READY STATUS RESTARTS AGE
argo-workflows-server-759c874fc4-fwc4t 1/1 Running 0 178m
argo-workflows-workflow-controller-b759bf54-lx89p 1/1 Running 0 178m
6.argo-workflow 配置¶
# Argo Workflow CRD API版本
apiVersion: argoproj.io/v1alpha1
# 资源类型：单次执行工作流
kind: WorkflowTemplate
metadata:
# 工作流名称
name: blog
# 运行命名空间
namespace: workflow
spec:
# 入口DAG模板，流水线起点
entrypoint: main-flow
# 全局入参，整个流程统一使用
arguments:
parameters:
# 业务代码Git仓库地址
- name: repo-url
value: http://${GITLAB_USER}:${GITLAB_TOKEN}@192.168.5.121:30103/ngcrm/blog.git
# 业务代码拉取分支
- name: branch
value: master
# 最终构建推送镜像完整地址(仓库/镜像名:tag)
- name: image-url
value: 192.168.5.179/test/java-test:v1
# GitOps部署配置仓库地址
- name: gitops-repo-url
value: http://${GITLAB_USER}:${GITLAB_TOKEN}@192.168.5.121:30103/ngcrm/deploy-java.git
# GitOps配置仓库分支
- name: gitops-branch
value: master
# 当前业务服务名称，用于目录隔离、镜像缓存、yaml匹配
- name: service-name
value: javatest
# 全局存储卷定义，所有模板共享
volumes:
# 工作目录持久化卷，代码克隆/构建/GitOps拉取统一存放
- name: workspace-storage
hostPath:
path: /data/argo-workflow/workspace
type: DirectoryOrCreate
# 镜像仓库认证配置，挂载kaniko用于私有仓库推送
- name: docker-config
secret:
secretName: registry-secret
items:
- key: .dockerconfigjson
path: config.json
# 流水线所有步骤模板集合
templates:
# 顶层DAG流程模板，控制任务串行依赖
- name: main-flow
dag:
tasks:
# 任务1：拉取业务源代码
- name: clone-code
template: clone-code
arguments:
parameters:
- name: repo-url
value: "{{workflow.parameters.repo-url}}"
- name: branch
value: "{{workflow.parameters.branch}}"
- name: service-name
value: "{{workflow.parameters.service-name}}"
# 任务2：Kaniko构建+推送镜像，依赖代码拉取完成
- name: build-push
template: build-push
arguments:
parameters:
- name: image-url
value: "{{workflow.parameters.image-url}}"
- name: service-name
value: "{{workflow.parameters.service-name}}"
dependencies:
- clone-code
# 任务3：更新GitOps配置仓库镜像tag，依赖镜像推送完成
- name: update-gitops
template: update-gitops
arguments:
parameters:
- name: image-url
value: "{{workflow.parameters.image-url}}"
- name: service-name
value: "{{workflow.parameters.service-name}}"
- name: gitops-repo-url
value: "{{workflow.parameters.gitops-repo-url}}"
- name: gitops-branch
value: "{{workflow.parameters.gitops-branch}}"
dependencies:
- build-push
# 模板：克隆业务代码
- name: clone-code
inputs:
parameters:
- name: repo-url
- name: branch
- name: service-name
container:
# git基础镜像，执行代码拉取
image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/alpine/git:v2.49.0
command: [sh, -c]
args:
- >
set -x
# 清理历史服务目录，避免旧文件干扰
rm -rf /workspace/{{inputs.parameters.service-name}}
echo "开始克隆代码..."
# 按指定分支克隆代码到独立服务目录
git clone --branch "{{inputs.parameters.branch}}" "{{inputs.parameters.repo-url}}" /workspace/{{inputs.parameters.service-name}}
# 克隆失败直接退出流程
if [ $? -ne 0 ]; then
echo "错误：代码拉取失败！"
exit 1
fi
echo "代码拉取完成，分支：{{inputs.parameters.branch}}"
volumeMounts:
# 挂载共享工作卷存放源码
- name: workspace-storage
mountPath: /workspace
# 模板：Kaniko无daemon构建镜像并推送仓库
- name: build-push
inputs:
parameters:
- name: image-url
- name: service-name
container:
# kaniko构建镜像(带debug工具)，无需宿主机docker
image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/gcr.io/kaniko-project/executor:v1.23.2-debug
command: [sh, -c]
args:
- >
# 校验Dockerfile是否存在，缺失直接失败
if [ ! -f /workspace/{{inputs.parameters.service-name}}/Dockerfile ]; then
echo "错误：未找到Dockerfile"
exit 1
fi
echo "开始构建镜像：{{inputs.parameters.image-url}}"
# kaniko构建推送核心参数
/kaniko/executor \
--context=/workspace/{{inputs.parameters.service-name}} \
--dockerfile=/workspace/{{inputs.parameters.service-name}}/Dockerfile \
--destination="{{inputs.parameters.image-url}}" \
--cache=true \ # 开启层缓存加速构建
--cache-repo=192.168.5.179/test/{{inputs.parameters.service-name}}/cache \ # 缓存仓库隔离服务
--skip-tls-verify=true # 内网私有仓库跳过证书校验
# 构建结果判断
if [ $? -eq 0 ]; then
echo "镜像构建推送成功！"
else
echo "镜像构建推送失败！"
exit 1
fi
volumeMounts:
- name: workspace-storage
mountPath: /workspace
# 挂载镜像仓库认证密钥用于推送
- name: docker-config
mountPath: /kaniko/.docker
# 模板：修改GitOps仓库values.yaml镜像tag并提交推送
- name: update-gitops
inputs:
parameters:
- name: image-url
- name: service-name
- name: gitops-repo-url
- name: gitops-branch
container:
image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/alpine/git:v2.49.0
command: [sh, -c]
args:
- >
set -x
# 截取镜像地址冒号后tag部分
IMAGE_TAG=$(echo "{{inputs.parameters.image-url}}" | cut -d':' -f2)
SERVICE="{{inputs.parameters.service-name}}"
echo "更新服务: $SERVICE 新Tag: $IMAGE_TAG"
# 存在本地目录则拉取最新代码，不存在则全新克隆GitOps仓库
if [ -d /workspace/gitops ]; then
cd /workspace/gitops
git fetch origin {{inputs.parameters.gitops-branch}}
git reset --hard origin/{{inputs.parameters.gitops-branch}}
git clean -fd
else
git clone --branch {{inputs.parameters.gitops-branch}} {{inputs.parameters.gitops-repo-url}} /workspace/gitops
cd /workspace/gitops
fi
# sed匹配对应服务块，替换tag值
sed -i -E "/^[[:space:]]*${SERVICE}:/,/^[^[:space:]]/ { s/^([[:space:]]*tag:)[[:space:]]*\".*\"/\\1 \"${IMAGE_TAG}\"/ }" values.yaml
# 打印修改后的tag校验
sed -n "/^ ${SERVICE}:/,/^[^ ]/ { /^ *tag: /p }" values.yaml
# 设置Git提交用户信息
git config user.email "workflow@argocd.com"
git config user.name "Argo Workflow"
git add values.yaml
# 提交备注[skip ci]跳过Git仓库自身流水线
git commit -m "Update ${SERVICE} image tag to ${IMAGE_TAG} [skip ci]"
# 拉取远端避免冲突，强制推送覆盖
git pull origin {{inputs.parameters.gitops-branch}} --rebase
git push origin {{inputs.parameters.gitops-branch}} --force
echo "✅ GitOps仓库更新完成，ArgoCD自动同步部署"
volumeMounts:
- name: workspace-storage
mountPath: /workspace
7.eventsource配置¶
eventsource 是一个用于监听 Kubernetes 事件的资源，它可以通过定义事件源（EventSource）和事件规则（EventRule）来监听 Kubernetes 事件。
apiVersion: argoproj.io/v1alpha1
kind: EventSource
metadata:
name: cicd
namespace: argo-events #因为eventbus在argo-evnets 的ns
spec:
service:
ports:
- port: 12001
targetPort: 12001
webhook:
example:
endpoint: /blog
method: POST
port: "12001"
8.sensor配置¶
sensor 是一个用于处理 Kubernetes 事件的资源，它可以通过定义传感器（Sensor）和触发器（Trigger）来处理 Kubernetes 事件。
apiVersion: argoproj.io/v1alpha1
kind: Sensor
metadata:
name: blog
namespace: argo-events
spec:
dependencies:
- eventName: example
eventSourceName: blog
name: cicd-webhook-dep
template:
serviceAccountName: operate-workflow-sa
triggers:
- template:
name: trigger-blog
argoWorkflow:
operation: submit
parameters:
# 以下参数虽然在wtf定义了默认的但是传的话还是要全部传
- dest: spec.arguments.parameters.0.value # 这个1是模板中第几个参数，从0开始
src:
dependencyName: cicd-webhook-dep
value: "http://${GITLAB_USER}:${GITLAB_TOKEN}@192.168.5.121:30103/ngcrm/blog.git"
- dest: spec.arguments.parameters.1.value
src:
dependencyName: cicd-webhook-dep
value: "master"
- dest: spec.arguments.parameters.2.value
src:
dataTemplate: '192.168.5.179/blog/blog:{{ printf "%.8s" .Input.body.head_commit.id }}' #取 git head_commit.id 的前8位
dependencyName: cicd-webhook-dep
- dest: spec.arguments.parameters.3.value
src:
dependencyName: cicd-webhook-dep
value: "http://${GITLAB_USER}:${GITLAB_TOKEN}@192.168.5.121:30103/ngcrm/deploy-java.git"
- dest: spec.arguments.parameters.4.value
src:
dependencyName: cicd-webhook-dep
value: "master"
- dest: spec.arguments.parameters.5.value
src:
dependencyName: cicd-webhook-dep
value: "blog"
- dest: spec.arguments.parameters.6.value
src:
dataTemplate: '{{ .Input.body.name }}'
dependencyName: cicd-webhook-dep
source:
resource:
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
generateName: auto-ci-cd-
namespace: workflow
spec:
arguments:
parameters:
- name: repo-url
value: "" #默认值，可自定义
- name: branch
value: ""
- name: image-url
value: ""
- name: gitops-repo-url
value: ""
- name: gitops-branch
value: ""
- name: service-name
value: ""
- name: name
value: ""
workflowTemplateRef:
name: blog #触发workflow的模板名称
9.查看整体效果¶
gitea webhook 触发
sensor日志
argo ui 查看
