第19章 CI/CD

部署gitlab

编写资源配置清单

创建命名空间：

kubectl create namespace devops

创建harbor-secret

cat > harbor-secret.yaml << 'EOF'
apiVersion: v1
kind: Secret
metadata:
  namespace: devops
  name: harbor-secret
data:
  .dockerconfigjson: ewoJImF1dGhzIjogewoJCSJsdWZmeS5jb20iOiB7CgkJCSJhdXRoIjogIllXUnRhVzQ2U0dGeVltOXlNVEl6TkRVPSIKCQl9Cgl9Cn0=
type: kubernetes.io/dockerconfigjson
EOF
kubectl apply -f harbor-secret.yaml

编写dp资源

cat > gitlab-dp.yaml << 'EOF'
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: gitlab
  namespace: devops
  labels:
    app: gitlab
spec:
  selector:
    matchLabels:
      app: gitlab
  template:
    metadata:
      name: gitlab
      labels:
        app: gitlab
    spec:
      nodeName: node-02
      imagePullSecrets:
      - name: harbor-secret
      initContainers:
      - name: fix-permissions
        image: busybox
        imagePullPolicy: IfNotPresent
        command: ["sh", "-c", "chown -R 1000:1000 /var/opt/gitlab"]
        securityContext:
          privileged: true
        volumeMounts:
        - name: gitlab
          mountPath: /var/opt/gitlab
      containers:
      - name: gitlab
        # image: gitlab/gitlab-ce:latest
        image: luffy.com/devops/gitlab-ce:latest
        imagePullPolicy: IfNotPresent
        env:
        - name: GITLAB_OMNIBUS_CONFIG
          value: |
            external_url 'http://gitlab-svc'
        #    alertmanager['enable'] = false
        #    grafana['enable'] = false
        #    prometheus['enable'] = false
        #    node_exporter['enable'] = false
        #    redis_exporter['enable'] = false
        #    postgres_exporter['enable'] = false
        #    pgbouncer_exporter['enable'] = false
        #    gitlab_exporter['enable'] = false
        ports:
        - name: http
          containerPort: 80
        - name: ssh
          containerPort: 22
        volumeMounts:
        - mountPath: /var/opt/gitlab
          name: gitlab
      volumes:
        - name: gitlab
          hostPath:
            path: /data/gitlab/
            type: DirectoryOrCreate
EOF

gitlab-svc

cat > gitlab-svc.yaml << 'EOF'
---
apiVersion: v1
kind: Service
metadata:
  name: gitlab-svc
  namespace: devops
  labels:
    app: gitlab
spec:
  selector:
    app: gitlab
  ports:
    - name: http
      port: 80
      targetPort: 80
    - name: ssh
      port: 22
      targetPort: 22
  type: ClusterIP
EOF

gitlab-ingress

cat > gitlab-ingress.yaml << 'EOF'
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: gitlab
  namespace: devops
spec:
  ingressClassName: nginx
  rules:
  - host: gitlab.local
    http:
      paths:
      - path: /
        pathType: ImplementationSpecific
        backend:
          service:
            name: gitlab-svc
            port:
              number: 80
EOF

查看密码:

kubectl -n devops exec -it $(kubectl -n devops get pod|awk '/gitlab/{print $1}') -- grep "^Password" /etc/gitlab/initial_root_password

实验代码

[root@master-01 ~/xzs]# tree -L 2
.
├── docker-compose.yaml
├── Dockerfile
├── pom.xml
├── README.md
├── settings.xml
├── src
│   └── main
└── start.sh

阿里云的settings

nexus的settings

<?xml version="1.0" encoding="UTF-8"?>
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
  <pluginGroups>
  </pluginGroups>

  <proxies>
  </proxies>

  <servers>
    <server>
      <id>snapshots</id>
      <username>admin</username>
      <password>admin</password>
    </server>
    <server>
      <id>releases</id>
      <username>admin</username>
      <password>admin</password>
    </server>
    <server>
      <id>public</id>
      <username>admin</username>
      <password>admin</password>
    </server>
  </servers>

  <mirrors>
    <mirror>
      <id>public</id>
      <mirrorOf>*</mirrorOf>
      <url>http://10.0.0.51:8081/repository/maven-public/</url>
    </mirror>
  </mirrors>

  <profiles>
    <profile>
      <id>public</id>
      <repositories>
        <repository>
          <id>public</id>
          <url>http://10.0.0.51:8081/repository/maven-public/</url>
          <releases><enabled>true</enabled></releases>
          <snapshots><enabled>true</enabled></snapshots>
        </repository>
      </repositories>
      <pluginRepositories>
        <pluginRepository>
          <id>public</id>
          <url>http://10.0.0.51:8081/repository/maven-public/</url>
          <releases><enabled>true</enabled></releases>
          <snapshots><enabled>true</enabled></snapshots>
        </pluginRepository>
      </pluginRepositories>
    </profile>
  </profiles>

  <activeProfiles>
    <activeProfile>public</activeProfile>
  </activeProfiles>
</settings>

部署jenkins

编写资源配置清单

创建命名空间

kubectl create namespace devops

创建RBAC

cat > jenkins-rbac.yaml << 'EOF'
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: jenkins-admin
rules:
  - apiGroups: [""]
    resources: ["*"]
    verbs: ["*"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: jenkins-admin
  namespace: devops
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: jenkins-admin
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: jenkins-admin
subjects:
- kind: ServiceAccount
  name: jenkins-admin
  namespace: devops
EOF

创建pv和pvc

cat > jenkins-pv.yaml << 'EOF'
---
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: local-storage
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsumer
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: jenkins-pv-volume
  labels:
    type: local
spec:
  storageClassName: local-storage
  claimRef:
    name: jenkins-pv-claim
    namespace: devops
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  local:
    path: /data/jenkins
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - node-01
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: jenkins-pv-claim
  namespace: devops
spec:
  storageClassName: local-storage
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 3Gi
EOF

创建DP

cat > jenkins-dp.yaml << 'EOF'
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: jenkins-server
  namespace: devops
spec:
  replicas: 1
  selector:
    matchLabels:
      app: jenkins-server
  template:
    metadata:
      labels:
        app: jenkins-server
    spec:
      nodeName: node-01
      imagePullSecrets:
      - name: harbor-secret
      securityContext:
            fsGroup: 1000
            runAsUser: 1000
      serviceAccountName: jenkins-admin
      initContainers:
      - name: fix-permissions
        image: busybox
        imagePullPolicy: IfNotPresent
        command: ["sh", "-c", "chown -R 1000:1000 /var/jenkins_home"]
        securityContext:
          privileged: true
        volumeMounts:
        - name: jenkins-data
          mountPath: /var/jenkins_home
      containers:
        - name: jenkins-server
          # image: jenkins/jenkins:lts-jdk17
          image: luffy.com/devops/jenkins:lts-jdk17
          imagePullPolicy: IfNotPresent
          resources:
            limits:
              memory: "500Mi"
              cpu: "500m"
            requests:
              memory: "500Mi"
              cpu: "500m"
          ports:
            - name: httpport
              containerPort: 8080
            - name: jnlpport
              containerPort: 50000
          livenessProbe:
            httpGet:
              path: "/login"
              port: 8080
            initialDelaySeconds: 90
            periodSeconds: 10
            timeoutSeconds: 5
            failureThreshold: 5
          readinessProbe:
            httpGet:
              path: "/login"
              port: 8080
            initialDelaySeconds: 60
            periodSeconds: 10
            timeoutSeconds: 5
            failureThreshold: 3
          volumeMounts:
            - name: jenkins-data
              mountPath: /var/jenkins_home
      #volumes:
      #  - name: jenkins-data
      #    persistentVolumeClaim:
      #        claimName: jenkins-pv-claim
      volumes:
        - name: jenkins-data
          hostPath:
            path: /data/jenkins/
            type: DirectoryOrCreate
EOF

创建svc

cat > jenkins-svc.yaml << 'EOF'
---
apiVersion: v1
kind: Service
metadata:
  name: jenkins-svc
  namespace: devops
  labels:
    app: jenkins-svc
spec:
  selector:
    app: jenkins-server
  ports:
    - name: web
      port: 8080
      targetPort: httpport
    - name: agent
      port: 50000
      targetPort: jnlpport
EOF

创建ingress

cat > jenkins-ingress.yaml << 'EOF'
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: jenkins-ingress
  namespace: devops
spec:
  ingressClassName: nginx
  rules:
  - host: jenkins.local
    http:
      paths:
      - path: /
        pathType: ImplementationSpecific
        backend:
          service:
            name: jenkins-svc
            port:
              number: 8080
EOF

安装必备插件

kubernetes
git
git parameter
pipeline
pipeline: stage view
active choices
Localization: Chinese (Simplified)

配置kubernetes插件

jenkins/inbound-agent

运行测试

保存kubeconfig到jenkins凭证

保存ssh公钥和私钥到jenkins凭证

这里有坑注意：

这里的公钥和私钥不要使用Centos7系统上生成的，要使用jenkins-agent容器上生成的，因为jenkins-agent容器是Debian系统，Centos7的公钥和私钥似乎在Debian上不太正常。最保险的，还是在jenkins-agent容器上生成，然后保存在jenkins凭证里。

添加私钥

添加公钥

配置信任harbor地址

cat > /etc/docker/daemon.json << 'EOF'
{
      "insecure-registries": ["https://luffy.com"]
}
EOF

配置habor-Secret

cat > harbor-secret.yaml << 'EOF'
apiVersion: v1
kind: Secret
metadata:
  namespace: devops
  name: harbor-secret
data:
  .dockerconfigjson: ewoJImF1dGhzIjogewoJCSJsdWZmeS5jb20iOiB7CgkJCSJhdXRoIjogIllXUnRhVzQ2U0dGeVltOXlNVEl6TkRVPSIKCQl9Cgl9Cn0=
type: kubernetes.io/dockerconfigjson
EOF
kubectl apply -f harbor-secret.yaml

运行测试

pipeline模式1：简略配置

pipeline{
    agent {
      kubernetes {
        cloud 'kubernetes'
        inheritFrom 'jenkins-jnlp'
        namespace 'devops'
      }
    }
    stages{
        stage('准备 SSH 密钥') {
            steps {
                withCredentials([
                    sshUserPrivateKey(credentialsId: 'jenkins-ssh-key', keyFileVariable: 'SSH_KEY_FILE'),
                    file(credentialsId: 'jenkins-ssh-pub-key', variable: 'SSH_PUB_KEY_FILE')
                ]) {
                    sh '''
                        mkdir -p ~/.ssh
                        chmod 700 ~/.ssh
                        cp ${SSH_KEY_FILE} ~/.ssh/id_rsa
                        cp ${SSH_PUB_KEY_FILE} ~/.ssh/id_rsa.pub
                        chmod 600 ~/.ssh/id_rsa
                        chmod 644 ~/.ssh/id_rsa.pub
                        ssh-keyscan gitlab-svc >> ~/.ssh/known_hosts
                    '''
                }
            }
        }
        stage("准备docker配置"){
            steps{
                withCredentials([file(credentialsId: 'docker-config', variable: 'DOCKERCONFIG')]) {
                    sh '''
                        echo "登录harbor"
                        mkdir -p ~/.docker && cp ${DOCKERCONFIG} ~/.docker/config.json
                        docker login luffy.com
                        docker pull luffy.com/base/nginx:latest
                    '''
                }
            }
        }
        stage("准备k8s配置"){
            steps{
                withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
                    echo "查看 K8S 集群 Pod 列表"
                    sh "mkdir -p ~/.kube && cp ${KUBECONFIG} ~/.kube/config"
                    sh "kubectl get pod"
                }
            }
        }
        stage("拉取代码"){
            steps{
                sh '''
                    rm -rf xzs
                    git clone git@gitlab-svc:root/xzs.git
                '''
            }
        }
        stage('构建镜像'){
            steps{
                sh '''
                    cd xzs
                    docker build -t luffy.com/app/xzs:v1 .
                    docker push luffy.com/app/xzs:v1
                '''
            }
        }
        stage('上传镜像'){
            steps{
                sh '''
                    docker push luffy.com/app/xzs:v1
                '''
            }
        }
    }
}

pipeline模式2：显式yaml配置

pipeline{
    agent {
	    kubernetes{
			cloud 'kubernetes'
			yaml '''
---
apiVersion: "v1"
kind: "Pod"
metadata:
  labels:
    jenkins: "slave"
  namespace: "devops"
spec:
  containers:
  - env:
    - name: "JENKINS_TUNNEL"
      value: "jenkins-svc.devops.svc.cluster.local:50000"
    - name: "JENKINS_AGENT_WORKDIR"
      value: "/home/jenkins/agent"
    - name: "JENKINS_URL"
      value: "http://jenkins-svc.devops.svc.cluster.local:8080/"
    image: "jenkins/inbound-agent"
    imagePullPolicy: "IfNotPresent"
    name: "jnlp"
    volumeMounts:
    - mountPath: "/usr/libexec/docker"
      name: "volume-1"
      readOnly: false
    - mountPath: "/usr/bin/kubectl"
      name: "volume-3"
      readOnly: false
    - mountPath: "/var/run/docker.sock"
      name: "volume-0"
      readOnly: false
    - mountPath: "/etc/docker"
      name: "volume-2"
      readOnly: false
    - mountPath: "/usr/bin/docker"
      name: "volume-4"
      readOnly: false
    - mountPath: "/home/jenkins/agent"
      name: "workspace-volume"
      readOnly: false
    workingDir: "/home/jenkins/agent"
  imagePullSecrets:
  - name: "harbor-secret"
  nodeSelector:
    kubernetes.io/os: "linux"
  securityContext:
    runAsGroup: 0
    runAsUser: 0
  serviceAccountName: "jenkins-admin"
  volumes:
  - hostPath:
      path: "/var/run/docker.sock"
    name: "volume-0"
  - hostPath:
      path: "/etc/docker"
    name: "volume-2"
  - hostPath:
      path: "/usr/libexec/docker"
    name: "volume-1"
  - emptyDir:
      medium: ""
    name: "workspace-volume"
  - hostPath:
      path: "/usr/bin/docker"
    name: "volume-4"
  - hostPath:
      path: "/usr/bin/kubectl"
    name: "volume-3"
'''
		}
	}
    stages{
        stage('准备SSH密钥') {
            steps {
                withCredentials([
                    sshUserPrivateKey(credentialsId: 'jenkins-ssh-key', keyFileVariable: 'SSH_KEY_FILE'),
                    file(credentialsId: 'jenkins-ssh-pub-key', variable: 'SSH_PUB_KEY_FILE')
                ]) {
                    sh '''
                        mkdir -p ~/.ssh
                        chmod 700 ~/.ssh
                        cp ${SSH_KEY_FILE} ~/.ssh/id_rsa
                        cp ${SSH_PUB_KEY_FILE} ~/.ssh/id_rsa.pub
                        chmod 600 ~/.ssh/id_rsa
                        chmod 644 ~/.ssh/id_rsa.pub
                        ssh-keyscan gitlab-svc >> ~/.ssh/known_hosts
                    '''
                }
            }
        }
        stage("准备docker配置"){
            steps{
                withCredentials([file(credentialsId: 'docker-config', variable: 'DOCKERCONFIG')]) {
                    sh '''
                        echo "登录harbor"
                        mkdir -p ~/.docker && cp ${DOCKERCONFIG} ~/.docker/config.json
                        docker login luffy.com
                        docker pull luffy.com/base/nginx:latest
                    '''
                }
            }
        }
        stage("准备k8s配置"){
            steps{
                withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
                    echo "查看 K8S 集群 Pod 列表"
                    sh "mkdir -p ~/.kube && cp ${KUBECONFIG} ~/.kube/config"
                    sh "kubectl get pod"
                }
            }
        }
        stage("拉取代码"){
            steps{
                sh '''
                    rm -rf xzs
                    git clone git@gitlab-svc:root/xzs.git
                '''
            }
        }
        stage('构建镜像'){
            steps{
                sh '''
                    cd xzs
                    docker build -t luffy.com/app/xzs:v1 .
                    docker push luffy.com/app/xzs:v1
                '''
            }
        }
        stage('上传镜像'){
            steps{
                sh '''
                    docker push luffy.com/app/xzs:v1
                '''
            }
        }
    }
}

部署harbor

省略

部署nexus

更新: 2024-09-26 11:02:26