Harbor镜像仓库是一个开源的企业级Docker镜像仓库管理系统，它提供了可靠的镜像存储、赋值、扫描和标记等功能，同时支持用户身份验证、访问控制和日志记录等安全特性。Harbor可以帮助组织和团队管理和发布Docker镜像，并提供了用户友好的Web界面和RestfulAPI供用户操作和管理。

        Ansible是一个自动化工具，用于配置管理、应用程序部署、编排和协调多个服务器。它无需在目标服务器上安装额外的客户端软件，使用基于YAML的语法来描述和执行任务，使其变得简单，可读性高且易于维护。无论是小规模的部署还是大规模的基础架构，ansible都是一种高效的自动化解决方案。

        Ansible的核心概念是playbook，它是一个用于描述目标主机上一系列任务的文本文件，包括变量、模块、任务和处理器等元素，用于指定主机配置和执行顺序。通过编写playbook，可以定义一组操作，并通过Ansible来自动化执行操作，实现系统配置、软件部署等任务。

部署Harbor

部署机安装Ansible

sudo yum install epel-release
sudo yum update
sudo yum install ansible

加载Harbor部署需要的镜像

# 解压other-images.tar.gz，
tar zxvf other-images.tar.gz
cd other-images
./harbor_init_registry.sh xxx.ccr.ctyun.cn:443

#harbor_init_registry.sh

#!/bin/bash

# Check arguments.
if [ $# -lt 1 ];
then
        echo "Please enter the endpoint of the local registry to be pushed"
        exit
fi

# The 1st argument is local registry endpoint.
registry=$1

# Get the folder where the image file is located
dir=$(dirname "$0")
if [ $# -gt 1 ];
then
        dir=$2
fi

# List image files.
for file in `ls $dir`; do
        # Only *.tar files
        if echo "$file" | grep -q -E '\.tar$'
        then
                # Load image.
                docker load -i $file
                # Remove `.tar`.
                prefix=${file%\.tar}
                # Convert to array.
                array=(${prefix//+/ })
                num=${#array[@]}
                # Generate image repo:tag.
                image=""
                for ((j=1;j<num-1;j++))
                {
                        image=$image"/"${array[j]};
                }
                image=$image":"${array[num-1]}
                # Get source and target image.
                array=(${array[0]//_/ })
                result="${array[0]}:${array[1]}"
                source=$result$image
                target=$registry$image
                # Rename image.
                docker tag $source $target
                # Push image.
                docker push $target
        fi
done

配置s3cfg

       s3cfg是一个配置文件，用于配置和管理与Amazon S3对象存储服务的连接，s3是一种可扩展的云存储解决方案，提供了可靠的数据存储和访问服务。s3cfg文件包括连接到s3服务需要的认证依据、区域设置和其他配置选项，通过编辑s3cfg文件，可以指定要使用的密钥及权限等。

       s3cfg文件可以通过命令行工具或其他支持S3存储服务的工具来加载和使用，通过读取s3cfg文件的配置信息，工具可以建立与s3服务的连接，执行上传文件，下载文件，创建存储桶等操作。

vi /root/.s3cfg

[default]
access_key = XXX
secret_key = XXX
host_base = http://0.0.0.0
host_bucket = http://0.0.0.0
use_https = False

yum install s3cmd
s3cmd mb s3://harbor

部署Harbor服务

# 解压ansible-harbor.tar.gz
tar -zxvf ansible-harbor.tar.gz

# 修改ansible-harbor/hosts/host.yaml，参考下面的示例，注意这里的MINIO_SK要经过base64编码
#echo -n nhDSIDX0ae28uanldfg5HP7cDjKYmwgfd6hMAnT | base64
#bmhEU0lEWDBhZTI4dWFubgrgfgdgUDdjRGpLWWfgdf13bDZoTUFuVA==

all:
  hosts:
    cce-master3:
      ansible_host: 33.1.0.4
  vars:
    IMAGE_REGION: xxx
    MINIO_AK: "0X7J2LLUUBH8ZHDNKWWL"
    MINIO_SK: "bmhEU0lEWDBhZTI4dWFubgrgfgdgUDdjRGpLWWfgdf13bDZoTUFuVA=="
    REGION_ENDPOINT: "http://100.123.89.1"

# 在ansible-harbor目录下执行
ansible-playbook -i hosts/host.yaml deploy.yaml

# deploy.yaml
- hosts: cce-master3
  become: yes
  tasks:
    - name: copy local-path storage template
      template:
        src: "templates/local-path.yml.j2"
        dest: "/tmp/local-path.yaml"

    - name: Replace text in local-path.yaml
      replace:
        path: /tmp/local-path.yaml
        regexp: 'IMAGE_REGION'
        replace: '{{ IMAGE_REGION }}'

    - name: deploy local-path storage
      command: /bin/bash
      args:
        stdin: "kubectl apply -f /tmp/local-path.yaml"

    - name: set local-path as default storage class
      command: /bin/bash
      args:
        stdin: "kubectl patch storageclass local-path -p '{\"metadata\": {\"annotations\":{\"storageclass.kubernetes.io/is-default-class\":\"true\"}}}'"

    - name: wait to be ready for local-path storage
      shell: kubectl get pod -n local-path-storage | grep "1/1"
      register: result
      until: result.rc == 0
      retries: 10
      delay: 1


    - name: copy cert-manager template
      template:
        src: "templates/cert-manager.yml.j2"
        dest: "/tmp/cert-manager.yaml"

    - name: Replace text in cert-manager.yaml
      replace:
        path: /tmp/cert-manager.yaml
        regexp: 'IMAGE_REGION'
        replace: '{{ IMAGE_REGION }}'

    - name: deploy cert-manager
      shell: kubectl apply -f /tmp/cert-manager.yaml

    - name: wait to be ready for cert-manager
      shell: kubectl get secret -n cert-manager | grep cert-manager-webhook-ca
      register: result
      until: result.rc == 0
      retries: 10
      delay: 5


    - name: copy harbor-operator template
      template:
        src: "templates/operator.yml.j2"
        dest: "/tmp/deployment.yaml"

    - name: Replace text in deployment.yaml
      replace:
        path: /tmp/deployment.yaml
        regexp: 'IMAGE_REGION'
        replace: '{{ IMAGE_REGION }}'

    - name: deploy harbor operator
      shell: kubectl apply -f /tmp/deployment.yaml

    - name: wait to be ready for harbor operator
      shell: kubectl get pod -n harbor-operator | grep "1/1"
      register: result
      until: result.rc == 0
      retries: 10
      delay: 1


    - name: copy redis template
      template:
        src: "templates/redis.yml.j2"
        dest: "/tmp/redis.yaml"

    - name: Replace text in redis.yaml
      replace:
        path: /tmp/redis.yaml
        regexp: 'IMAGE_REGION'
        replace: '{{ IMAGE_REGION }}'

    - name: deploy redis
      shell: kubectl apply -f /tmp/redis.yaml

#    - name: wait to be ready for redis
#      shell: kubectl get pod -n harbor | grep "1/1"
#      register: result
#      until: result.rc == 0
#      retries: 10
#      delay: 2

    - name: copy postgres template
      template:
        src: "templates/postgres.yml.j2"
        dest: "/tmp/postgres.yaml"

    - name: Replace text in postgres.yaml
      replace:
        path: /tmp/postgres.yaml
        regexp: 'IMAGE_REGION'
        replace: '{{ IMAGE_REGION }}'

    - name: deploy postgres
      shell: kubectl apply -f /tmp/postgres.yaml

    - name: wait to be ready for postgres
      shell: kubectl get secret harbor.postgresql-harbor-harborcluster.credentials -n harbor -o yaml | grep password | grep -v f | awk '{ print $2 }'
      register: result
      until: result.stdout != ""
      retries: 10
      delay: 5


    - name: debug password variable
      debug:
        var: result


    - name: copy password template
      template:
        src: "templates/password.yml.j2"
        dest: "/tmp/password.yaml"

    - name: Replace text in password.yaml
      replace:
        path: /tmp/password.yaml
        regexp: 'PASSWORD'
        replace: '{{ result.stdout }}'

    - name: deploy password
      shell: kubectl apply -f /tmp/password.yaml


    - name: copy harborcluster template
      template:
        src: "templates/harborcluster.yml.j2"
        dest: "/tmp/harborcluster.yaml"

    - name: Replace AK
      replace:
        path: /tmp/harborcluster.yaml
        regexp: 'MINIO_AK'
        replace: '{{ MINIO_AK }}'

    - name: Replace SK
      replace:
        path: /tmp/harborcluster.yaml
        regexp: 'MINIO_SK'
        replace: '{{ MINIO_SK }}'

    - name: Replace REGION_ENDPOINT
      replace:
        path: /tmp/harborcluster.yaml
        regexp: 'REGION_ENDPOINT'
        replace: '{{ REGION_ENDPOINT }}'

    - name: Replace IMAGE_REGION
      replace:
        path: /tmp/harborcluster.yaml
        regexp: 'IMAGE_REGION'
        replace: '{{ IMAGE_REGION }}'

    - name: deploy harbor cluster
      shell: kubectl apply -f /tmp/harborcluster.yaml

镜像仓库迁移

预制区服务的镜像仓库需要从部署机的register迁移到master的harbor，这样预制区的kubernetes依赖的镜像就可以从harbor仓库获取

# 获取harbor根证书公钥(tls.crt)，经过base64解码后替换部署机的/etc/docker/certs.d/xxx.ccr.ctyun.cn:443/ca.crt
kubectl get secret -n harbor harbor-ca -o yaml
# 进入kubespray源码目录，执行如下命令：
docker run --rm -it -v "$(pwd)":/kubespray -v /etc/docker/certs.d:/etc/docker/certs.d -v "${HOME}"/.ssh/id_rsa:/root/.ssh/id_rsa quay.io/kubespray/kubespray:v2.17.1 bash
# 替换集群内所有节点的镜像仓库证书与地址
ansible-playbook -i inventory/host.yaml -e kube_version=v1.20.2 -e 'ansible_python_interpreter=/usr/bin/python3' -e registry_host=xxx.ccr.ctyun.cn:443/caas post-cluster.yml

验证与创建项目

# 登录到deploy节点，配置hosts指向部署了部署Harbor的节点地址，这里把之前指向deploy的删掉
33.1.0.8 xxx.ccr.ctyun.cn
# docker login
docker login --username=admin xxx.ccr.ctyun.cn:443
password:XXX

创建caas项目并推送镜像到仓库

# 创建harbor仓库下caas项目
curl -kv -X POST https://xxx.ccr.ctyun.cn:443/api/v2.0/projects -d '{"project_name": "caas", "metadata": {"public": "true"}}' -u admin:XX... -H "Content-Type: application/json"

# 把部署机上面的镜像推送到harbor镜像仓库，执行如下的脚本文件docker-push.sh

#!/bin/bash

images=$(docker images | grep xxx)

# 遍历镜像列表，将每个镜像上传到镜像仓库
while IFS= read -r image; do
  # 提取镜像名称和标签
  repo=$(echo "$image" | awk '{ print $1 }')
  tag=$(echo "$image" | awk '{ print $2 }')

  # 上传镜像
  echo "正在上传镜像 $repo:$tag 到镜像仓库..."
  docker push "$repo:$tag"
done <<< "$images"

echo "镜像上传完成。"