Gitlab-异库同步

Gitlab-异库同步

结构

├── gitlab_sync_ignore.lst      # 不同步的库
├── gitlab_sync_lib.py          # 计算本次需同步的库，分全量和增量
├── gitlab_sync.sh              # 同步脚本
├── gitlab_sync_old.lst
├── gitlab_sync_to_full.lst     # 全同步列表，脚本生成
├── gitlab_sync_to_inc.lst      # 增量同步列表，脚本生成

gitlab_sync.sh

#!/bin/bash

ROOT_PATH=/home/gitlab_sync
cd $ROOT_PATH

git_url1="192.168.8.1:8090"
git_user1=jenkins
git_key1=xxxxxxxxxxx

git_url2="10.249.2.40:8690"
git_user2=abc
git_key2=xxxxxxxxxxxx

git_group2='aa/bb/10/operations'

mkdir -p $ROOT_PATH/repo_full
mkdir -p $ROOT_PATH/repo_inc

rm -f $ROOT_PATH/gitlab_sync_to_full.lst
rm -f $ROOT_PATH/gitlab_sync_to_inc.lst

$ROOT_PATH/gitlab_sync_lib.py $git_url1 $git_key1

if [ -f "$ROOT_PATH/gitlab_sync_to_full.lst" ]; then

cat $ROOT_PATH/gitlab_sync_to_full.lst | while read row;do

  gitname1=$(echo $row |awk -F ":8090/" '{print $2}')
  gitname2=$(echo $gitname1| sed 's/\//-/g')

  echo "***start full sync ... $row"
  rm -rf $ROOT_PATH/repo_full/$gitname2
  git clone --bare http://${git_user1}:${git_key1}@${git_url1}/$gitname1 $ROOT_PATH/repo_full/$gitname2
  cd $ROOT_PATH/repo_full/$gitname2
  git push --mirror http://${git_user2}:${git_key2}@${git_url2}/${git_group2}/$gitname2
  cd $ROOT_PATH

  rm -rf $ROOT_PATH/repo_inc/$gitname2
  git clone http://${git_user1}:${git_key1}@${git_url1}/$gitname1 $ROOT_PATH/repo_inc/$gitname2
  echo $row >> $ROOT_PATH/gitlab_sync_old.lst

cd $ROOT_PATH
done
fi

if [ -f "$ROOT_PATH/gitlab_sync_to_inc.lst" ]; then

cat $ROOT_PATH/gitlab_sync_to_inc.lst | while read row;do

  gitname1=$(echo $row |awk -F ":8090/" '{print $2}')
  gitname2=$(echo $gitname1| sed 's/\//-/g')

  echo "***start inc sync ... $row"
  cd $ROOT_PATH/repo_inc/$gitname2
  git branch -r | grep -v '\->' | while read remote; do git branch --track "${remote#origin/}" "$remote"; done
  git fetch --all
  git branch -r  |grep -v HEAD |awk -F"origin/" '{print $2}' | while read repo; do git checkout $repo;git pull --rebase; done
  git pull --all --rebase
  git push --all http://${git_user2}:${git_key2}@${git_url2}/${git_group2}/$gitname2

cd $ROOT_PATH
done
fi

gitlab_sync_lib.py

#!/usr/bin/python3
import gitlab
import time
import os
import sys
import pandas as pd

repo_url = "http://" + sys.argv[1]
token = sys.argv[2]

df1 = pd.DataFrame(columns=['url'])
g1 = gitlab.Gitlab(repo_url, private_token=token)
projects = g1.projects.list(all=True)
for project in projects:
     df1 = df1.append({'url':project.http_url_to_repo}, ignore_index=True)

if os.path.isfile("gitlab_sync_ignore.lst"):
  df2 = pd.read_csv("gitlab_sync_ignore.lst",sep=' ',header=None)
  df2.rename(columns={df2.columns[0]:"url"}, inplace=True)
  ##df1-df2
  df3 = pd.concat([df1, df2, df2]).drop_duplicates(keep=False)
else:
  df3 = df1

if os.path.isfile("gitlab_sync_old.lst"):
  df4 = pd.read_csv("gitlab_sync_old.lst",sep=' ',header=None)
  df4.rename(columns={df4.columns[0]:"url"}, inplace=True)
  ##df5=df3-df4,df4=df3-df5
  df5 = pd.concat([df3, df4, df4]).drop_duplicates(keep=False)
  df4 = pd.concat([df3, df5, df5]).drop_duplicates(keep=False)
  df4.to_csv("gitlab_sync_to_inc.lst",index=0,header=0)
  df5.to_csv("gitlab_sync_to_full.lst",index=0,header=0)
else:
  df5 = df3
  df5.to_csv("gitlab_sync_to_full.lst",index=0,header=0)