Ankur Parashar
Articles by Ankur Parashar

Follow

Articles by Ankur Parashar

Follow
Continuous Gitlab backup and restore using rsync

Continuous Gitlab backup and restore using rsync

Ankur Parashar's photo
Ankur Parashar
·Nov 29, 2022·

5 min read

This article covers backup and restore guidelines for community version of gitlab.

Objective:

Step 1: Configure machine

sudo apt-get update

Step 2: Create LVM on machine for snapshot

# Install GUI partition software

$ sudo apt-get install gparted

# Display all disk partitions

$ sudo fdisk -l

$ sudo vgcreate ubuntu /dev/sda5

$ sudo lvcreate — extents 100%FREE — name lvgitlab ubuntu

$ sudo lvresize — size -100G /dev/mapper/ubuntu-lvgitlab

$ sudo lvcreate — size 14.5G — name lvgitlabswap ubuntu

$ sudo mkfs.ext4 /dev/ubuntu/lvgitlab

$ sudo mount /dev/ubuntu/lvgitlab /var/opt/gitlab/

Note: Guide Guide

Step 3: Configure LVM to mount permanent

$ sudo vi /etc/lvm/lvm.conf

# update use_lvmetad setting from 1 to 0

use_lvmetad = 0

Step 4: Install Gitlab: Installation of gitlab over Ubuntu is very easy just follow the steps provided in official documentation. https://about.gitlab.com/installation/#ubuntu or steps here.

Step 5: Configure Gitlab:

# Open gitlab configuration file

$ sudo nano /etc/gitlab/gitlab.rb

## Backup settings

gitlab_rails[‘manage_backup_path’] \= true

gitlab_rails[‘backup_path’] = “/mnt/gitlabdb-backup”
gitlab_rails[‘backup_archive_permissions’] = 0644

gitlab_rails[‘backup_keep_time’] = 604800

## GitLab user privileges

gitlab_rails[‘gitlab_default_can_create_group’] = false

gitlab_rails[‘gitlab_username_changing_enabled’] = false

manage_accounts[‘enable’] = false

## Default project feature settings

gitlab_rails[‘gitlab_default_projects_features_issues’] = false

gitlab_rails[‘gitlab_default_projects_features_merge_requests’] = false

gitlab_rails[‘gitlab_default_projects_features_wiki’] = false

gitlab_rails[‘gitlab_default_projects_features_snippets’] = false

gitlab_rails[‘gitlab_default_projects_features_builds’] = false

gitlab_rails[‘gitlab_default_projects_features_container_registry’] = false

## LDAP settings

gitlab_rails[‘ldap_enabled’] = true

gitlab_rails[‘ldap_servers’] = YAML.load <<-’EOS’

main: # ‘main’ is the GitLab ‘provider ID’ of this LDAP server

label: ‘LDAP’

host: ‘’

port: 389

uid: ‘sAMAccountName’

bind_dn: ‘\’

password: ‘’

encryption: ‘plain’ # “start_tls” or “simple_tls” or “plain”

verify_certificates: true

active_directory: true

allow_username_or_email_login: true

block_auto_created_users: false

base: ‘’

user_filter: ‘’

# ## EE only

# group_base: ‘’

# admin_group: ‘’

# sync_ssh_keys: false

# Gitlab reconfigure

$ sudo gitlab-ctl reconfigure

# Disable SignUp

Admin Area > Settings > Sign-up Restrictions : Uncheck Sign-up enabled → Save

# Update Admin Profile:

User Settings > Account > Change username : replace root with gitlab.admin

User Settings > Edit Profile > Main settings > Email : ****@

Step 4: Backup:

$ sudo mkdir /mnt/gitlab-backup

$ sudo mkdir /mnt/gitlabdb-backup

Copy following scripts inside this folder:

$ sudo mkdir gitlab-scripts

# Weekly backup: ftp_backup.bash

# Daily backup: remotegitlab_backup.bash

# Daily Mail flush: postfix_flush.bash

# Daily Old files flush: clearold_backup.bash

$ chmod +x /home/ubuntu/gitlab-scripts/ftp_backup.bash

$ chmod +x /home/ubuntu/gitlab-scripts/remotegitlab_backup.bash

$ chmod +x /home/ubuntu/gitlab-scripts/postfix_flush.bash

$ chmod +x /home/ubuntu/gitlab-scripts/clearold_backup.bash

$ sudo crontab -e

# Configure crons

## weekly backup

30 6 * * 6 /home/ubuntu/gitlab-scripts/ftp_backup.bash &> /dev/null

## daily backup

30 1 * * * /home/ubuntu/gitlab-scripts/remotegitlab_backup.bash &> /dev/null

## daily flush

0 7 * * * /home/ubuntu/gitlab-scripts/postfix_flush.bash &> /dev/null

## weekly clear

10 7 * * * /home/ubuntu/gitlab-scripts/clearold_backup.bash &> /dev/null

Step 4: Setup Mail Server:

  • Follow guidelines from here

Setup postfix as relay server, for creating app follow this link:

https://linode.com/docs/email/postfix/postfix-smtp-debian7/

$ sudo vi /etc/postfix/sasl/sasl_passwd

# Copy paste following (change app-password to account app password)

smtp.gmail.com]:587 noreply@.com:[app-password]

# Create the hash db file for Postfix by running the postmap command:

sudo postmap /etc/postfix/sasl_passwd

# Run the following commands to change the ownership to root and update the permissions for the two files:

sudo chown root:root /etc/postfix/sasl/sasl_passwd /etc/postfix/sasl/sasl_passwd.db

sudo chmod 0600 /etc/postfix/sasl/sasl_passwd /etc/postfix/sasl/sasl_passwd.db

# Configure postfix:

sudo vi /etc/postfix/main.cf

# Add specify SMTP relay host

relayhost =[smtp.gmail.com]:587

Add following at the end of file:

# enable SASL authentication

smtp_sasl_auth_enable = yes

# disallow methods that allow anonymous authentication.

smtp_sasl_security_options = noanonymous

# where to find sasl_passwd

smtp_sasl_password_maps = hash:/etc/postfix/sasl/sasl_passwd

# Enable STARTTLS encryption

smtp_use_tls = yes

# where to find CA certificates

smtp_tls_CAfile = /etc/ssl/certs/ca-certificates.crt

That’s it now restart postfix

sudo service postfix restart

# Testing Postfix

echo “This is the body of the email” | mail -s “This is the subject line”

Step 5: Check the permissions of following folders:

# Git repositories directory

/var/opt/gitlab/git-data 0700 git:root

# Git repositories

/var/opt/gitlab/git-data/repositories 2770 git:git

# Holds large object directories

/var/opt/gitlab/gitlab-rails/shared 0751 git:gitlab-www

# Holds CI artifacts

/var/opt/gitlab/gitlab-rails/shared/artifacts 0700 git:root

# Holds LFS objects

/var/opt/gitlab/gitlab-rails/shared/lfs-objects 0700 git:root

# Holds user attachments

/var/opt/gitlab/gitlab-rails/uploads 0700 git:root

# Holds user pages

/var/opt/gitlab/gitlab-rails/shared/pages 0750 git:gitlab-www

# Holds CI build logs

/var/opt/gitlab/gitlab-ci/builds 0700 git:root

# Holds authorized keys

/var/opt/gitlab/.ssh 0700 git:git

Step 7: Configure routing as git..com

$ ssh ankur@

$ sudo vi /etc/hosts

# Add following

git..com

# Create nginx configuration file

$ sudo vi /etc/nginx/sites-available/git

# update: server_name, proxy_pass

server {

listen *:80;

server_name http://git..com;

client_max_body_size 200M;

location / {

proxy_set_header Host $host;

proxy_set_header X-Real-IP $remote_addr;

proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

proxy_pass http://git..com;

}

}

# Enable nginx site configuration

$ sudo ln -s /etc/nginx/sites-available/git /etc/nginx/sites-enabled/git

$ sudo service nginx reload

Step 6: Restore

This step is only needed in-case if we are setting up new gitlab server for local network and would like to restore it from remote gitlab server.

# Restore from remote gitlab server script

$ sudo vi restore_gitlab.bash

#! bin/bash

$ chmod +x restore_gitlab.bash

$ ./restore_gitlab.bash

Note: Open Gitlab in browser and check whether everything is working correctly or not.

clearold_backup.sh

#! /bin/bash

# Path : /home/ubuntu/gitlab-scripts/clearold_backup.bash

# clear db files older than 5 days

sudo find ‘/mnt/gitlabdb-backup/’ -mtime +4 -type f -delete

local_backup_restore.sh

#!/bin/bash

# Path: /home/ubuntu/gitlab-scripts/local_backup_restore.bash

# Resync backups to local gitlab server

[[ $( whoami ) != ‘root’ ]] && echo “sudo please” && exit 1

set -o errexit

set -o nounset

# Remote path

postfix_flush.bash

#!/bin/bash

# Path: /home/ubuntu/gitlab-scripts/postfix_flush.bash

sudo systemctl restart postfix

postqueue -f

restore_backup.sh

#!/bin/bash

# Backup tar file for restore

backup_path=/mnt/gitlabdb-backup/

BACKUPFILE=`ls -t $backup_path | head -n 1`

# gitlab backup git-data path

git_data=/mnt/gitlab-backup/git-data

# copy backup git-data to /var/opt/gitlab/git-data

echo “copying $git_data to /var/opt/gitlab” >&2

#sudo cp -r $git_data /var/opt/gitlab/

#sudo sh -c ‘rsync — archive — verbose — bwlimit=50M /mnt/gitlab-backup/git-data/repositories/. /var/opt/gitlab/git-data/repositories’

#sudo chown -R git:git /var/opt/gitlab/git-data

echo “Start Copying”

# copy gitlab db backup tar

sudo cp $backup_path$BACKUPFILE /var/opt/gitlab/backups/

sudo chown -R git:git /var/opt/gitlab/backups/

echo “Restarting Gitlab”

sudo gitlab-ctl stop unicorn

sudo gitlab-ctl stop sidekiq

fname=”${BACKUPFILE%_gitlab_backup.tar}”

sudo gitlab-rake gitlab:backup:restore BACKUP=$fname force=yes

sudo gitlab-ctl restart

echo “DONE”

 
Share this