本文是基于centos7.6 版使用ceph-deploy 工具 安装ceph nautilus 版本。
[root@ceph ~]# cat /etc/centos-release
CentOS Linux release 7.6.1810 (Core)
[root@ceph ~]# uname -a
Linux ceph.novalocal 3.10.0-957.el7.x86_64 #1 SMP Thu Nov 8 23:39:32 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux
[root@ceph ~]#
主机名 | IP | 磁盘 | 角色 |
---|---|---|---|
ceph001 | 172.31.185.127 | 系统盘:/dev/vda 数据盘:/dev/vdb | 时间服务器,ceph-deploy,monitor,mgr,mds,osd |
ceph002 | 172.31.185.198 | 系统盘:/dev/vda 数据盘:/dev/vdb | monitor,mgr,mds,osd |
ceph003 | 172.31.185.203 | 系统盘:/dev/vda 数据盘:/dev/vdb | monitor,mgr,mds,osd |
以下安装,若无特殊说明,三台节点均需要安装。
以172.31.185.127 为例子
[root@ceph ~]# hostnamectl set-hostname ceph001
[root@ceph ~]#
PS:
172.31.185.198 hostnamectl set-hostname ceph002
172.31.185.203 hostnamectl set-hostname ceph003
[root@ceph ~]# systemctl stop firewalld | systemctl disable firewalld | systemctl status firewalld
● firewalld.service - firewalld - dynamic firewall daemon
Loaded: loaded (/usr/lib/systemd/system/firewalld.service; disabled; vendor preset: enabled)
Active: inactive (dead)
Docs: man:firewalld(1)
[root@ceph ~]#
[root@ceph001 ~]# cp /etc/selinux/config /etc/selinux/config.bak.orig
[root@ceph001 ~]# sed -i ‘s/^SELINUX=.*/SELINUX=disabled/‘ /etc/selinux/config
[root@ceph001 ~]# cat /etc/selinux/config
# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
# enforcing - SELinux security policy is enforced.
# permissive - SELinux prints warnings instead of enforcing.
# disabled - No SELinux policy is loaded.
SELINUX=disabled
# SELINUXTYPE= can take one of these three values:
# targeted - Targeted processes are protected,
# minimum - Modification of targeted policy. Only selected processes are protected.
# mls - Multi Level Security protection.
SELINUXTYPE=targeted
[root@ceph001 ~]#
[root@ceph001 ~]# cp /etc/hosts /etc/hosts.bak.orig
[root@ceph001 ~]# vi /etc/hosts
新增
172.31.185.127 ceph001
172.31.185.198 ceph002
172.31.185.203 ceph003
[root@ceph001 ~]# reboot
因为centos7 一般默认安装了chrony ,若没有安装直接 yum install chrony
172.31.185.127 作为时间服务器server
[root@ceph001 ~]# cp /etc/chrony.conf /etc/chrony.conf.bak.orig
根据操作系统版本以及需要安装的ceph 版本配置ceph 源
vim ceph.repo
[Ceph]
name=Ceph packages for $basearch
baseurl=http://mirrors.aliyun.com/ceph/rpm-nautilus/el7/$basearch
enabled=1
gpgcheck=0
type=rpm-md
gpgkey=https://mirrors.aliyun.com/ceph/keys/release.asc
priority=1
[Ceph-noarch]
name=Ceph noarch packages
baseurl=http://mirrors.aliyun.com/ceph/rpm-nautilus/el7/noarch
enabled=1
gpgcheck=0
type=rpm-md
gpgkey=https://mirrors.aliyun.com/ceph/keys/release.asc
priority=1
[ceph-source]
name=Ceph source packages
baseurl=http://mirrors.aliyun.com/ceph/rpm-nautilus/el7/SRPMS
enabled=1
gpgcheck=0
type=rpm-md
gpgkey=https://mirrors.aliyun.com/ceph/keys/release.asc
priority=1
给另外两台机器也配置上
[root@ceph001 yum.repos.d]# scp ceph.repo root@172.31.185.198:/etc/yum.repos.d/
root@172.31.185.198‘s password:
ceph.repo 100% 611 310.3KB/s 00:00
[root@ceph001 yum.repos.d]# scp ceph.repo root@172.31.185.203:/etc/yum.repos.d/
The authenticity of host ‘172.31.185.203 (172.31.185.203)‘ can‘t be established.
ECDSA key fingerprint is SHA256:ES6ytBX1siYV4WMG2CF3/21VKaDd5y27lbWQggeqRWM.
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added ‘172.31.185.203‘ (ECDSA) to the list of known hosts.
root@172.31.185.203‘s password:
Permission denied, please try again.
root@172.31.185.203‘s password:
ceph.repo 100% 611 312.3KB/s 00:00
[root@ceph001 yum.repos.d]#
三台节点都要创建该用户,并设置sudo
[root@ceph001 yum.repos.d]# useradd cephadmin
[root@ceph001 ~]# echo "cephnau@2020" | passwd --stdin cephadmin
Changing password for user cephadmin.
passwd: all authentication tokens updated successfully.
[root@ceph001 ~]#
[root@ceph001 yum.repos.d]# echo "cephadmin ALL = (root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/cephadmin
cephadmin ALL = (root) NOPASSWD:ALL
[root@ceph001 yum.repos.d]# chmod 0440 /etc/sudoers.d/cephadmin
[root@ceph001 yum.repos.d]#
此处部署节点是ceph001 节点即ceph001
部署节点上cephadmin 能够免密登录到三个ceph 集群集群 ceph001,ceph002,ceph003
[root@ceph001 ~]# su - cephadmin
Last login: Mon Nov 30 11:35:15 CST 2020 from 172.31.185.198 on pts/1
[cephadmin@ceph001 ~]$ ssh-keygen
Generating public/private rsa key pair.
Enter file in which to save the key (/home/cephadmin/.ssh/id_rsa):
Created directory ‘/home/cephadmin/.ssh‘.
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /home/cephadmin/.ssh/id_rsa.
Your public key has been saved in /home/cephadmin/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:sw1pd339wzApgtVCwDQV4fGq8/oT7hug7P5WK2ifbpE cephadmin@ceph001
The key‘s randomart image is:
+---[RSA 3072]----+
| o+o*o |
| .+ + |
| + o |
| o.o .. .|
| oSo...+. o|
| . .EoO... +..|
| o.o=.+ o.|
| .o +++. .|
| oo**==o |
+----[SHA256]-----+
[cephadmin@ceph001 ~]$ ssh-copy-id cephadmin@ceph001
/usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/home/cephadmin/.ssh/id_rsa.pub"
The authenticity of host ‘ceph001 (172.31.185.127)‘ can‘t be established.
ECDSA key fingerprint is SHA256:ES6ytBX1siYV4WMG2CF3/21VKaDd5y27lbWQggeqRWM.
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
/usr/bin/ssh-copy-id: INFO: attempting to log in with the new key(s), to filter out any that are already installed
/usr/bin/ssh-copy-id: INFO: 1 key(s) remain to be installed -- if you are prompted now it is to install the new keys
cephadmin@ceph001‘s password:
Number of key(s) added: 1
Now try logging into the machine, with: "ssh ‘cephadmin@ceph001‘"
and check to make sure that only the key(s) you wanted were added.
[cephadmin@ceph001 ~]$ ssh-copy-id cephadmin@ceph002
/usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/home/cephadmin/.ssh/id_rsa.pub"
The authenticity of host ‘ceph002 (172.31.185.198)‘ can‘t be established.
ECDSA key fingerprint is SHA256:ES6ytBX1siYV4WMG2CF3/21VKaDd5y27lbWQggeqRWM.
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
/usr/bin/ssh-copy-id: INFO: attempting to log in with the new key(s), to filter out any that are already installed
/usr/bin/ssh-copy-id: INFO: 1 key(s) remain to be installed -- if you are prompted now it is to install the new keys
cephadmin@ceph002‘s password:
Number of key(s) added: 1
Now try logging into the machine, with: "ssh ‘cephadmin@ceph002‘"
and check to make sure that only the key(s) you wanted were added.
[cephadmin@ceph001 ~]$ ssh-copy-id cephadmin@ceph003
在部署节点ceph001 上使用cephadmin 用户安装ceph-deploy
将这些安装包下载到本地,为以后的离线安装做准备。
sudo yum -y install --downloadonly --downloaddir=/home/cephadmin/software/ceph-deploy/ ceph-deploy python-pip
[cephadmin@ceph001 ~]$ ls software/ceph-deploy/
ceph-deploy-2.0.1-0.noarch.rpm python2-pip-8.1.2-12.el7.noarch.rpm
[cephadmin@ceph001 ~]$
[cephadmin@ceph001 ~]$ sudo yum -y install ceph-deploy python-pip
所有ceph 集群节点都需安装
采用yum 方式安装ceph 包,把相关依赖全部下载下来
sudo yum -y install --downloadonly --downloaddir=/home/cephadmin/software/ceph/ ceph ceph-radosgw
三台节点安装ceph
sudo yum -y install ceph ceph-radosgw
在ceph-deploy 部署节点 操作
部署节点上
[cephadmin@ceph001 ~]$ mkdir /home/cephadmin/cephcluster
[cephadmin@ceph001 ~]$ ll
total 0
drwxrwxr-x 2 cephadmin cephadmin 6 Nov 30 16:48 cephcluster
drwxr-xr-x 4 root root 58 Nov 30 16:45 software
[cephadmin@ceph001 ~]$ cd cephcluster/
[cephadmin@ceph001 cephcluster]$ pwd
/home/cephadmin/cephcluster
[cephadmin@ceph001 cephcluster]$ ceph-deploy new ceph001 ceph002 ceph003
[ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /bin/ceph-deploy new ceph001 ceph002 ceph003
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] func : <function new at 0x7f296c15bd70>
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f296bae7950>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] ssh_copykey : True
[ceph_deploy.cli][INFO ] mon : [‘ceph001‘, ‘ceph002‘, ‘ceph003‘]
[ceph_deploy.cli][INFO ] public_network : None
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] cluster_network : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] fsid : None
[ceph_deploy.new][DEBUG ] Creating new cluster named ceph
。。。。
以上可见 本次集群名是ceph
并生成以下文件
[cephadmin@ceph001 cephcluster]$ ll
total 16
-rw-rw-r-- 1 cephadmin cephadmin 247 Nov 30 16:50 ceph.conf
-rw-rw-r-- 1 cephadmin cephadmin 5231 Nov 30 16:50 ceph-deploy-ceph.log
-rw------- 1 cephadmin cephadmin 73 Nov 30 16:50 ceph.mon.keyring
[cephadmin@ceph001 cephcluster]$
PS:
ceph-deploy –cluster {cluster-name} new node1 node2 //创建一个自定集群名称的ceph集群,默
认为 ceph
修改ceph.conf 新增网络配置
[global]
fsid = 69002794-cf45-49fa-8849-faadae48544f
mon_initial_members = ceph001, ceph002, ceph003
mon_host = 172.31.185.127,172.31.185.198,172.31.185.203
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
public network = 172.31.185.0/24
cluster network = 172.31.185.0/24
~
public network = 172.31.185.0/24
cluster network = 172.31.185.0/24
应该使用两个不同的网段,因为我这里只有一张网卡。
比如eth0 ,eth1
[cephadmin@ceph001 cephcluster]$ ceph-deploy mon create-initial #配置初始 monitor(s)、并收集所有密钥
[ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /bin/ceph-deploy mon create-initial
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] subcommand : create-initial
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f0a8fa7c0e0>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] func : <function mon at 0x7f0a8fcdf398>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] keyrings : None
[ceph_deploy.mon][DEBUG ] Deploying mon, cluster ceph hosts ceph001 ceph002 ceph003
生成的密钥
[cephadmin@ceph001 cephcluster]$ ls -l *.keyring
-rw------- 1 cephadmin cephadmin 113 Nov 30 17:17 ceph.bootstrap-mds.keyring
-rw------- 1 cephadmin cephadmin 113 Nov 30 17:17 ceph.bootstrap-mgr.keyring
-rw------- 1 cephadmin cephadmin 113 Nov 30 17:17 ceph.bootstrap-osd.keyring
-rw------- 1 cephadmin cephadmin 113 Nov 30 17:17 ceph.bootstrap-rgw.keyring
-rw------- 1 cephadmin cephadmin 151 Nov 30 17:17 ceph.client.admin.keyring
-rw------- 1 cephadmin cephadmin 73 Nov 30 16:50 ceph.mon.keyring
[cephadmin@ceph001 cephcluster]$ pwd
/home/cephadmin/cephcluster
[cephadmin@ceph001 cephcluster]$
配置信息将拷贝到各节点/etc/ceph 目录下
[cephadmin@ceph001 cephcluster]$ ceph-deploy admin ceph001 ceph002 ceph003 #配置信息拷贝到三台节点
检查是否可以
切换到root 账号
[root@ceph001 ~]# ceph -s
cluster:
id: 69002794-cf45-49fa-8849-faadae48544f
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph001,ceph002,ceph003 (age 10m)
mgr: no daemons active
osd: 0 osds: 0 up, 0 in
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs:
[root@ceph001 ~]#
[root@ceph002 ~]# ceph -s
cluster:
id: 69002794-cf45-49fa-8849-faadae48544f
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph001,ceph002,ceph003 (age 12m)
mgr: no daemons active
osd: 0 osds: 0 up, 0 in
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs:
[root@ceph002 ~]#
如果想使用cephadmin 账号执行ceph -s,则需要修改/etc/ceph 目录权限
[root@ceph001 ~]# su - cephadmin
Last login: Mon Nov 30 16:42:45 CST 2020 on pts/0
[cephadmin@ceph001 ~]$ ceph -s
[errno 2] error connecting to the cluster
[cephadmin@ceph001 ~]$ ll /etc/ceph/
total 12
-rw------- 1 root root 151 Nov 30 17:25 ceph.client.admin.keyring
-rw-r--r-- 1 root root 313 Nov 30 17:25 ceph.conf
-rw-r--r-- 1 root root 92 Nov 24 03:33 rbdmap
-rw------- 1 root root 0 Nov 30 17:16 tmp10a3zI
[cephadmin@ceph001 ~]$ sudo chown -R cephadmin:cephadmin /etc/ceph
[cephadmin@ceph001 ~]$ ceph -s
cluster:
id: 69002794-cf45-49fa-8849-faadae48544f
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph001,ceph002,ceph003 (age 14m)
mgr: no daemons active
osd: 0 osds: 0 up, 0 in
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs:
[cephadmin@ceph001 ~]$
三个节点都需要执行 sudo chown -R cephadmin:cephadmin /etc/ceph
首先查看数据盘名称,三个节点都要看
[root@ceph001 ~]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sr0 11:0 1 478K 0 rom
vda 253:0 0 50G 0 disk
├─vda1 253:1 0 200M 0 part /boot
└─vda2 253:2 0 49.8G 0 part /
vdb 253:16 0 50G 0 disk
可见我数据盘时vdb
我这里写了各脚本,该脚本需要在/home/cephadmin/cephcluster 目录下执行
for dev in /dev/vdb
do
ceph-deploy disk zap ceph001 $dev
ceph-deploy osd create ceph001 --data $dev
ceph-deploy disk zap ceph002 $dev
ceph-deploy osd create ceph002 --data $dev
ceph-deploy disk zap ceph003 $dev
ceph-deploy osd create ceph003 --data $dev
done
[cephadmin@ceph001 cephcluster]$ for dev in /dev/vdb
> do
> ceph-deploy disk zap ceph001 $dev
> ceph-deploy osd create ceph001 --data $dev
> ceph-deploy disk zap ceph002 $dev
> ceph-deploy osd create ceph002 --data $dev
> ceph-deploy disk zap ceph003 $dev
> ceph-deploy osd create ceph003 --data $dev
> done
[ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /bin/ceph-deploy disk zap ceph001 /dev/vdb
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] debug : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] subcommand : zap
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7fcd19fcd8c0>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] host : ceph001
[ceph_deploy.cli][INFO ] func : <function disk at 0x7fcd1a21c8c0>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] disk : [‘/dev/vdb‘]
[ceph_deploy.osd][DEBUG ] zapping /dev/vdb on ceph001
[ceph001][DEBUG ] connection detected need for sudo
检查命令是否执行成功
[cephadmin@ceph001 cephcluster]$ ceph -s
cluster:
id: 69002794-cf45-49fa-8849-faadae48544f
health: HEALTH_WARN
no active mgr
services:
mon: 3 daemons, quorum ceph001,ceph002,ceph003 (age 26m)
mgr: no daemons active
osd: 3 osds: 3 up (since 4m), 3 in (since 4m)
## 我这里时三个节点,每个节点一块数据盘,可见已成功
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs:
[cephadmin@ceph001 cephcluster]$
[cephadmin@ceph001 cephcluster]$ ceph-deploy mgr create ceph001 ceph002 ceph003
[cephadmin@ceph001 cephcluster]$ ceph -s
cluster:
id: 69002794-cf45-49fa-8849-faadae48544f
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph001,ceph002,ceph003 (age 30m)
mgr: ceph002(active, since 83s), standbys: ceph003, ceph001
#mgr 已经部署成功
osd: 3 osds: 3 up (since 8m), 3 in (since 8m)
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 3.0 GiB used, 147 GiB / 150 GiB avail
pgs:
[cephadmin@ceph001 cephcluster]$
nautilus 版本需要安装dashboard
sudo yum -y install --downloadonly --downloaddir=/home/cephadmin/software/cephmgrdashboard/ ceph-mgr-dashboard
sudo yum -y install ceph-mgr-dashboard
[cephadmin@ceph002 ~]$ ceph mgr module enable dashboard
[cephadmin@ceph002 ~]$ ceph dashboard create-self-signed-cert
Self-signed certificate created
[cephadmin@ceph002 ~]$ ceph dashboard set-login-credentials admin admin
******************************************************************
*** WARNING: this command is deprecated. ***
*** Please use the ac-user-* related commands to manage users. ***
******************************************************************
Username and password updated
[cephadmin@ceph002 ~]$ ceph mgr services
{
"dashboard": "https://ceph002:8443/"
}
[cephadmin@ceph002 ~]$
linux (centos7) 使用ceph-deploy 安装ceph
原文:https://www.cnblogs.com/weiwei2021/p/14060186.html