• bclinux aarch64 openeuler 20.03 LTS SP1 部署 fastCFS

    基于已配置好的4个节点部署ceph-0 ceph-1 ceph-2 ceph-3（早期ceph测试环境，名称就不修改了）

    获取fcfs.sh

    1. mkdir /etc/fcfs
    2. cd /etc/fcfs
    3. wget http://fastcfs.net/fastcfs/ops/fcfs.sh

    配置/etc/fcfs/fcfs.settings

    1. # 要安装的集群版本号（例如：5.0.0）
    2. fastcfs_version=4.3.0
    3.  
    4. # 要安装 fuseclient 客户端的IP列表，多个ip以英文逗号分隔
    5. fuseclient_ips=172.17.163.105,172.17.112.206,172.17.227.100,172.17.67.157

    conf目录

    [root@ceph-0 fcfs]# cd /etc/fcfs/
    [root@ceph-0 fcfs]# wget http://fastcfs.net/fastcfs/ops/fcfs-config-sample.tar.gz
    --2023-11-21 10:37:55--  http://fastcfs.net/fastcfs/ops/fcfs-config-sample.tar.gz
    Resolving fastcfs.net (fastcfs.net)... 39.106.8.170
    Connecting to fastcfs.net (fastcfs.net)|39.106.8.170|:80... connected.
    HTTP request sent, awaiting response... 200 OK
    Length: 4569 (4.5K) [application/octet-stream]
    Saving to: ‘fcfs-config-sample.tar.gz’

    fcfs-config-sample.tar.gz                 100%[==================================================================================>]   4.46K  --.-KB/s    in 0s      

    2023-11-21 10:37:55 (79.9 MB/s) - ‘fcfs-config-sample.tar.gz’ saved [4569/4569]

    [root@ceph-0 fcfs]# tar -xvf fcfs-config-sample.tar.gz 
    conf/
    conf/fcfs/
    conf/fcfs/fuse.conf
    conf/auth/
    conf/auth/server.conf
    conf/auth/client.conf
    conf/auth/session.conf
    conf/auth/cluster.conf
    conf/auth/keys/
    conf/auth/keys/session_validate.key
    conf/auth/auth.conf
    conf/vote/
    conf/vote/server.conf
    conf/vote/client.conf
    conf/vote/cluster.conf
    conf/fdir/
    conf/fdir/storage.conf
    conf/fdir/server.conf
    conf/fdir/client.conf
    conf/fdir/cluster.conf
    conf/fstore/
    conf/fstore/storage.conf
    conf/fstore/server.conf
    conf/fstore/client.conf
    conf/fstore/cluster.conf

    cluster.conf

    [root@ceph-0 fcfs]# vim conf/auth/cluster.conf 
    [root@ceph-0 fcfs]# vim conf/fdir/cluster.conf 
    [root@ceph-0 fcfs]# vim conf/fstore/cluster.conf 
    [root@ceph-0 fcfs]# vim conf/vote/cluster.conf 

    修改server信息

    1. [server-1]
    2. host = 172.17.163.105
    3. [server-2]
    4. host = 172.17.112.206
    5. [server-3]
    6. host = 172.17.227.100
    7. [server-4]
    8. host = 172.17.67.157

    存储参数 store-path

    [root@ceph-0 fcfs]# vim conf/fdir/storage.conf

    1. [store-path-1]
    2. # the path to store the file
    3. # default value is the data path of storage engine
    4. path = /opt/fastcfs/fdir/db

    [root@ceph-0 fcfs]# vim conf/fstore/storage.conf

    1. [store-path-1]
    2. # the path to store the file
    3. path = /opt/faststore/data

    初始化lvm（每个节点均执行）

    pv

    [root@ceph-0 fcfs]# pvcreate /dev/vdb 
      Physical volume "/dev/vdb" successfully created.

    vg

    [root@ceph-0 fcfs]# vgcreate vg_fastcfs /dev/vdb
      Volume group "vg_fastcfs" successfully created

    pv

    [root@ceph-0 fcfs]# lvcreate -y -L 50G -n lv_fastcfs_fdir1 vg_fastcfs
      Wiping ceph_bluestore signature on /dev/vg_fastcfs/lv_fastcfs_fdir1.
      Logical volume "lv_fastcfs_fdir1" created.

    [root@ceph-0 fcfs]# lvcreate -y -l 100%FREE -n lv_fastcfs_fstore1 vg_fastcfs
      Logical volume "lv_fastcfs_fstore1" created.

    lvm最终状态

    挂载存储目录（每个节点均执行）

    创建数据目录

    [root@ceph-0 fcfs]# mkdir -p /opt/fastcfs/fdir/db
    [root@ceph-0 fcfs]# mkdir -p /opt/faststore/data

    配置/etc/fstab

    添加两行

    1. /dev/mapper/vg_fastcfs-lv_fastcfs_fdir1 /opt/fastcfs/fdir/db xfs defaults 0 0
    2. /dev/mapper/vg_fastcfs-lv_fastcfs_fstore1 /opt/faststore/data xfs defaults 0 0

    xfs格式化

    [root@ceph-0 fcfs]# mkfs.xfs /dev/mapper/vg_fastcfs-lv_fastcfs_fdir1
    meta-data=/dev/mapper/vg_fastcfs-lv_fastcfs_fdir1 isize=512    agcount=4, agsize=3276800 blks
             =                       sectsz=512   attr=2, projid32bit=1
             =                       crc=1        finobt=1, sparse=1, rmapbt=0
             =                       reflink=1
    data     =                       bsize=4096   blocks=13107200, imaxpct=25
             =                       sunit=0      swidth=0 blks
    naming   =version 2              bsize=4096   ascii-ci=0, ftype=1
    log      =internal log           bsize=4096   blocks=6400, version=2
             =                       sectsz=512   sunit=0 blks, lazy-count=1
    realtime =none                   extsz=4096   blocks=0, rtextents=0
    Discarding blocks...Done.
    [root@ceph-0 fcfs]# mkfs.xfs /dev/mapper/vg_fastcfs-lv_fastcfs_fstore1
    meta-data=/dev/mapper/vg_fastcfs-lv_fastcfs_fstore1 isize=512    agcount=4, agsize=3276544 blks
             =                       sectsz=512   attr=2, projid32bit=1
             =                       crc=1        finobt=1, sparse=1, rmapbt=0
             =                       reflink=1
    data     =                       bsize=4096   blocks=13106176, imaxpct=25
             =                       sunit=0      swidth=0 blks
    naming   =version 2              bsize=4096   ascii-ci=0, ftype=1
    log      =internal log           bsize=4096   blocks=6399, version=2
             =                       sectsz=512   sunit=0 blks, lazy-count=1
    realtime =none                   extsz=4096   blocks=0, rtextents=0
    Discarding blocks...Done.

    挂载

    mount -a

    其他三台初始化

    1. pvcreate /dev/vdb 
    2. vgcreate vg_fastcfs /dev/vdb
    3. lvcreate -y -L 50G -n lv_fastcfs_fdir1 vg_fastcfs
    4. lvcreate -y -l 100%FREE -n lv_fastcfs_fstore1 vg_fastcfs
    5. mkdir -p /opt/fastcfs/fdir/db
    6. mkdir -p /opt/faststore/data
    7.  
    8. echo "/dev/mapper/vg_fastcfs-lv_fastcfs_fdir1 /opt/fastcfs/fdir/db xfs defaults 0 0" >> /etc/fstab
    9. echo "/dev/mapper/vg_fastcfs-lv_fastcfs_fstore1 /opt/faststore/data xfs defaults 0 0" >> /etc/fstab
    10. mkfs.xfs /dev/mapper/vg_fastcfs-lv_fastcfs_fdir1
    11. mkfs.xfs /dev/mapper/vg_fastcfs-lv_fastcfs_fstore1
    12. mount -a

    适配BigCloud(即:bclinux)

    修改脚本，差异如下[root@ceph-0 fcfs]# diff -Npr fcfs.sh fcfs.sh.bak
     

    1. *** fcfs.sh    2023-11-21 13:45:20.742219402 +0800
    2. --- fcfs.sh.bak    2023-11-21 13:41:49.109731058 +0800
    3. *************** declare -ir MIN_VERSION_OF_Anolis=7
    4. *** 22,28 ****
    5.   declare -ir MIN_VERSION_OF_Amazon=2
    6.   declare -ir MIN_VERSION_OF_openEuler=20
    7.   declare -ir MIN_VERSION_OF_UOS=20
    8. ! YUM_OS_ARRAY=(Red Rocky Oracle Fedora CentOS AlmaLinux Alibaba Anolis Amazon BigCloud openEuler Kylin UOS)
    9.   APT_OS_ARRAY=(Ubuntu Debian Deepin)
    10.   
    11.   fcfs_settings_file="fcfs.settings"
    12. --- 22,28 ----
    13.   declare -ir MIN_VERSION_OF_Amazon=2
    14.   declare -ir MIN_VERSION_OF_openEuler=20
    15.   declare -ir MIN_VERSION_OF_UOS=20
    16. ! YUM_OS_ARRAY=(Red Rocky Oracle Fedora CentOS AlmaLinux Alibaba Anolis Amazon openEuler Kylin UOS)
    17.   APT_OS_ARRAY=(Ubuntu Debian Deepin)
    18.   
    19.   fcfs_settings_file="fcfs.settings"
    20. *************** check_remote_osname() {
    21. *** 850,856 ****
    22.         else
    23.           os_major_version=8
    24.         fi
    25. !     elif [ $osname = 'BigCloud' ] || [ $osname = 'openEuler' ] || [ $osname = 'Kylin' ] || [ $osname = 'UOS' ]; then
    26.         os_major_version=8
    27.       fi
    28.     else
    29. --- 850,856 ----
    30.         else
    31.           os_major_version=8
    32.         fi
    33. !     elif [ $osname = 'openEuler' ] || [ $osname = 'Kylin' ] || [ $osname = 'UOS' ]; then
    34.         os_major_version=8
    35.       fi
    36.     else
    37.  
    38.

    执行安装

    bash fcfs.sh install

    配置文件分发

    bash fcfs.sh config

    分发的配置文件清单

    1. [root@ceph-1 ~]# find /etc/fastcfs/
    2. /etc/fastcfs/
    3. /etc/fastcfs/fdir
    4. /etc/fastcfs/fdir/cluster.conf
    5. /etc/fastcfs/fdir/client.conf
    6. /etc/fastcfs/fdir/server.conf
    7. /etc/fastcfs/fdir/storage.conf
    8. /etc/fastcfs/auth
    9. /etc/fastcfs/auth/cluster.conf
    10. /etc/fastcfs/auth/client.conf
    11. /etc/fastcfs/auth/server.conf
    12. /etc/fastcfs/auth/auth.conf
    13. /etc/fastcfs/auth/session.conf
    14. /etc/fastcfs/auth/keys
    15. /etc/fastcfs/auth/keys/session_validate.key
    16. /etc/fastcfs/vote
    17. /etc/fastcfs/vote/cluster.conf
    18. /etc/fastcfs/vote/client.conf
    19. /etc/fastcfs/vote/server.conf
    20. /etc/fastcfs/fcfs
    21. /etc/fastcfs/fcfs/papi.conf
    22. /etc/fastcfs/fcfs/fuse.conf
    23. /etc/fastcfs/fstore
    24. /etc/fastcfs/fstore/cluster.conf
    25. /etc/fastcfs/fstore/client.conf
    26. /etc/fastcfs/fstore/server.conf
    27. /etc/fastcfs/fstore/storage.conf
    28. /etc/fastcfs/fstore/dbstore.conf

    启动集群

    bash fcfs.sh start

    查看集群日志

    bash fcfs.sh tail fdir

    故障：ERROR  connect to fdir server Connection refused

    [2023-11-21 14:21:26] ERROR - file: connection_pool.c, line: 140, connect to fdir server 172.17.112.206:11011 fail, errno: 111, error info: Connection refused
    [2023-11-21 14:21:26] ERROR - file: connection_pool.c, line: 140, connect to fdir server 172.17.227.100:11011 fail, errno: 111, error info: Connection refused
    [2023-11-21 14:21:26] ERROR - file: connection_pool.c, line: 140, connect to fdir server 172.17.67.157:11011 fail, errno: 111, error info: Connection refused
    [2023-11-21 14:21:27] ERROR - file: connection_pool.c, line: 140, connect to fdir server 172.17.67.157:11011 fail, errno: 111, error info: Connection refused
     

    过一会儿就好了

    查看状态

    查看fastDIR状态

    fdir_cluster_stat

    查看挂载目录空间情况

    df -h /opt/fastcfs/fuse/

    ====

    其他节点未同步，直接升级到5.0.0

    故障：fstore cluster ip配置未同步

    fcfs.sh config 执行后，

    除了 ceph-0节点，其他节点fstore/cluster.conf host地址均不对！

    正确的地址

    手动修改 ceph-1 2 3节点 /etc/fastcfs/fstore/cluster.conf

    重启

    cd /etc/fcfs; ./fcfs.sh restart

    基本上OK了（还不会用）

    简单dd进去一个1G文件，所有节点fuse目录已经同步了

    1. [root@ceph-0 fcfs]# ssh ceph-0 df -h /opt/fastcfs/fuse/
    2. Filesystem      Size  Used Avail Use% Mounted on
    3. /dev/fuse        45G  1.0G   44G   3% /opt/fastcfs/fuse
    4. [root@ceph-0 fcfs]# ssh ceph-1 df -h /opt/fastcfs/fuse/
    5. Filesystem      Size  Used Avail Use% Mounted on
    6. /dev/fuse        45G  1.0G   44G   3% /opt/fastcfs/fuse
    7. [root@ceph-0 fcfs]# ssh ceph-2 df -h /opt/fastcfs/fuse/
    8. Filesystem      Size  Used Avail Use% Mounted on
    9. /dev/fuse        45G  1.0G   44G   3% /opt/fastcfs/fuse
    10. [root@ceph-0 fcfs]# ssh ceph-3 df -h /opt/fastcfs/fuse/
    11. Filesystem      Size  Used Avail Use% Mounted on
    12. /dev/fuse        45G  1.0G   44G   3% /opt/fastcfs/fuse

    客户端

    1. rpm -ivh http://www.fastken.com/yumrepo/el8/noarch/FastOSrepo-1.0.1-1.el8.noarch.rpm
    2. yum install -y FastCFS-fused-5.0.0

    配置cluster.conf

    [root@ceph-client fastcfs]# vim /etc/fastcfs/fstore/cluster.conf 

    vim /etc/fastcfs/auth/cluster.conf 

    vim /etc/fastcfs/fdir/cluster.conf 

    1. [server-1]
    2. host = 172.17.163.105
    3. [server-2]
    4. host = 172.17.112.206
    5. [server-3]
    6. host = 172.17.227.100
    7. [server-4]
    8. host = 172.17.67.157

    systemctl daemon-reload

    systemctl start fastcfs

    vdbench压测

    fastcfs.conf

    1. hd=default,vdbench=/root/vdbench,user=root,shell=ssh
    2. hd=hd1,system=ceph-0
    3. fsd=fsd1,anchor=/opt/fastcfs/fuse,depth=2,width=10,files=40,size=4M,shared=yes
    4. fwd=format,threads=24,xfersize=1m
    5. fwd=default,xfersize=1m,fileio=sequential,fileselect=sequential,operation=write,threads=24
    6. fwd=fwd1,fsd=fsd1,host=hd1
    7. rd=rd1,fwd=fwd*,fwdrate=max,format=restart,elapsed=600,interval=1

    压测

    1. cd /root/vdbench
    2. ./vdbench -f fastcfs.conf

    中途卡了一会儿？

    对比之前做的ceph集群，峰值更大，但是有停顿！

    客户端消耗了大部分CPU资源（未在ceph-client上执行- -#，测试方案不严谨）

    告一段落。

    独立一个客户端ceph-client云主机测试

    1884/8=235.5 虚拟网卡满了？比本地测试峰值要低

    附：离线安装参考

    参考

    docs/fcfs-ops-tool-zh_CN.md · happyfish100/FastCFS - Gitee.com

  • 相关阅读:
    Puppeteer+RabbitMQ：Node.js 批量加工pdf服务架构设计与落地
    区块链技术中的共识机制算法：以工作量证明（PoW）为例
    每次重启完IDEA,application.properties文件里的中文变成?
    控制器连接Profinet转Modbus RTU网关与精密数显温控仪通讯
    AtCoder Beginner Contest 327 A-F
    MySQL高级篇知识点——主从复制
    javacofig几个常用注解
    22【解释器设计模式】
    Python自学教程5-字符串有哪些常用操作
    DASCTF2022.07赋能赛 web 复现
  • 原文地址：https://blog.csdn.net/hknaruto/article/details/134525961