1. Nova 配置
    disk_cachemodes = "network=writeback” (启用)
    改为 disk_cachemodes = “network=none"（关闭）

  2. Ceph 配置

    1. 开启 ceph rbd cache
      client]
      rbd_cache = true
      rbd_cache_writethrough_until_flush = true
      admin_socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok
      log_file = /var/log/qemu/qemu-guest-$pid.log
      rbd_concurrent_management_ops = 20

    2. 关闭ceph rbd cache
      [client]
      #rbd_cache = true
      #rbd_cache_writethrough_until_flush = true
      admin_socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok
      #log_file = /var/log/qemu/qemu-guest-$pid.log
      #rbd_concurrent_management_ops = 20

  3. 验证cache是否生效

    1. 验证 生效命令
      [root@l2-4-30 guests]# ceph --admin-daemo ceph-client.cinder.27346.94699498717184.asok perf dump
      {
      "finisher-librbd::TaskFinisher::m_finisher": {
      "queue_len": 0,
      "complete_latency": {
      "avgcount": 0,
      "sum": 0.000000000
      }
      },
      "librbd-1373b25a3be0cd-glance_images-86e30f80-5a93-47ee-88c9-de5c1db6df10": {
      "rd": 32853,
      "rd_bytes": 2060131840,
      "rd_latency": {
      "avgcount": 32853,
      "sum": 42.348302430
      },
      "wr": 0,
      "wr_bytes": 0,
      "wr_latency": {
      "avgcount": 0,
      "sum": 0.000000000
      },
      "discard": 0,
      "discard_bytes": 0,
      "discard_latency": {
      "avgcount": 0,
      "sum": 0.000000000
      },
      "flush": 0,
      "aio_flush": 0,
      "aio_flush_latency": {
      "avgcount": 0,
      "sum": 0.000000000
      },
      "snap_create": 0,
      "snap_remove": 0,
      "snap_rollback": 0,
      "snap_rename": 0,
      "notify": 0,
      "resize": 0,
      "readahead": 103,
      "readahead_bytes": 27451904,
      "invalidate_cache": 0
      },
      "librbd-bf1708496d088e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk": {
      "rd": 40116,
      "rd_bytes": 650777600,
      "rd_latency": {
      "avgcount": 40116,
      "sum": 38.547805260
      },
      "wr": 7348,
      "wr_bytes": 253340672,
      "wr_latency": {
      "avgcount": 7348,
      "sum": 55.895274365
      },
      "discard": 0,
      "discard_bytes": 0,
      "discard_latency": {
      "avgcount": 0,
      "sum": 0.000000000
      },
      "flush": 0,
      "aio_flush": 434,
      "aio_flush_latency": {
      "avgcount": 434,
      "sum": 0.540130077
      },
      "snap_create": 0,
      "snap_remove": 0,
      "snap_rollback": 0,
      "snap_rename": 0,
      "notify": 0,
      "resize": 0,
      "readahead": 130,
      "readahead_bytes": 37353472,
      "invalidate_cache": 0
      },
      "objectcacher-librbd-1373b25a3be0cd-glance_images-86e30f80-5a93-47ee-88c9-de5c1db6df10": {
      "cache_ops_hit": 2761,
      "cache_ops_miss": 30195,
      "cache_bytes_hit": 1500713472,
      "cache_bytes_miss": 2063611392,
      "data_read": 2087583744,
      "data_written": 0,
      "data_flushed": 0,
      "data_overwritten_while_flushing": 0,
      "write_ops_blocked": 0,
      "write_bytes_blocked": 0,
      "write_time_blocked": 0.000000000
      },
      "objectcacher-librbd-bf1708496d088e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk": {
      "cache_ops_hit": 829,
      "cache_ops_miss": 7979,
      "cache_bytes_hit": 191516672,
      "cache_bytes_miss": 153311232,
      "data_read": 168047616,
      "data_written": 253340672,
      "data_flushed": 253340672,
      "data_overwritten_while_flushing": 0,
      "write_ops_blocked": 0,
      "write_bytes_blocked": 0,
      "write_time_blocked": 0.000000000
      },
      "objecter": {
      "op_active": 0,
      "op_laggy": 0,
      "op_send": 45229,
      "op_send_bytes": 0,
      "op_resend": 0,
      "op_ack": 45105,
      "op_commit": 7893,
      "op": 45229,
      "op_r": 37337,
      "op_w": 7892,
      "op_rmw": 0,
      "op_pg": 0,
      "osdop_stat": 7045,
      "osdop_create": 0,
      "osdop_read": 37319,
      "osdop_write": 7519,
      "osdop_writefull": 0,
      "osdop_append": 0,
      "osdop_zero": 0,
      "osdop_truncate": 0,
      "osdop_delete": 0,
      "osdop_mapext": 0,
      "osdop_sparse_read": 0,
      "osdop_clonerange": 0,
      "osdop_getxattr": 0,
      "osdop_setxattr": 0,
      "osdop_cmpxattr": 0,
      "osdop_rmxattr": 0,
      "osdop_resetxattrs": 0,
      "osdop_tmap_up": 0,
      "osdop_tmap_put": 0,
      "osdop_tmap_get": 0,
      "osdop_call": 1117,
      "osdop_watch": 122,
      "osdop_notify": 1,
      "osdop_src_cmpxattr": 0,
      "osdop_pgls": 0,
      "osdop_pgls_filter": 0,
      "osdop_other": 1,
      "linger_active": 1,
      "linger_send": 2,
      "linger_resend": 0,
      "linger_ping": 121,
      "poolop_active": 0,
      "poolop_send": 0,
      "poolop_resend": 0,
      "poolstat_active": 0,
      "poolstat_send": 0,
      "poolstat_resend": 0,
      "statfs_active": 0,
      "statfs_send": 0,
      "statfs_resend": 0,
      "command_active": 0,
      "command_send": 0,
      "command_resend": 0,
      "map_epoch": 0,
      "map_full": 0,
      "map_inc": 0,
      "osd_sessions": 27966,
      "osd_session_open": 236,
      "osd_session_close": 0,
      "osd_laggy": 1,
      "omap_wr": 0,
      "omap_rd": 0,
      "omap_del": 0
      },
      "throttle-msgr_dispatch_throttler-radosclient": {
      "val": 0,
      "max": 104857600,
      "get": 52511,
      "get_sum": 2152111179,
      "get_or_fail_fail": 0,
      "get_or_fail_success": 0,
      "take": 0,
      "take_sum": 0,
      "put": 52511,
      "put_sum": 2152111179,
      "wait": {
      "avgcount": 0,
      "sum": 0.000000000
      }
      },
      "throttle-objecter_bytes": {
      "val": 0,
      "max": 104857600,
      "get": 45108,
      "get_sum": 2419831296,
      "get_or_fail_fail": 0,
      "get_or_fail_success": 45108,
      "take": 0,
      "take_sum": 0,
      "put": 44725,
      "put_sum": 2419831296,
      "wait": {
      "avgcount": 0,
      "sum": 0.000000000
      }
      },
      "throttle-objecter_ops": {
      "val": 0,
      "max": 1024,
      "get": 45108,
      "get_sum": 45108,
      "get_or_fail_fail": 0,
      "get_or_fail_success": 45108,
      "take": 0,
      "take_sum": 0,
      "put": 45108,
      "put_sum": 45108,
      "wait": {
      "avgcount": 0,
      "sum": 0.000000000
      }
      }
      } 完整配置

    过滤 cache 字符串

        [root@l2-4-30 guests]# ceph --admin-daemo ceph-client.cinder.27346.94699498717184.asok   perf  dump |grep cache -A 5 -B 5
            "snap_rename": 0,
            "notify": 0,
            "resize": 0,
            "readahead": 103,
            "readahead_bytes": 27451904,
            "invalidate_cache": 0
    },
    "librbd-bf1708496d088e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk": {
            "rd": 57149,
            "rd_bytes": 1148467200,
            "rd_latency": {

    --
    "snap_rename": 0,
    "notify": 0,
    "resize": 0,
    "readahead": 130,
    "readahead_bytes": 37353472,
    "invalidate_cache": 0
    },
    "objectcacher-librbd-1373b25a3be0cd-glance_images-86e30f80-5a93-47ee-88c9-de5c1db6df10": {
    "cache_ops_hit": 2791,
    "cache_ops_miss": 45597,
    "cache_bytes_hit": 1526398976,
    "cache_bytes_miss": 3029461504,
    "data_read": 3057748992,
    "data_written": 0,
    "data_flushed": 0,
    "data_overwritten_while_flushing": 0,
    "write_ops_blocked": 0,
    "write_bytes_blocked": 0,
    "write_time_blocked": 0.000000000
    },
    "objectcacher-librbd-bf1708496d088e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk": {
    "cache_ops_hit": 886,
    "cache_ops_miss": 9957,
    "cache_bytes_hit": 248139264,
    "cache_bytes_miss": 225620480,
    "data_read": 240729088,
    "data_written": 462767104,
    "data_flushed": 462767104,
    "data_overwritten_while_flushing": 0,
    "write_ops_blocked": 0,

    1. 验证 无cache命令
      [root@l23-4-40 guests]# ceph --admin-daemo ceph-client.cinder.138869.94547912278016.asok perf dump |grep cache -A 5 -B 5
      "snap_rename": 0,
      "notify": 0,
      "resize": 0,
      "readahead": 0,
      "readahead_bytes": 0,
      "invalidate_cache": 0
      },
      "librbd-bee65365887c0b-nova_instances-5cf85c18-6725-4674-88e4-60408c97f487_disk": {
      "rd": 62937,
      "rd_bytes": 1261381632,
      "rd_latency": {

                  "snap_rename": 0,
            "notify": 0,
            "resize": 0,
            "readahead": 0,
            "readahead_bytes": 0,
            "invalidate_cache": 0
    },
    "objecter": {
            "op_active": 0,
            "op_laggy": 0,
            "op_send": 95327
    }