IO被谁吃了？-ioqa
phyger
|
|
9
|
未分类

905 字
|
29 分钟

背景

近期，兄弟部门在部署某环境的时候发现自己的业务容器莫名的 unavailable，这是怎么回事呢？今天我们一起来分析分析。

谁吃了 IO？

问题分析

运维同事登录到虚拟机看到虚拟机的负载很高，这是怎么回事呢？运维同事发了 Top 的截图给我，分析后发现这台虚拟机的部分进程发生了 IO 等待，所以我们怀疑是 IO 性能受限。

安装工具

目前虚机无法访问外网，我们为了快速定位，所以建议直接使用 iopp。

代码：

#include <stdio.h>
#include <sys/types.h>
#include <dirent.h>
#include <ctype.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <getopt.h>

#define PROC "/proc"

#define GET_VALUE(v) 
        p = strchr(p, ':'); 
        ++p; 
        ++p; 
        q = strchr(p, 'n'); 
        length = q - p; 
        if (length >= BUFFERLEN) 
        { 
            printf("ERROR - value is larger than the buffer: %dn", __LINE__); 
            exit(1); 
        } 
        strncpy(value, p, length); 
        value[length] = ' '; 
        v = atoll(value);

#define BTOKB(b) b >> 10
#define BTOMB(b) b >> 20

#define BUFFERLEN 255
#define COMMANDLEN 1024
#define VALUELEN 63

#define NUM_STRINGS 8

struct io_node
{
    int pid;
    long long rchar;
    long long wchar;
    long long syscr;
    long long syscw;
    long long read_bytes;
    long long write_bytes;
    long long cancelled_write_bytes;
    char command[COMMANDLEN + 1];
    struct io_node *next;
};

struct io_node *head = NULL;
int command_flag = 0;
int idle_flag = 0;
int mb_flag = 0;
int kb_flag = 0;
int hr_flag = 0;

/* Prototypes */
char *format_b(long long);
struct io_node *get_ion(int);
struct io_node *new_ion(char *);
void upsert_data(struct io_node *);

char *
format_b(long long amt)
{
    static char retarray[NUM_STRINGS][16];
    static int  index = 0;
    register char *ret;
    register char tag = 'B';

    ret = retarray[index];
    index = (index + 1) % NUM_STRINGS;

    if (amt >= 10000) {
        amt = (amt + 512) / 1024;
        tag = 'K';
        if (amt >= 10000) {
            amt = (amt + 512) / 1024;
            tag = 'B';
            if (amt >= 10000) {
                amt = (amt + 512) / 1024;
                tag = 'G';
            }
        }
    }

    snprintf(ret, sizeof(retarray[index]) - 1, "%lld%c", amt, tag);

    return (ret);
}

int
get_cmdline(struct io_node *ion)
{
    int fd;
    int length;
    char filename[BUFFERLEN + 1];
    char buffer[COMMANDLEN + 1];
    char *p;
    char *q;

    length = snprintf(filename, BUFFERLEN, "%s/%d/cmdline", PROC, ion->pid);
    if (length == BUFFERLEN)
        printf("WARNING - filename length may be too big for buffer: %dn",
                __LINE__);
    fd = open(filename, O_RDONLY);
    if (fd == -1)
        return 1;
    length = read(fd, buffer, sizeof(buffer) - 1);
    close(fd);
    buffer[length] = ' ';
    if (length == 0)
        return 2;
    if (command_flag == 0)
    {
        /*
         * The command is near the beginning; we don't need to be able to
         * the entire stat file.
         */
        p = strchr(buffer, '(');
        ++p;
        q = strchr(p, ')');
        length = q - p;
    }
    else
        p = buffer;
    length = length < COMMANDLEN ? length : COMMANDLEN;
    strncpy(ion->command, p, length);
    ion->command[length] = ' ';
    return 0;
}

struct io_node *
get_ion(int pid)
{
    struct io_node *c = head;

    while (c != NULL)
    {
        if (c->pid == pid)
            break;
        c = c->next;
    }
    return c;
}

int
get_tcomm(struct io_node *ion)
{
    int fd;
    int length;
    char filename[BUFFERLEN + 1];
    char buffer[BUFFERLEN + 1];
    char *p;
    char *q;

    length = snprintf(filename, BUFFERLEN, "%s/%d/stat", PROC, ion->pid);
    if (length == BUFFERLEN)
        printf("WARNING - filename length may be too big for buffer: %dn",
                __LINE__);
    fd = open(filename, O_RDONLY);
    if (fd == -1)
        return 1;
    length = read(fd, buffer, sizeof(buffer) - 1);
    close(fd);
    /*
     * The command is near the beginning; we don't need to be able to
     * the entire stat file.
     */
    p = strchr(buffer, '(');
    ++p;
    q = strchr(p, ')');
    length = q - p;
    length = length < BUFFERLEN ? length : BUFFERLEN;

    strncpy(ion->command, p, length);
    ion->command[length] = ' ';
    return 0;
}

struct io_node *
insert_ion(struct io_node *ion)
{
    struct io_node *c;
    struct io_node *p;

    /* Check the head of the list as a special case. */
    if (ion->pid < head->pid)
    {
        ion->next = head;
        head = ion;
        return head;
    }

    c = head->next;
    p = head;
    while (c != NULL)
    {
        if (ion->pid < c->pid)
        {
            ion->next = c;
            p->next = ion;
            return head;
        }
        p = c;
        c = c->next;
    }

    /* Append to the end of the list. */
    if (c == NULL)
        p->next = ion;

    return head;
}

void
get_stats()
{
    DIR *dir = opendir(PROC);
    struct dirent *ent;
    char filename[BUFFERLEN + 1];
    char buffer[BUFFERLEN + 1];

    char value[BUFFERLEN + 1];

    /* Display column headers. */
    if (hr_flag == 1)
        printf("%5s %5s %5s %8s %8s %5s %6s %7s %sn", "pid", "rchar", "wchar",
                "syscr", "syscw", "reads", "writes", "cwrites", "command");
    else if (kb_flag == 1)
        printf("%5s %8s %8s %8s %8s %8s %8s %8s %sn", "pid", "rchar", "wchar",
                "syscr", "syscw", "rkb", "wkb", "cwkb", "command");
    else if (mb_flag == 1)
        printf("%5s %8s %8s %8s %8s %8s %8s %8s %sn", "pid", "rchar", "wchar",
                "syscr", "syscw", "rmb", "wmb", "cwmb", "command");
    else
        printf("%5s %8s %8s %8s %8s %8s %8s %8s %sn", "pid", "rchar", "wchar",
                "syscr", "syscw", "rbytes", "wbytes", "cwbytes", "command");

    /* Loop through the process table and display a line per pid. */
    while ((ent = readdir(dir)) != NULL)
    {
        int rc;
        int fd;
        int length;

        char *p;
        char *q;

        struct io_node *ion;
        struct io_node *old_ion;

        long long rchar;
        long long wchar;
        long long syscr;
        long long syscw;
        long long read_bytes;
        long long write_bytes;
        long long cancelled_write_bytes;

        if (!isdigit(ent->d_name[0]))
            continue;

        ion = new_ion(ent->d_name);

        if (command_flag == 1)
            rc = get_cmdline(ion);
        if (command_flag == 0 || rc != 0)
            /* If the full command line is not asked for or is empty... */
            rc = get_tcomm(ion);

        if (rc != 0)
        {
            free(ion);
            continue;
        }

        /* Read 'io' file. */
        length = snprintf(filename, BUFFERLEN, "%s/%s/io", PROC, ent->d_name);
        if (length == BUFFERLEN)
            printf("WARNING - filename length may be too big for buffer: %dn",
                    __LINE__);
        fd = open(filename, O_RDONLY);
        if (fd == -1)
        {
            free(ion);
            continue;
        }
        length = read(fd, buffer, sizeof(buffer) - 1);
        close(fd);
        buffer[length] = ' ';

        /* Parsing the io file data. */
        p = buffer;
        GET_VALUE(ion->rchar);
        GET_VALUE(ion->wchar);
        GET_VALUE(ion->syscr);
        GET_VALUE(ion->syscw);
        GET_VALUE(ion->read_bytes);
        GET_VALUE(ion->write_bytes);
        GET_VALUE(ion->cancelled_write_bytes);

        old_ion = get_ion(ion->pid);

        /* Display the pid's io data. */
        if (old_ion != NULL)
        {
            rchar = ion->rchar - old_ion->rchar;
            wchar = ion->wchar - old_ion->wchar;
            syscr = ion->syscr - old_ion->syscr;
            syscw = ion->syscw - old_ion->syscw;
            read_bytes = ion->read_bytes - old_ion->read_bytes;
            write_bytes = ion->write_bytes - old_ion->write_bytes;
            cancelled_write_bytes = ion->cancelled_write_bytes -
                    old_ion->cancelled_write_bytes;

            if (kb_flag == 1 && hr_flag == 0)
            {
                rchar = BTOKB(rchar);
                wchar = BTOKB(wchar);
                syscr = BTOKB(syscr);
                syscw = BTOKB(syscw);
                read_bytes = BTOKB(read_bytes);
                write_bytes = BTOKB(write_bytes);
                cancelled_write_bytes = BTOKB(cancelled_write_bytes);
            }
            else if (mb_flag == 1 && hr_flag == 0)
            {
                rchar = BTOMB(rchar);
                wchar = BTOMB(wchar);
                syscr = BTOMB(syscr);
                syscw = BTOMB(syscw);
                read_bytes = BTOMB(read_bytes);
                write_bytes = BTOMB(write_bytes);
                cancelled_write_bytes = BTOMB(cancelled_write_bytes);
            }

            if (!(idle_flag == 1 && rchar == 0 && wchar == 0 && syscr == 0 &&
                    syscw == 0 && read_bytes == 0 && write_bytes == 0 &&
                    cancelled_write_bytes == 0)) {
                if (hr_flag == 0)
                    printf("%5d %8lld %8lld %8lld %8lld %8lld %8lld %8lld %sn",
                            ion->pid,
                            rchar,
                            wchar,
                            syscr,
                            syscw,
                            read_bytes,
                            write_bytes,
                            cancelled_write_bytes,
                            ion->command);
                else
                    printf("%5d %5s %5s %8lld %8lld %5s %6s %7s %sn",
                            ion->pid,
                            format_b(rchar),
                            format_b(wchar),
                            syscr,
                            syscw,
                            format_b(read_bytes),
                            format_b(write_bytes),
                            format_b(cancelled_write_bytes),
                            ion->command);
            }
        }
        else if (idle_flag != 1)
            /*
             * No previous data, show 0's instead of calculating negatives
             * only if we are shoring idle processes.
             */
            printf("%5d %8d %8d %8d %8d %8d %8d %8d %sn",
                    ion->pid, 0, 0, 0, 0, 0, 0, 0, ion->command);

        upsert_data(ion);
    }
    closedir(dir);
    return;
}

struct io_node *
new_ion(char *pid)
{
    struct io_node *ion;

    ion = (struct io_node *) malloc(sizeof(struct io_node));
    bzero(ion, sizeof(struct io_node));
    ion->pid = atoi(pid);

    return ion;
}

void
upsert_data(struct io_node *ion)
{
    struct io_node *n;

    /* List is empty. */
    if (head == NULL)
    {
        head = ion;
        return;
    }

    /* Check if we have seen this pid before. */
    n = head;
    while (n != NULL)
    {
        if (n->pid == ion->pid)
        {
            n->rchar = ion->rchar;
            n->wchar = ion->wchar;
            n->syscr = ion->syscr;
            n->syscw = ion->syscw;
            n->read_bytes = ion->read_bytes;
            n->write_bytes = ion->write_bytes;
            n->cancelled_write_bytes = ion->cancelled_write_bytes;
            /*
             * If the pids wrap, then the command may be different then before.
             */
            strcpy(n->command, ion->command);
            free(ion);
            return;
        }
        n = n->next;
    }

    /* Add this pid to the list. */
    head = insert_ion(ion);
    return;
}

void
usage()
{
    printf("usage: iopp -h|--helpn");
    printf("usage: iopp [-ci] [-k|-m] [delay [count]]n");
    printf("            -c, --command display full command linen");
    printf("            -h, --help display helpn");
    printf("            -i, --idle hides idle processesn");
    printf("            -k, --kilobytes display data in kilobytesn");
    printf("            -m, --megabytes display data in megabytesn");
    printf("            -u, --human-readable display data in kilo-, mega-, or giga-bytesn");
}

int
main(int argc, char *argv[])
{
    int c;

    int delay = 0;
    int count = 0;
    int max_count = 1;

    while (1)
    {
        int option_index = 0;
        static struct option long_options[] = {
                { "command", no_argument, 0, 'c' },
                { "help", no_argument, 0, 'h' },
                { "human-readable", no_argument, 0, 'u' },
                { "idle", no_argument, 0, 'i' },
                { "kilobytes", no_argument, 0, 'k' },
                { "megabytes", no_argument, 0, 'm' },
                { 0, 0, 0, 0 }
        };

        c = getopt_long(argc, argv, "chikmu", long_options, &option_index);
        if (c == -1)
        {
            /* Handle delay and count arguments. */

            if (argc == optind)
                break; /* No additional arguments. */
            else if ((argc - optind) == 1)
            {
                delay = atoi(argv[optind]);
                max_count = -1;
            }
            else if ((argc - optind) == 2)
            {
                delay = atoi(argv[optind]);
                max_count = atoi(argv[optind + 1]);
            }
            else
            {
                /* Too many additional arguments. */
                usage();
                return 3;
            }
            break;
        }

        switch (c)
        {
        case 'c':
            command_flag = 1;
            break;
        case 'h':
            usage();
            return 0;
        case 'i':
            idle_flag = 1;
            break;
        case 'k':
            kb_flag = 1;
            break;
        case 'm':
            mb_flag = 1;
            break;
        case 'u':
            hr_flag = 1;
            break;
        default:
            usage();
            return 2;
        }
    }

    while (max_count == -1 || count++ < max_count)
    {
        get_stats();
        if (count != max_count)
            sleep(delay);
    }
    return 0;
}

我们将以上代码保存为 iopp.c，然后gcc -o iopp iopp.c编译即可使用。

查找吃 IO 的进程

命令：./iopp -i -k -c 1

如上图，我们已经发现了 IO 的消耗大户。但是针对具体问题我们还得依据服务器的资源限制和进程的实际开销来综合判断。

参数解释

pid 进程ID
rchar 计划从磁盘读取的字节数
wchar 计划要写入磁盘的字节数
syscr 读I/O数
syscw 写I/O数
rkb 真正从磁盘读取的字节数
wkb 真正写入到磁盘的字节数
cwkb 因为清空页面缓存而导致没有发生操作的字节数
command 执行的命令

Tip

目前 iopp 的 C 版本结果不是很易于观测，你也可以选择 iopp 的 C++版本来查看 IO 的占用情况。

安装

git clone https://github.com/hackerforward/iopp
cd iopp
make
编译完成后，在当前目录生成一个iopp二进制文件
mv iopp ioppc++

使用

命令：ioppc++

如上，我们已经定位到了占用 IO 较大的两个进程。

以上就是今天的全部内容了，感谢您的阅读，我们下节再会。

版权声明：除特殊说明，博客文章均为phyger原创，依据CC BY-SA 4.0许可证进行授权，转载请附上出处链接及本声明。来自：https://u1s1.vip/archives/138
暂无评论

发送评论 编辑评论 


				

			

						

				

					

						

							

								
							

							
						

					

				

				

					

						

							

								
							

							
						

					

				

				

					

						

							

								
							

							
							
													

					

				

			

			
			

				

					
				

			

				

											

							
							
						

																					

							
							
						

										
					
											
						

	

		
|´・ω・)ノ
ヾ(≧∇≦*)ゝ
(☆ω☆)
（╯‵□′）╯︵┴─┴
￣﹃￣
(/ω＼)
∠( ᐛ 」∠)＿
(๑•̀ㅁ•́ฅ)
→_→
୧(๑•̀⌄•́๑)૭
٩(ˊᗜˋ*)و
(ノ°ο°)ノ
(´இ皿இ｀)
⌇●﹏●⌇
(ฅ´ω`ฅ)
(╯°A°)╯︵○○○
φ(￣∇￣o)
ヾ(´･ ･｀｡)ノ"
( ง ᵒ̌皿ᵒ̌)ง⁼³₌₃
(ó﹏ò｡)
Σ(っ °Д °;)っ
( ,,´･ω･)ﾉ"(´っω･｀｡)
╮(╯▽╰)╭
o(*////▽////*)q
＞﹏＜
( ๑´•ω•) "(ㆆᴗㆆ)
😂
😀
😅
😊
🙂
🙃
😌
😍
😘
😜
😝
😏
😒
🙄
😳
😡
😔
😫
😱
😭
💩
👻
🙌
🖕
👍
👫
👬
👭
🌚
🌝
🙈
💊
😶
🙏
🍦
🍉
😣
Source: github.com/k4yt3x/flowerhd
	

	

		
颜文字
Emoji
小恐龙
花!
	


									

			

			
			
		

	






上一篇
下一篇