主要函数

  • void *zmalloc(size_t size);

  • void *zcalloc(size_t size);

  • void *zrealloc(void *ptr, size_t size);

  • void zfree(void *ptr);

  • char *zstrdup(const char *s);

  • size_t zmalloc_used_memory(void);

  • void zmalloc_set_oom_handler(void (*oom_handler)(size_t));

  • size_t zmalloc_get_rss(void);

  • int zmalloc_get_allocator_info(size_t *allocated, size_t *active, size_t *resident);

  • size_t zmalloc_get_private_dirty(long pid);

  • size_t zmalloc_get_smap_bytes_by_field(char *field, long pid);

  • size_t zmalloc_get_memory_size(void);

  • void zlibc_free(void *ptr);

void *zmalloc(size_t size)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27


void *zmalloc(size_t size) {

    void *ptr = malloc(size+PREFIX_SIZE);



    if (!ptr) zmalloc_oom_handler(size);

#ifdef HAVE_MALLOC_SIZE

    update_zmalloc_stat_alloc(zmalloc_size(ptr));

    return ptr;

#else

    *((size_t*)ptr) = size;

    update_zmalloc_stat_alloc(size+PREFIX_SIZE);

    return (char*)ptr+PREFIX_SIZE;

#endif

}

参数size是需要分配的空间大小。事实上我们需要分配的空间大小为size+PREFIX_SIZE。PREFIX_SIZE是根据平台的不同和HAVE_MALLOC_SIZE宏定义控制的。如果malloc()函数调用失败，就会调用zmalloc_oom_handler()函数来打印异常，并且会终止函数，zmalloc_oom_handler其实是一个函数指针，真正调用的函数是zmalloc_default_oom(),zmalloc_default_oom()函数源码如下:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17


static void zmalloc_default_oom(size_t size) {

    fprintf(stderr, "zmalloc: Out of memory trying to allocate %zu bytes\n",

        size);

    fflush(stderr);

    abort();

}



static void (*zmalloc_oom_handler)(size_t) = zmalloc_default_oom;

内存分配成功之后，会依据HAVE_MALLOC_SIZE的控制对前八个字节操作，用以记录分配内存的长度，会在update_zmalloc_stat_alloc()宏定义函数中更新used_memory这个静态变量的值，update_zmalloc_stat_alloc()源码如下:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11


#define update_zmalloc_stat_alloc(__n) do { \

    size_t _n = (__n); \

    if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1)); \

    atomicIncr(used_memory,__n); \

} while(0)


1
2
3


if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1));

这一行是为了将不为sizeof(long)的_n对sizeof(long)补齐

atomicIncr(used_memory,__n);会调用__atomic_add_fetch(&var,(count),__ATOMIC_RELAXED);用于保证更新used_memory变量的操作是一个原子操作。

void *zcalloc(size_t size)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27


void *zcalloc(size_t size) {

    void *ptr = calloc(1, size+PREFIX_SIZE);



    if (!ptr) zmalloc_oom_handler(size);

#ifdef HAVE_MALLOC_SIZE

    update_zmalloc_stat_alloc(zmalloc_size(ptr));

    return ptr;

#else

    *((size_t*)ptr) = size;

    update_zmalloc_stat_alloc(size+PREFIX_SIZE);

    return (char*)ptr+PREFIX_SIZE;

#endif

}

zcalloc函数和zmalloc函数处理的思路很是相似，就不做太多的解释了

void *zrealloc(void *ptr, size_t size)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61


void *zrealloc(void *ptr, size_t size) {

    // 如果没有定义HAVE_MALLOC_SIZE，就说明PREFIX_SIZE宏定义不为0，那么ptr并不是该段内存真正的开始地址

#ifndef HAVE_MALLOC_SIZE

    void *realptr;

#endif

    size_t oldsize;

    void *newptr;



    if (ptr == NULL) return zmalloc(size);

// 根据HAVE_MALLOC_SIZE宏定义，oldsize,newptr获取方式不一样，以及更新used_memory的细节

#ifdef HAVE_MALLOC_SIZE

    oldsize = zmalloc_size(ptr);

    newptr = realloc(ptr,size);

    if (!newptr) zmalloc_oom_handler(size);



    update_zmalloc_stat_free(oldsize);

    update_zmalloc_stat_alloc(zmalloc_size(newptr));

    return newptr;

#else

    realptr = (char*)ptr-PREFIX_SIZE;

    oldsize = *((size_t*)realptr);

    newptr = realloc(realptr,size+PREFIX_SIZE);

    if (!newptr) zmalloc_oom_handler(size);



    *((size_t*)newptr) = size;

    update_zmalloc_stat_free(oldsize+PREFIX_SIZE);

    update_zmalloc_stat_alloc(size+PREFIX_SIZE);

    return (char*)newptr+PREFIX_SIZE;

#endif

}

大致思路就是根据新的size进行重新分配内存，并且对used_memory变量进行更新。只不过获取原内存大小方式不一样，根据HAVE_MALLOC_SIZE进行区分。

void zfree(void *ptr)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35


void zfree(void *ptr) {

#ifndef HAVE_MALLOC_SIZE

    void *realptr;

    size_t oldsize;

#endif



    if (ptr == NULL) return;

#ifdef HAVE_MALLOC_SIZE

    update_zmalloc_stat_free(zmalloc_size(ptr));

    free(ptr);

#else

    realptr = (char*)ptr-PREFIX_SIZE;

    oldsize = *((size_t*)realptr);

    update_zmalloc_stat_free(oldsize+PREFIX_SIZE);

    free(realptr);

#endif

}

其实zfree函数和zrealloc函数做法差不到太多，都是对oldsize和realptr对HAVE_MALLOC_SIZE有无声明分别进行操作。

char *zstrdup(const char *s)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15


char *zstrdup(const char *s) {

    size_t l = strlen(s)+1;

    char *p = zmalloc(l);



    memcpy(p,s,l);

    return p;

}

该函数是创建一个字符串副本

size_t zmalloc_used_memory(void)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11


size_t zmalloc_used_memory(void) {

    size_t um;

    atomicGet(used_memory,um);

    return um;

}

获取used_memory变量的值，主要保证原子操作(在atomicGet(used_memory,um);中保证)

void zmalloc_set_oom_handler(void (*oom_handler)(size_t))

1
2
3
4
5
6
7


void zmalloc_set_oom_handler(void (*oom_handler)(size_t)) {

    zmalloc_oom_handler = oom_handler;

}

主要用来设置内存分配失败处理函数指针zmalloc_oom_handler的值

size_t zmalloc_get_rss(void)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63


size_t zmalloc_get_rss(void) {

    int page = sysconf(_SC_PAGESIZE);

    size_t rss;

    char buf[4096];

    char filename[256];

    int fd, count;

    char *p, *x;



    snprintf(filename,256,"/proc/%d/stat",getpid());

    if ((fd = open(filename,O_RDONLY)) == -1) return 0;

    if (read(fd,buf,4096) <= 0) {

        close(fd);

        return 0;

    }

    close(fd);



    p = buf;

    count = 23; /* RSS is the 24th field in /proc/<pid>/stat */

    while(p && count--) {

        p = strchr(p,' ');

        if (p) p++;

    }

    if (!p) return 0;

    x = strchr(p,' ');

    if (!x) return 0;

    *x = '\0';



    rss = strtoll(p,NULL,10);

    rss *= page;

    return rss;

}

返回驻留集大小

int zmalloc_get_allocator_info(size_t *allocated, size_t *active, size_t *resident)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61


#if defined(USE_JEMALLOC)

int zmalloc_get_allocator_info(size_t *allocated,

                               size_t *active,

                               size_t *resident) {

    uint64_t epoch = 1;

    size_t sz;

    *allocated = *resident = *active = 0;

    /* Update the statistics cached by mallctl. */

    sz = sizeof(epoch);

    je_mallctl("epoch", &epoch, &sz, &epoch, sz);

    sz = sizeof(size_t);

    /* Unlike RSS, this does not include RSS from shared libraries and other non

     * heap mappings. */

    je_mallctl("stats.resident", resident, &sz, NULL, 0);

    /* Unlike resident, this doesn't not include the pages jemalloc reserves

     * for re-use (purge will clean that). */

    je_mallctl("stats.active", active, &sz, NULL, 0);

    /* Unlike zmalloc_used_memory, this matches the stats.resident by taking

     * into account all allocations done by this process (not only zmalloc). */

    je_mallctl("stats.allocated", allocated, &sz, NULL, 0);

    return 1;

}

#else

int zmalloc_get_allocator_info(size_t *allocated,

                               size_t *active,

                               size_t *resident) {

    *allocated = *resident = *active = 0;

    return 1;

}

#endif

获取分配器的信息，主要在使用jemalloc前提下使用，获取jemalloc分配的信息，详细信息可在http://jemalloc.net/jemalloc.3.html查阅

size_t zmalloc_get_smap_bytes_by_field(char *field, long pid)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73


#if defined(HAVE_PROC_SMAPS)

size_t zmalloc_get_smap_bytes_by_field(char *field, long pid) {

    char line[1024];

    size_t bytes = 0;

    int flen = strlen(field);

    FILE *fp;



    if (pid == -1) {

        // /proc/pid/smaps反应了运行时的进程的内存影响，系统的运行时库(so)，堆，栈信息均可在其中看到。

        fp = fopen("/proc/self/smaps","r");

    } else {

        char filename[128];

        snprintf(filename,sizeof(filename),"/proc/%ld/smaps",pid);

        fp = fopen(filename,"r");

    }



    if (!fp) return 0;

    while(fgets(line,sizeof(line),fp) != NULL) {

        if (strncmp(line,field,flen) == 0) {

            char *p = strchr(line,'k');

            if (p) {

                *p = '\0';

                bytes += strtol(line+flen,NULL,10) * 1024;

            }

        }

    }

    fclose(fp);

    return bytes;

}

#else

size_t zmalloc_get_smap_bytes_by_field(char *field, long pid) {

    ((void) field);

    ((void) pid);

    return 0;

}

#endif

获取/proc/pid/smaps中某一个field的字节大小

size_t zmalloc_get_private_dirty(long pid)

1
2
3
4
5
6
7


size_t zmalloc_get_private_dirty(long pid) {

    return zmalloc_get_smap_bytes_by_field("Private_Dirty:",pid);

}

获取Rss中已改写的私有页面页面大小

size_t zmalloc_get_memory_size(void)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85


size_t zmalloc_get_memory_size(void) {

#if defined(__unix__) || defined(__unix) || defined(unix) || \

    (defined(__APPLE__) && defined(__MACH__))

#if defined(CTL_HW) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM64))

    int mib[2];

    mib[0] = CTL_HW;

#if defined(HW_MEMSIZE)

    mib[1] = HW_MEMSIZE;            /* OSX. --------------------- */

#elif defined(HW_PHYSMEM64)

    mib[1] = HW_PHYSMEM64;          /* NetBSD, OpenBSD. --------- */

#endif

    int64_t size = 0;               /* 64-bit */

    size_t len = sizeof(size);

    if (sysctl( mib, 2, &size, &len, NULL, 0) == 0)

        return (size_t)size;

    return 0L;          /* Failed? */



#elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)

    /* FreeBSD, Linux, OpenBSD, and Solaris. -------------------- */

    return (size_t)sysconf(_SC_PHYS_PAGES) * (size_t)sysconf(_SC_PAGESIZE);



#elif defined(CTL_HW) && (defined(HW_PHYSMEM) || defined(HW_REALMEM))

    /* DragonFly BSD, FreeBSD, NetBSD, OpenBSD, and OSX. -------- */

    int mib[2];

    mib[0] = CTL_HW;

#if defined(HW_REALMEM)

    mib[1] = HW_REALMEM;        /* FreeBSD. ----------------- */

#elif defined(HW_PHYSMEM)

    mib[1] = HW_PHYSMEM;        /* Others. ------------------ */

#endif

    unsigned int size = 0;      /* 32-bit */

    size_t len = sizeof(size);

    if (sysctl(mib, 2, &size, &len, NULL, 0) == 0)

        return (size_t)size;

    return 0L;          /* Failed? */

#else

    return 0L;          /* Unknown method to get the data. */

#endif

#else

    return 0L;          /* Unknown OS. */

#endif

}

获取物理内存的字节数

总结

看了redis内存分配的源码后，其实没有相信中的那么难以理解，或许只是心理上的作用，当然也说明redis源码写得真的是好，让我这种渣渣都能轻而易举的看懂，并且注释也很少，这里的函数几乎都是对glibc的malloc中的函数进行了一层包装，并且维护了一个叫做used_memory的全局变量，并且每一次对全局变量的操作都是原子操作。也对一些常用的函数进行了封装，例如：获取rss的大小，获取/proc/pid/smaps文件中某一field占用字节数的大小，获取物理内存字节数等等，总的来说，收益匪浅，没想到内存操作可以做到这样简单。