Memory Debugging: Detecting and Resolving Memory Errors

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

Application Layer
├── malloc(1024)                    // User requests 1KB
│
Memory Allocator (libc)
├── Lock allocator mutex            // Thread safety
├── Search free lists               // Find suitable block
├── Split large block if needed     // Fragmentation management
├── If no suitable block found:
│   └── sbrk() or mmap()           // Request more from OS
├── Update metadata                 // Track allocation
├── Unlock mutex
└── Return pointer to user
│
Operating System Kernel
├── Update virtual memory mappings  // For mmap()/sbrk()
├── Page tables modified            // If new pages needed
└── Physical frames allocated       // On-demand (page fault)
│
Hardware
└── MMU translates virtual → physical

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

// Typical memory block layout (simplified)
// 
// ┌─────────────────────────────────────────────────┐
// │  Metadata (hidden from user)                    │
// ├─────────────────────────────────────────────────┤
// │  size_t  block_size;     // Total block size   │
// │  void*   next_free;      // For free list      │
// │  void*   prev_free;      // Doubly-linked      │
// │  size_t  flags;          // USED, FREE, etc.   │
// ├─────────────────────────────────────────────────┤
// │  User Data (pointer returned to caller)        │  ← malloc returns this
// │  ...                                           │
// │  ...                                           │
// ├─────────────────────────────────────────────────┤
// │  Padding/Guard bytes (optional)                │
// └─────────────────────────────────────────────────┘
 
// Example: what malloc(100) actually allocates
// Requested: 100 bytes
// Metadata:  24 bytes (struct size)
// Alignment: 8 bytes (padding to 16-byte boundary)
// Total:     132 bytes (or more)
 
void* ptr = malloc(100);
// ptr points to User Data section
// Allocator secretly maintains metadata at ptr - 24

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

// BUGGY CODE - Memory leak on error paths
int process_data(const char* filename) {
    FILE* file = fopen(filename, "r");
    if (!file) return -1;
    
    char* buffer = malloc(4096);
    if (!buffer) return -2;  // LEAK: file handle not closed
    
    char* header = malloc(256);
    if (!header) return -3;  // LEAK: buffer not freed, file not closed
    
    // Read and process...
    if (fread(header, 1, 256, file) != 256) {
        return -4;  // LEAK: header, buffer not freed, file not closed
    }
    
    if (!validate_header(header)) {
        return -5;  // LEAK: same as above
    }
    
    // More processing...
    
    // Cleanup only reached on success
    free(header);
    free(buffer);
    fclose(file);
    return 0;
}
 
// FIXED CODE - Proper cleanup on all paths
int process_data_fixed(const char* filename) {
    int result = 0;
    FILE* file = NULL;
    char* buffer = NULL;
    char* header = NULL;
    
    file = fopen(filename, "r");
    if (!file) { result = -1; goto cleanup; }
    
    buffer = malloc(4096);
    if (!buffer) { result = -2; goto cleanup; }
    
    header = malloc(256);
    if (!header) { result = -3; goto cleanup; }
    
    if (fread(header, 1, 256, file) != 256) {
        result = -4; goto cleanup;
    }
    
    if (!validate_header(header)) {
        result = -5; goto cleanup;
    }
    
    // Success path processing...
    
cleanup:
    // All paths lead here
    free(header);   // free(NULL) is safe
    free(buffer);   // free(NULL) is safe
    if (file) fclose(file);
    return result;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

// Who owns this string?
typedef struct {
    char* name;
    char* address;
} Person;
 
// Option A: Caller retains ownership
void set_name_reference(Person* p, char* name) {
    p->name = name;  // Just stores pointer
    // Problem: What if caller frees name? Use-after-free!
    // Problem: What if old p->name exists? Leak!
}
 
// Option B: Function takes ownership
void set_name_transfer(Person* p, char* name) {
    free(p->name);   // Free old value
    p->name = name;  // Take ownership of new value
    // Problem: Caller might use 'name' after this call
    // Problem: Caller might pass stack or static memory
}
 
// Option C: Function copies (defensive)
void set_name_copy(Person* p, const char* name) {
    char* copy = strdup(name);
    if (!copy) return;  // Handle allocation failure
    free(p->name);      // Free old value
    p->name = copy;     // Store copy
    // Clear ownership: struct owns the copy, caller owns original
}
 
// The real solution: Document ownership explicitly
/**
 * set_name_copy - Set person's name, taking a copy
 * @p: Person struct (must not be NULL)
 * @name: Name to copy (must not be NULL)
 * 
 * The Person struct takes ownership of a copy of the name.
 * Caller retains ownership of the original name string.
 * Previous name is freed if present.
 */

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

// Classic: Reassignment without freeing
void process_items(void) {
    char* data = malloc(1000);
    
    // ... use data for first item ...
    
    // LEAK: Original data is now unreachable
    data = malloc(2000);  // New allocation, old one lost forever
    
    // ... use data for second item ...
    
    free(data);  // Only frees the second allocation
}
 
// Fixed version
void process_items_fixed(void) {
    char* data = malloc(1000);
    
    // ... use data for first item ...
    
    // Proper: Free before reassignment
    char* new_data = malloc(2000);
    if (new_data) {
        free(data);
        data = new_data;
    }
    // Or use realloc if growing the same logical buffer
    
    free(data);
}
 
// Variant: Leaking in loops
void process_lines(FILE* file) {
    char* line = NULL;
    size_t len = 0;
    
    while (getline(&line, &len, file) != -1) {
        // getline reuses buffer when possible, but...
        
        // LEAK: If we do this:
        line = strdup(line);  // Original buffer is lost!
        
        // Process duplicated line...
    }
    free(line);  // Only frees last duplication, not getline buffer
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

// Linked list where each node owns its data
typedef struct Node {
    char* data;           // Owned dynamically allocated string
    struct Node* next;
} Node;
 
typedef struct {
    Node* head;
    size_t count;
} LinkedList;
 
// BUGGY: Leaks all node data
void list_destroy_buggy(LinkedList* list) {
    Node* current = list->head;
    while (current) {
        Node* next = current->next;
        free(current);  // Frees node, but node->data is leaked!
        current = next;
    }
    list->head = NULL;
    list->count = 0;
}
 
// CORRECT: Free owned data before freeing container
void list_destroy_correct(LinkedList* list) {
    Node* current = list->head;
    while (current) {
        Node* next = current->next;
        free(current->data);  // Free owned data first
        free(current);        // Then free the node
        current = next;
    }
    list->head = NULL;
    list->count = 0;
}
 
// Even better: Destructor function pointer for flexibility
typedef void (*DestroyFunc)(void*);
 
void list_destroy_generic(LinkedList* list, DestroyFunc destroy_data) {
    Node* current = list->head;
    while (current) {
        Node* next = current->next;
        if (destroy_data) {
            destroy_data(current->data);
        }
        free(current);
        current = next;
    }
    list->head = NULL;
    list->count = 0;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

Linux OOM Killer Response:
 
1. Memory allocation fails internally
2. Kernel cannot reclaim enough memory
3. OOM killer is invoked
 
Selection Algorithm (simplified):
─────────────────────────────────
For each process P:
  score = (P.rss / total_ram) * 1000
  
  Adjustments:
  - Root processes: -30 points
  - Long-running processes: slight reduction
  - Recently forked: slight increase
  - oom_score_adj setting: direct adjustment (-1000 to +1000)
  
Selected victim = process with highest score
 
Note: The leaking process may not be killed if:
- It has low oom_score_adj (protected)
- Another process is using more RSS at this moment
- The leaker has been running longer (penalty)
 
This can lead to innocent processes being killed
while the actual leaker continues!

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

# Watch a process's memory usage over time
# RSS = Resident Set Size (physical memory used)
# VSZ = Virtual Size (total virtual address space)
 
# Simple: Watch with top
top -p <PID>
 
# Continuous logging
while true; do
    ps -p <PID> -o pid,rss,vsz,comm
    sleep 60
done >> memory_log.txt
 
# More detailed with /proc
watch -n 1 cat /proc/<PID>/status | grep -E 'VmRSS|VmSize|VmPeak'
 
# Memory map analysis
cat /proc/<PID>/smaps | grep -E '^[0-9a-f]|Rss|Pss'
 
# Track allocations indirectly via malloc_stats() in glibc
# Add to your program:
# #include <malloc.h>
# malloc_info(0, stdout);  // XML output of heap state

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

# glibc malloc debugging
export MALLOC_CHECK_=3        # Enable consistency checks
export MALLOC_TRACE=/tmp/mtrace.log
# Then run your program
 
# Analyze with mtrace
mtrace ./your_program /tmp/mtrace.log
 
# macOS with leaks utility
leaks --atExit -- ./your_program
 
# jemalloc profiling (requires jemalloc build)
export MALLOC_CONF="prof:true,prof_prefix:jeprof.out"
./your_program
# Analyze with jeprof
 
# tcmalloc heap profiler
HEAPPROFILE=/tmp/myprofile HEAP_PROFILE_ALLOCATION_INTERVAL=1073741824 ./your_program
pprof --pdf ./your_program /tmp/myprofile.0001.heap > heap.pdf

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

# Clang Static Analyzer
scan-build make
 
# Cppcheck
cppcheck --enable=warning,performance,style --inconclusive ./src
 
# Coverity (commercial, very thorough)
cov-build --dir cov-int make
cov-analyze --dir cov-int
cov-format-errors --dir cov-int --html-output report
 
# GCC with -fanalyzer (GCC 10+)
gcc -fanalyzer -Wall -Wextra source.c
 
# PVS-Studio (commercial)
pvs-studio-analyzer analyze -o log.log
plog-converter -a GA:1,2 -t tasklist log.log

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

// Clear ownership documented in function signatures and comments
 
/**
 * create_buffer - Allocate a new buffer
 * @size: Size in bytes
 * 
 * Returns: Newly allocated buffer (caller takes ownership), 
 *          or NULL on failure
 * 
 * Caller is responsible for calling destroy_buffer() when done.
 */
Buffer* create_buffer(size_t size);
 
/**
 * destroy_buffer - Free a buffer and all associated resources
 * @buf: Buffer to destroy (ownership transferred to this function)
 * 
 * After this call, the pointer is invalid. Passing NULL is safe.
 */
void destroy_buffer(Buffer* buf);
 
/**
 * get_buffer_data - Get pointer to buffer's internal data
 * @buf: Buffer to access
 * 
 * Returns: Pointer to internal data (NOT owned by caller)
 * 
 * Note: Returned pointer is valid only while buf is valid.
 * Do NOT free the returned pointer.
 */
const char* get_buffer_data(const Buffer* buf);
 
/**
 * copy_buffer - Create a copy of a buffer
 * @src: Source buffer (read-only, ownership retained by caller)
 * 
 * Returns: New buffer (caller takes ownership), or NULL on failure
 */
Buffer* copy_buffer(const Buffer* src);

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

// C++: RAII with smart pointers
#include <memory>
#include <vector>
#include <fstream>
 
class DataProcessor {
private:
    // unique_ptr: Single ownership, auto-deleted when Processor destroyed
    std::unique_ptr<char[]> buffer_;
    
    // shared_ptr: Shared ownership, deleted when last owner gone
    std::shared_ptr<Config> config_;
    
    // RAII for non-memory resources
    std::ifstream file_;  // Auto-closed when object destroyed
    
public:
    DataProcessor(size_t buffer_size, std::shared_ptr<Config> config)
        : buffer_(std::make_unique<char[]>(buffer_size))
        , config_(std::move(config))
        , file_("data.txt")  // Opens file in constructor
    {
        if (!file_.is_open()) {
            throw std::runtime_error("Failed to open file");
            // Note: No cleanup needed - unique_ptr handles buffer_
        }
    }
    
    // Destructor is implicitly correct:
    // - buffer_ is automatically freed
    // - config_ decrements refcount (freed if last)
    // - file_ is automatically closed
    ~DataProcessor() = default;
    
    // No manual cleanup needed anywhere!
};
 
// Exception-safe function - no leaks possible
void process_with_raii() {
    auto processor = std::make_unique<DataProcessor>(4096, config);
    
    do_something_that_might_throw();  // If this throws...
    
    // processor is automatically destroyed - no leak
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

// The goto cleanup pattern - used throughout Linux kernel
 
int complex_operation(const char* input) {
    int ret = 0;
    
    // Initialize all resources to NULL/invalid state
    char* buffer1 = NULL;
    char* buffer2 = NULL;
    FILE* file = NULL;
    int fd = -1;
    
    // Allocate resources, check each, goto cleanup on failure
    buffer1 = malloc(1024);
    if (!buffer1) {
        ret = -ENOMEM;
        goto cleanup;
    }
    
    buffer2 = malloc(2048);
    if (!buffer2) {
        ret = -ENOMEM;
        goto cleanup;
    }
    
    file = fopen("/tmp/data", "r");
    if (!file) {
        ret = -EIO;
        goto cleanup;
    }
    
    fd = open("/dev/device", O_RDWR);
    if (fd < 0) {
        ret = -errno;
        goto cleanup;
    }
    
    // Main operation - any failure uses goto cleanup
    if (read_data(file, buffer1) < 0) {
        ret = -EIO;
        goto cleanup;
    }
    
    if (process_data(buffer1, buffer2) < 0) {
        ret = -EINVAL;
        goto cleanup;
    }
    
    if (write_to_device(fd, buffer2) < 0) {
        ret = -EIO;
        goto cleanup;
    }
    
    // Success!
    ret = 0;
    
cleanup:
    // Single cleanup location - all paths lead here
    // Order: reverse of acquisition (though not strictly required)
    if (fd >= 0) close(fd);
    if (file) fclose(file);
    free(buffer2);  // free(NULL) is safe
    free(buffer1);  // free(NULL) is safe
    
    return ret;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64

// Simple arena allocator - impossible to leak individual allocations
 
typedef struct {
    char* base;
    char* current;
    size_t capacity;
} Arena;
 
Arena* arena_create(size_t capacity) {
    Arena* arena = malloc(sizeof(Arena));
    if (!arena) return NULL;
    
    arena->base = malloc(capacity);
    if (!arena->base) {
        free(arena);
        return NULL;
    }
    
    arena->current = arena->base;
    arena->capacity = capacity;
    return arena;
}
 
// Allocate from arena - no individual free needed
void* arena_alloc(Arena* arena, size_t size) {
    size_t aligned_size = (size + 7) & ~7;  // 8-byte alignment
    
    if (arena->current + aligned_size > arena->base + arena->capacity) {
        return NULL;  // Arena exhausted
    }
    
    void* ptr = arena->current;
    arena->current += aligned_size;
    return ptr;
}
 
// Reset arena - all allocations become invalid
void arena_reset(Arena* arena) {
    arena->current = arena->base;
    // All previous allocations are now "freed" implicitly
}
 
// Destroy arena - single cleanup frees everything
void arena_destroy(Arena* arena) {
    free(arena->base);  // One free instead of many
    free(arena);
}
 
// Usage: Per-request arena in a server
void handle_request(Request* req) {
    Arena* arena = arena_create(64 * 1024);  // 64KB per request
    
    // All allocations during request handling use arena
    char* name = arena_alloc(arena, strlen(req->name) + 1);
    strcpy(name, req->name);
    
    char* data = arena_alloc(arena, req->data_len);
    memcpy(data, req->data, req->data_len);
    
    // ... lots more allocations ...
    
    // One destroy cleans up everything - no possibility of leaks
    arena_destroy(arena);
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

// Simplified illustration of DTLS memory leak pattern
 
int dtls_process_handshake(SSL* s, unsigned char* data, size_t len) {
    DTLSMessage* msg = NULL;
    
    // Parse message header
    if (!parse_header(data, &msg->header)) {
        return -1;  // Early return - no allocation yet, OK
    }
    
    // Allocate message buffer
    msg = malloc(sizeof(DTLSMessage));
    if (!msg) return -1;
    
    msg->body = malloc(msg->header.length);
    if (!msg->body) {
        free(msg);
        return -1;  // Proper cleanup here
    }
    
    // Copy message body
    memcpy(msg->body, data + HEADER_SIZE, msg->header.length);
    
    // Validate message
    if (!validate_message(msg)) {
        // BUG: In certain error paths, the following free was missing
        // free(msg->body);
        // free(msg);
        return -1;  // LEAK: msg and msg->body leaked
    }
    
    // Additional processing that might fail
    if (msg->header.type == MSG_CERTIFICATE) {
        if (!process_certificate(msg)) {
            // BUG: Another path where cleanup was forgotten
            return -1;  // LEAK
        }
    }
    
    // Normal processing and eventual cleanup
    // ...
    
    free(msg->body);
    free(msg);
    return 0;
}
 
// The fix: consolidated cleanup
int dtls_process_handshake_fixed(SSL* s, unsigned char* data, size_t len) {
    int ret = -1;
    DTLSMessage* msg = NULL;
    
    // ... allocation code ...
    
    if (!validate_message(msg)) {
        goto cleanup;  // All errors go to cleanup
    }
    
    if (msg->header.type == MSG_CERTIFICATE) {
        if (!process_certificate(msg)) {
            goto cleanup;
        }
    }
    
    ret = 0;  // Success
    
cleanup:
    if (msg) {
        free(msg->body);
        free(msg);
    }
    return ret;
}