Hello. I'm trying to write an application that can fork and launch many thousands of lightweight child processes (targeting 20,000 child processes) and read their TTY. The child process it is launching just prints out a line of text in a loop, sleeping for 100 seconds in between prints. I've increased the maximum number of files and processes, but after forking and execing 3246 child processes, the following forks fail with ENOMEM. There is plenty of RAM (3GB free) left available.

I can continue to open files, so I haven't exceeded any maximum file amount. It seems to be an issue of the system running out of pipes possibly. For instance, if in my code below I simply don't create a pipe for stdin for each of the child processes, I can launch about another 1K processes. I can't simply close the pipe in the parent process because I need to read the TTY. I've looked but can't seem to find any way to increase the pipe capacity on the system.

Anyways, sample code illustrating the issue is below. Any help would be very much appreciated.

Thanks,
Matt

Code:
#include <stdio.h>
#include <unistd.h>
#include <fcntl.h>
#include <malloc.h>

#include <sys/resource.h>
#include <sys/types.h>

#include <string.h>

#include <errno.h>

#define SET_NONBLOCK(fd)     fcntl(fd, F_SETFL, fcntl(fd,F_GETFL)|O_NONBLOCK)

void setrlimit(unsigned int numFiles, unsigned int numProcs) {
	struct rlimit n_limit;

	int ret = getrlimit(RLIMIT_NOFILE, &n_limit);
	fprintf(stderr, "oldNumFilesCur = (cur=%d, max=%d)\n", n_limit.rlim_cur, n_limit.rlim_max);
	n_limit.rlim_max = numFiles;
	n_limit.rlim_cur = numFiles;
	setrlimit(RLIMIT_NOFILE, &n_limit);
	fprintf(stderr, "newNumFilesCur = (cur=%d, max=%d)\n", n_limit.rlim_cur, n_limit.rlim_max);

	ret = getrlimit(RLIMIT_NPROC, &n_limit);
	fprintf(stderr, "oldNumProcsCur = (cur=%d, max=%d)\n", n_limit.rlim_cur, n_limit.rlim_max);
	n_limit.rlim_max = numProcs;
	n_limit.rlim_cur = numProcs;
	setrlimit(RLIMIT_NPROC, &n_limit);
	fprintf(stderr, "newNumProcsCur = (cur=%d, max=%d)\n", n_limit.rlim_cur, n_limit.rlim_max);
}

int main (int argc, char *argv[]) {
	const char *execArgs[] = {"/home/dartmaster/image/tty_spam/tty_spam", "100000000", NULL};

	setrlimit(200000, 77824);

	int hStdout[2] = {-1, -1};	
	int hStderr[2] = {-1, -1};	
	int hStdin[2] = {-1, -1};	

	int hFilesToClose[20480];
	int totalFilesToCloseCount = 0;

	for (int i = 0; i < 99000; i++) {	
		int ret = pipe(hStdout);
		if (ret == -1) {
			fprintf(stderr, "pipe FAILED\n");
			return 0;
		}
		ret = pipe(hStderr);
		if (ret == -1) {
			fprintf(stderr, "pipe FAILED\n");
			return 0;
		}
		ret = pipe(hStdin);
		if (ret == -1) {
			fprintf(stderr, "pipe FAILED\n");
			return 0;
		}

		SET_NONBLOCK(hStdout[0]);
		SET_NONBLOCK(hStderr[0]);
		SET_NONBLOCK(hStdin[1]);

		hFilesToClose[totalFilesToCloseCount++] = hStdout[0];
		hFilesToClose[totalFilesToCloseCount++] = hStderr[0];
		hFilesToClose[totalFilesToCloseCount++] = hStdin[1];
		close(hStdout[1]);
		close(hStderr[1]);
		close(hStdin[0]);

		fprintf(stderr, "Done with #%d\n", i);

		fcntl(hStdout[0], F_SETFD, FD_CLOEXEC);	
		fcntl(hStderr[0], F_SETFD, FD_CLOEXEC);	
		fcntl(hStdin[1], F_SETFD, FD_CLOEXEC);

		int pid = fork();
		if (pid == -1) {
			fprintf(stderr, "Fork failed with errno %d\n", errno);
			fprintf(stderr, "Failed forked for the %dth process\n", i);
			while(1) {
				usleep(1000000000);
			}
			return 0;
		} else if (pid == 0) {
			//child process
			setrlimit(1024, 1024);

			fprintf(stderr, "Succesfully forked for the %dth process, closing files\n", i);
			for (int j = 0; j < totalFilesToCloseCount; j++) {
				close(hFilesToClose[j]);
			}	
			close(STDOUT_FILENO);
			close(STDERR_FILENO);
			close(STDIN_FILENO);
			dup2(hStdout[1], STDOUT_FILENO);
			dup2(hStderr[1], STDERR_FILENO);
			dup2(hStdin[0], STDIN_FILENO);
			close(hStdout[1]);
			close(hStderr[1]);
			close(hStdin[0]);

			execvp(execArgs[0], (char * const *)execArgs);
			break;
		} else {
			close(hStdout[1]);
			close(hStderr[1]);
			close(hStdin[0]);
		}
	}
	while(1) {
		usleep(1000000000);
	}
	return 0;
}