Hi,

I am trying to find out what libraries & languages that would be smartest to base a UDP server where many CCU's ( 10K-50K) - and ultraspeed is the top importance - what do people recommend here in 2021 ?

Ive been dancing around Enet and can also see discussions here but they all seem to be years back in time so wondering if there are better options out there ?

Preferable languages : C, Java, Go

Preferable OS : Linux

Bonus : Android client side also

Am not so fond of C# unfortunately but if this is the go-to-language to use then its an option too.

What do you guys use today for serverside development ?

I use C++. Below is a code to do UDP sending and receiving. It uses winsock, but the change to Linux shouldn't be too difficult.

#include <winsock2.h>
#include <Ws2tcpip.h>
#include <windows.h>
#pragma comment(lib, "ws2_32")

#include <iostream>
#include <string>
#include <sstream>
#include <cstring>
using std::cout;
using std::endl;
using std::string;
using std::istringstream;
using std::ostringstream;
using std::ios;

bool stop = false;
SOCKET udp_socket = INVALID_SOCKET;
enum program_mode { talk_mode, listen_mode };

void print_usage(void)
{
	cout << "  USAGE:" << endl;
	cout << "    Listen mode:" << endl;
	cout << "      udpspeed PORT_NUMBER" << endl;
	cout << endl;
	cout << "    Talk mode:" << endl;
	cout << "      udpspeed TARGET_HOST PORT_NUMBER" << endl;
	cout << endl;
	cout << "    ie:" << endl;
	cout << "      Listen mode: udpspeed 1920" << endl;
	cout << "      Talk mode:   udpspeed www 342" << endl;
	cout << "      Talk mode:   udpspeed 127.0.0.1 950" << endl;
	cout << endl;
}

bool verify_port(const string &port_string, unsigned long int &port_number)
{
	for (size_t i = 0; i < port_string.length(); i++)
	{
		if (!isdigit(port_string[i]))
		{
			cout << "  Invalid port: " << port_string << endl;
			cout << "  Ports are specified by numerals only." << endl;
			return false;
		}
	}

	istringstream iss(port_string);
	iss >> port_number;

	if (port_string.length() > 5 || port_number > 65535 || port_number == 0)
	{
		cout << "  Invalid port: " << port_string << endl;
		cout << "  Port must be in the range of 1-65535" << endl;
		return false;
	}

	return true;
}

bool init_winsock(void)
{
	WSADATA wsa_data;
	WORD ver_requested = MAKEWORD(2, 2);

	if (WSAStartup(ver_requested, &wsa_data))
	{
		cout << "Could not initialize Winsock 2.2.";
		return false;
	}

	if (LOBYTE(wsa_data.wVersion) != 2 || HIBYTE(wsa_data.wVersion) != 2)
	{
		cout << "Required version of Winsock (2.2) not available.";
		return false;
	}

	return true;
}

BOOL console_control_handler(DWORD control_type)
{
	stop = true;
	closesocket(udp_socket);

	return TRUE;
}

bool init_options(const int &argc, char **argv, enum program_mode &mode, string &target_host_string, long unsigned int &port_number)
{
	if (!SetConsoleCtrlHandler((PHANDLER_ROUTINE)console_control_handler, TRUE))
	{
		cout << "  Could not add console control handler." << endl;
		return false;
	}

	if (!init_winsock())
		return false;

	string port_string = "";

	if (2 == argc)
	{
		mode = listen_mode;
		port_string = argv[1];
	}
	else if (3 == argc)
	{
		mode = talk_mode;
		target_host_string = argv[1];
		port_string = argv[2];
	}
	else
	{
		print_usage();
		return false;
	}

	cout.setf(ios::fixed, ios::floatfield);
	cout.precision(2);

	return verify_port(port_string, port_number);
}

void cleanup(void)
{
	// if the program was aborted, flush cout and print a final goodbye
	if (stop)
	{
		cout.flush();
		cout << endl << "  Stopping." << endl;
	}

	// if the socket is still open, close it
	if (INVALID_SOCKET != udp_socket)
		closesocket(udp_socket);

	// shut down winsock
	WSACleanup();

	// remove the console control handler
	SetConsoleCtrlHandler((PHANDLER_ROUTINE)console_control_handler, FALSE);
}

int main(int argc, char **argv)
{
	cout << endl << "udpspeed 1.2 - UDP speed tester" << endl << "Copyright 2002/2018, Shawn Halayka" << endl << endl;

	program_mode mode = listen_mode;

	string target_host_string = "";
	long unsigned int port_number = 0;

	const long unsigned int tx_buf_size = 1450;
	char tx_buf[1450];

	const long unsigned int rx_buf_size = 8196;
	char rx_buf[8196];

	// initialize winsock and all of the program's options
	if (!init_options(argc, argv, mode, target_host_string, port_number))
	{
		cleanup();
		return 1;
	}

	if (talk_mode == mode)
	{
		cout << "  Sending on port " << port_number << " - CTRL+C to exit." << endl;

		struct addrinfo hints;
		struct addrinfo *result;

		memset(&hints, 0, sizeof(struct addrinfo));
		hints.ai_family = AF_INET;
		hints.ai_socktype = SOCK_DGRAM;
		hints.ai_flags = 0;
		hints.ai_protocol = IPPROTO_UDP;

		ostringstream oss;
		oss << port_number;

		if(0 != getaddrinfo(target_host_string.c_str(), oss.str().c_str(), &hints, &result))
		{
			cout << "  getaddrinfo error." << endl;
			freeaddrinfo(result);
			cleanup();
			return 2;
		}

		if (INVALID_SOCKET == (udp_socket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)))
		{
			cout << "  Could not allocate a new socket." << endl;
			freeaddrinfo(result);
			cleanup();
			return 3;
		}

		while (!stop)
		{
			if (SOCKET_ERROR == (sendto(udp_socket, tx_buf, tx_buf_size, 0, result->ai_addr, sizeof(struct sockaddr))))
			{

				if (!stop)
					cout << " (TX ERR)" << endl;

				break;
			}
		}

		freeaddrinfo(result);
	}
	else if (listen_mode == mode)
	{
		cout << "  Listening on UDP port " << port_number << " - CTRL+C to exit." << endl;

		struct sockaddr_in my_addr;
		struct sockaddr_in their_addr;
		int addr_len = 0;

		my_addr.sin_family = AF_INET;
		my_addr.sin_port = htons((unsigned short int)port_number);
		my_addr.sin_addr.s_addr = INADDR_ANY;
		memset(&(my_addr.sin_zero), '\0', 8);
		addr_len = sizeof(struct sockaddr);

		if (INVALID_SOCKET == (udp_socket = socket(AF_INET, SOCK_DGRAM, 0)))
		{
			cout << "  Could not allocate a new socket." << endl;
			cleanup();
			return 4;
		}

		if (SOCKET_ERROR == bind(udp_socket, (struct sockaddr *) &my_addr, sizeof(struct sockaddr)))
		{
			cout << "  Could not bind socket to port " << port_number << "." << endl;
			cleanup();
			return 5;
		}

		long unsigned int start_loop_ticks = 0;
		long unsigned int end_loop_ticks = 0;
		long unsigned int elapsed_loop_ticks = 0;

		long long unsigned int total_elapsed_ticks = 0;
		long long unsigned int total_bytes_received = 0;
		long long unsigned int last_reported_at_ticks = 0;
		long long unsigned int last_reported_total_bytes_received = 0;

		double record_bps = 0;
		long unsigned int temp_bytes_received = 0;

		while (!stop)
		{
			start_loop_ticks = GetTickCount();

			if (SOCKET_ERROR == (temp_bytes_received = recvfrom(udp_socket, rx_buf, rx_buf_size, 0, (struct sockaddr *) &their_addr, &addr_len)))
			{
				if (!stop)
				{
					cout << "  Socket error." << endl;
					cleanup();
					return 6;
				}
			}
			else
			{
				total_bytes_received += temp_bytes_received;
			}

			end_loop_ticks = GetTickCount();

			if (end_loop_ticks < start_loop_ticks)
				elapsed_loop_ticks = MAXDWORD - start_loop_ticks + end_loop_ticks;
			else
				elapsed_loop_ticks = end_loop_ticks - start_loop_ticks;

			total_elapsed_ticks += elapsed_loop_ticks;

			if (total_elapsed_ticks >= last_reported_at_ticks + 1000)
			{
				long long unsigned int bytes_sent_received_between_reports = total_bytes_received - last_reported_total_bytes_received;

				double bytes_per_second = static_cast<double>(bytes_sent_received_between_reports) / ((static_cast<double>(total_elapsed_ticks) - static_cast<double>(last_reported_at_ticks)) / 1000.0);

				if (bytes_per_second > record_bps)
					record_bps = bytes_per_second;

				last_reported_at_ticks = total_elapsed_ticks;
				last_reported_total_bytes_received = total_bytes_received;

				static const double mbits_factor = 8.0 / (1024.0 * 1024);
				cout << "  " << bytes_per_second * mbits_factor << " Mbit/s, Record: " << record_bps * mbits_factor << " Mbit/s" << endl;
			}
		}
	}

	cleanup();

	return 0;
}

joopyM said:
I am trying to find out what libraries & languages that would be smartest to base a UDP server where many CCU's ( 10K-50K) - and ultraspeed is the top importance - what do people recommend here in 2021 ?

Most any modern server framework can be tuned to handle 50k concurrent connections - that was mostly a solved problem by the millennium. The limiting factors are liable to be message forwarding, cost of the game logic itself, and request handling (in that rough order). To understand those, we need more data ?

How many requests per second will each client send on average? How wide is the average fan out (i.e. when a single user takes an action, how many other users does the server need to notify)? How much raw compute will the game logic require on each request?

swiftcoder said:

joopyM said:
I am trying to find out what libraries & languages that would be smartest to base a UDP server where many CCU's ( 10K-50K) - and ultraspeed is the top importance - what do people recommend here in 2021 ?

Most any modern server framework can be tuned to handle 50k concurrent connections - that was mostly a solved problem by the millennium. The limiting factors are liable to be message forwarding, cost of the game logic itself, and request handling (in that rough order). To understand those, we need more data ?

How many requests per second will each client send on average? How wide is the average fan out (i.e. when a single user takes an action, how many other users does the server need to notify)? How much raw compute will the game logic require on each request?

Part of my issue is i am going to have pretty heavy load in an audio part where the a client should be able to stream its microphone to other players that are inside that room/hall an idea is also be able to ‘stream’ some camera possible in the future as part of AR but this is still just on very loose wild thoughts- the sound records in 20 ms packets and i intend to send it as fast as possible but maybe accumulating 5-10 audio packets - this is where i worry the most about traffic and load and speed.

The other part of the server that handles the game itself is smaller packets and consist of states and coordinates back and forth but havent completed this part yet as i would like to find the best suitable approach for being able to handle both this audio situation & the game logic.

[quote]ultraspeed is the top importance[/quote]

If that is the case, you want to use C/C++ or Rust for the code. Java has reasonable performance (about 50% of good C/C++/Rust) but has occasional pauses for garbage collection and synchronization (even with the modern low-latency GC options) and Go is just a distant third, with very poor code generation, slow garbage collection, and abysmal performance at picking apart network protocols. (Both JSON and Protobuf decoders for Go run at some small fraction of similar functions in C++ and other high-performance languages.)

As others have said, 50,000 clients isn't out of the question for a single socket these days, using parallel I/O mechanisms (evented I/O on Linux, IOCP on windows.) Although if there's significant bulk involved (audio and video chat) then you may want to split it up for that reason. Even if you get 10 Gbps network interfaces, you don't want your normal throughput to be sent out of whack by a single large packet, dropped packet, buffer stall, or similar interruption. You can look at real-time interactive video services like caffeine.tv (used for rap battles streaming for example) where they don't jam 50,000 people onto a single server :-)

Also, good quality video might be 2 Mbps or so, and even at full utilization, a 10 Gbps channel can only do a max of 5000 streams in parallel theoretically (and much less than that in practice.)

@hplus0603 thanks alot - really good points ! and i agree video was probably a bad brainstorm :D sound though i really would like to be able to pass through and its abit this that is my starting goal which is why im trying to aim at speed ( will be using C/C++ on Linux ) as if i can do that with lets say 1000 clients then my base must be hopefully solid to also handle less amount of data.

But dont know if it can handle 1000 or 100 on a server if the theory was to stream soundpackets to all clients - it would be unrealistic to stream to 1000 at the same time but in theory i guess its possible with some kind of traffic if every area/room has someone speaking then there could be generated alot of packets that are to be rerouted - but am still totally blank here what to expect or if i end up with only being able to support 2 players :D

@hplus0603 i think i decided i wanna try to give it a go using Enet - reading up on Go i can see that this might not be the best option as you said due to GC and in relation to UDP a syscall overhead in the /net implementation - could have been nice though with Go and their channel approach but guess i can achieve the same in C.

joopyM said:
But dont know if it can handle 1000 or 100 on a server if the theory was to stream soundpackets to all clients - it would be unrealistic to stream to 1000 at the same time but in theory i guess its possible with some kind of traffic if every area/room has someone speaking then there could be generated alot of packets that are to be rerouted - but am still totally blank here what to expect or if i end up with only being able to support 2 players :D

Usually you don't want to actually send all audio streams to all players. Nobody can really make out 1,000 individual voices like that.

Instead you want to perform interest management on the server, so that you can send each player the audio only from the players close to them in the game world. And in that model, you end up either mixing a single unique audio stream per player on the server, or pay the bandwidth penalty of sending the closest N audio streams to each player.

You can also do server-side mixing, guaranteeing that each player only hears one stream of audio. Depending on hohow much work you want to do.

hplus0603 said:

You can also do server-side mixing, guaranteeing that each player only hears one stream of audio. Depending on hohow much work you want to do.

i was thinking to keep track of which player-id's were present in a room/hall and keep them in a room-cache and then pass the data to those clients through the server like a multicaster ( but using unicast UDP as its across the internet ).

what is serverside-mixing ?