I am writing a server to stream my desktop screen from my computer to a C# client in the Unity3D engine. I have written a simple python program to capture and send this data over TCP, however it is quite slow in sending, and I understand that UDP is more preferable for video streaming. I am really aiming for lowest possible latency.
Just an FYI, the frame data I have is output in the form of a numpy array. I have a 100 frame buffer, containing the last 100 frames captured from my screen.
I am not really sure how to go about making this UDP, especially since each frame is quite large (around 1555200 bytes or more). I probably need to do some kind of compression, but I am not familiar at all with any methods to compress an image from a byte array in real time. Additionally I am not sure how to go about ordering this info into individual frames on the client side since I likely won't be receiving the data all neat and tidy.
If there is any other way I can get lower latency and higher frame rate or optimize live video in any other way, I would like to know as well. I know all this is very possible in real-time because I have seen it done so many times. Also, I should mention that this will only be used over a local network.
I am not asking for you to write me a server (I mean unless you really want to), but rather point me in the right direction with what I really should be doing here. I will provide my code for the TCP server and client here, and please let me know if I should give any more info.
TCP python server
import socket, numpy, d3dshot, cv2 import mouse #create socket sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_address = ('127.0.0.1', 845) sock.bind(server_address) print ('started server on %s port %s' % server_address) sock.listen(1) #screen capture objects d = d3dshot.create(capture_output="numpy") d.display = d.displays[0] #capture on a seperate thread continuously d.capture() connection = None client_address = None def main(): global connection, client_address #await connection while connection == None: print ('Wait...') connection, client_address = sock.accept() print ('Connection from', client_address) try: #continuously send new packets as quickly as the client will take them while True: #get latest screen image - as numpy array img = d.get_latest_frame() res = cv2.resize(img, dsize=(960, 540)) #get mouse data including the length to send to the client mousebytes = bytes(str(mouse.get_position()), "ASCII") nummouse = len(bytes(str(len(mousebytes)), "ASCII")) #always send 5 bytes, then filter out the ! on the client mouselen = bytes(str(len(mousebytes)), "ASCII") + ((5-nummouse) * b"!") #get image data including length to send to client imagebytes = res.tobytes() numimage = len(bytes(str(len(imagebytes)), "ASCII")) #always send 12 bytes, then filter out the ! on the client imagelen = bytes(str(len(imagebytes)), "ASCII") + ((12-numimage) * b"!") #send the packet - usually sends around 1555200 bytes depending on the resolution connection.sendall(mouselen + mousebytes + imagelen + imagebytes) print("sent") except: main() finally: connection.close() main() TCP C# client
- This script is a version of my client written as a C# console application so should work without unity
using System; using System.Net; using System.Net.Sockets; using System.Threading; using System.Text; using System.Collections.Generic; using System.Linq; // State object for receiving data from remote device. public class StateObject { // Client socket. public Socket workSocket = null; // Size of receive buffer. public const int BufferSize = 1024; // Receive buffer. public byte[] buffer = new byte[BufferSize]; public List<byte> dynamicBuffer = new List<byte>(); // Received data string. public StringBuilder sb = new StringBuilder(); public int remainingBytes = 0; public bool initiate = true; } public class AsynchronousClient { // The port number for the remote device. private const int port = 845; // ManualResetEvent instances signal completion. private static ManualResetEvent connectDone = new ManualResetEvent(false); private static ManualResetEvent sendDone = new ManualResetEvent(false); private static ManualResetEvent receiveDone = new ManualResetEvent(false); // The response from the remote device. private static String response = String.Empty; public static Socket client; public static List<byte> save = new List<byte>(); public static string mousePosition; private static void StartClient() { // Connect to a remote device. try { // Establish the remote endpoint for the socket. // The name of the IPHostEntry ipHostInfo = Dns.GetHostEntry(Dns.GetHostName()); IPAddress ipAddress = IPAddress.Parse("127.0.0.1"); IPEndPoint remoteEP = new IPEndPoint(ipAddress, port); // Create a TCP/IP socket. client = new Socket(ipAddress.AddressFamily, SocketType.Stream, ProtocolType.Tcp); // Connect to the remote endpoint. client.BeginConnect(remoteEP, new AsyncCallback(ConnectCallback), client); connectDone.WaitOne(); } catch (Exception e) { Console.WriteLine(e.ToString()); } } private static void StopClient() { // Release the socket. client.Shutdown(SocketShutdown.Both); client.Close(); } private static void ConnectCallback(IAsyncResult ar) { try { // Retrieve the socket from the state object. Socket client = (Socket)ar.AsyncState; // Complete the connection. client.EndConnect(ar); Console.WriteLine("Socket connected to {0}", client.RemoteEndPoint.ToString()); // Signal that the connection has been made. connectDone.Set(); } catch (Exception e) { Console.WriteLine(e.ToString()); } } private static void Receive(Socket client) { try { // Create the state object. StateObject state = new StateObject(); state.workSocket = client; // Begin receiving the data from the remote device. client.BeginReceive(state.buffer, 0, StateObject.BufferSize, 0, new AsyncCallback(ReceiveCallback), state); } catch (Exception e) { Console.WriteLine(e.ToString()); } } private static void ReceiveCallback(IAsyncResult ar) { try { StateObject state = (StateObject)ar.AsyncState; Socket client = state.workSocket; // Read data from the remote device. int bytesRead = client.EndReceive(ar); //Console.WriteLine(bytesRead); int readPos = 0; byte[] buffer = state.buffer; if (state.initiate) { state.dynamicBuffer = state.buffer.ToList(); if(save.Count > 0) { state.dynamicBuffer.InsertRange(0, save); bytesRead += save.Count; save = new List<byte>(); } buffer = state.dynamicBuffer.ToArray(); state.buffer = buffer; int len = int.Parse(Encoding.ASCII.GetString(buffer, readPos, 5).Trim('!')); readPos += 5; mousePosition = Encoding.ASCII.GetString(buffer, readPos, len); readPos += len; state.remainingBytes = int.Parse(Encoding.ASCII.GetString(buffer, readPos, 12).Trim('!')); readPos += 12; } int bytesToRead = Math.Clamp(buffer.Length - readPos, 0, state.remainingBytes); string image = Encoding.ASCII.GetString(buffer, readPos, bytesToRead); state.remainingBytes -= bytesToRead; readPos += bytesToRead; state.sb.Append(image); if (state.remainingBytes == 0) { if(readPos < bytesRead) { save = buffer.ToList().GetRange(readPos, buffer.Length - readPos); } response = state.sb.ToString(); receiveDone.Set(); } else { state.initiate = false; client.BeginReceive(buffer, 0, buffer.Length, 0, new AsyncCallback(ReceiveCallback), state); } } catch (Exception e) { Console.WriteLine(e.ToString()); } } private static void Send(Socket client, String data) { // Convert the string data to byte data using ASCII encoding. byte[] byteData = Encoding.ASCII.GetBytes(data); // Begin sending the data to the remote device. client.BeginSend(byteData, 0, byteData.Length, 0, new AsyncCallback(SendCallback), client); } private static void SendCallback(IAsyncResult ar) { try { // Retrieve the socket from the state object. Socket client = (Socket)ar.AsyncState; // Complete sending the data to the remote device. int bytesSent = client.EndSend(ar); Console.WriteLine("Sent {0} bytes to server.", bytesSent); // Signal that all bytes have been sent. sendDone.Set(); } catch (Exception e) { Console.WriteLine(e.ToString()); } } public static void Update() { // Receive the response from the remote device. Receive(client); receiveDone.WaitOne(); Console.WriteLine("received file of size: " + response.Length); response = ""; receiveDone.Reset(); Update(); } public static int Main() { StartClient(); ThreadStart st = new ThreadStart(Update); Thread t = new Thread(st); t.Start(); Console.ReadLine(); return 0; } } 
没有评论:
发表评论