1 module served.lsp.filereader; 2 3 import core.thread; 4 import core.sync.mutex; 5 6 import std.algorithm; 7 import std.stdio; 8 9 /// A simple file reader continuously reading into a 1 byte buffer and appending 10 /// it to the data. Ensures data is never buffered on any platform at the cost 11 /// of being a lot more CPU intensive. 12 class StdFileReader : FileReader 13 { 14 this(File file) 15 { 16 super(); 17 this.file = file; 18 } 19 20 override void stop() 21 { 22 file.close(); 23 } 24 25 override void run() 26 { 27 ubyte[1] buffer; 28 while (!file.eof) 29 { 30 auto chunk = file.rawRead(buffer[]); 31 synchronized (mutex) 32 data ~= chunk; 33 } 34 } 35 36 override bool isReading() 37 { 38 return isRunning && !file.eof; 39 } 40 41 File file; 42 } 43 44 /// A file reader implementation using the Win32 API using events. Reads as much 45 /// content as possible when new data is available at once, making the file 46 /// reading operation much more efficient when large chunks of data are being 47 /// transmitted. 48 version (Windows) class WindowsStdinReader : FileReader 49 { 50 import core.sys.windows.windows; 51 52 this() 53 { 54 super(); 55 } 56 57 override void stop() 58 { 59 wantStop = true; 60 } 61 62 override void run() 63 { 64 auto stdin = GetStdHandle(STD_INPUT_HANDLE); 65 INPUT_RECORD inputRecord; 66 DWORD nbRead; 67 ubyte[4096] buffer; 68 69 while (!wantStop) 70 { 71 switch (WaitForSingleObject(stdin, 1000)) 72 { 73 case WAIT_TIMEOUT: 74 break; 75 case WAIT_OBJECT_0: 76 DWORD len; 77 if (!ReadFile(stdin, &buffer, buffer.length, &len, null)) 78 { 79 stderr.writeln("ReadFile failed ", GetLastError()); 80 break; 81 } 82 if (len == 0) 83 { 84 stderr.writeln("WindowsStdinReader EOF"); 85 return; 86 } 87 synchronized (mutex) 88 data ~= buffer[0 .. len]; 89 break; 90 case WAIT_FAILED: 91 stderr.writeln("stdin read failed ", GetLastError()); 92 break; 93 case WAIT_ABANDONED: 94 stderr.writeln("stdin read wait was abandoned ", GetLastError()); 95 break; 96 default: 97 stderr.writeln("Unexpected WaitForSingleObject response"); 98 break; 99 } 100 } 101 } 102 103 override bool isReading() 104 { 105 return isRunning; 106 } 107 108 private bool wantStop; 109 } 110 111 /// A file reader implementation using the POSIX select API using events. Reads 112 /// as much content as possible when new data is available at once, making the 113 /// file reading operation much more efficient when large chunks of data are 114 /// being transmitted. 115 version (Posix) class PosixStdinReader : FileReader 116 { 117 import core.stdc.errno; 118 import core.sys.posix.sys.select; 119 import core.sys.posix.sys.time; 120 import core.sys.posix.sys.types; 121 import core.sys.posix.unistd; 122 123 override void stop() 124 { 125 stdin.close(); 126 } 127 128 override void run() 129 { 130 131 ubyte[4096] buffer; 132 scope (exit) 133 stdin.close(); 134 135 while (!stdin.error && !stdin.eof) 136 { 137 fd_set rfds; 138 timeval tv; 139 140 FD_ZERO(&rfds); 141 FD_SET(0, &rfds); 142 143 tv.tv_sec = 1; 144 145 auto ret = select(1, &rfds, null, null, &tv); 146 147 if (ret == -1) 148 { 149 int err = errno; 150 if (err == EINTR) 151 continue; 152 stderr.writeln("[fatal] PosixStdinReader error ", err, " in select()"); 153 break; 154 } 155 else if (ret) 156 { 157 auto len = read(0, buffer.ptr, buffer.length); 158 if (len == -1) 159 { 160 int err = errno; 161 if (err == EINTR) 162 continue; 163 stderr.writeln("PosixStdinReader error ", errno, " in read()"); 164 break; 165 } 166 else if (len == 0) 167 { 168 break; // eof 169 } 170 else 171 { 172 synchronized (mutex) 173 data ~= buffer[0 .. len]; 174 } 175 } 176 } 177 } 178 179 override bool isReading() 180 { 181 return isRunning && !stdin.eof && !stdin.error; 182 } 183 } 184 185 /// Base class for file readers which can read a file or standard handle line 186 /// by line in a Fiber context, yielding until a line is available. 187 abstract class FileReader : Thread 188 { 189 this() 190 { 191 super(&run); 192 mutex = new Mutex(); 193 } 194 195 string yieldLine() 196 { 197 ptrdiff_t index; 198 string ret; 199 while (true) 200 { 201 bool hasData; 202 synchronized (mutex) 203 { 204 index = data.countUntil([cast(ubyte) '\r', cast(ubyte) '\n']); 205 if (index != -1) 206 { 207 ret = cast(string) data[0 .. index].dup; 208 data = data[index + 2 .. $]; 209 break; 210 } 211 212 hasData = data.length != 0; 213 } 214 215 if (!hasData && !isReading) 216 return ret.length ? ret : null; 217 218 Fiber.yield(); 219 } 220 return ret; 221 } 222 223 /// Yields until the specified length of data is available, then removes the 224 /// data from the incoming data stream atomically and returns a duplicate of 225 /// it. 226 /// Returns null if the file reader stops while reading. 227 ubyte[] yieldData(size_t length) 228 { 229 while (true) 230 { 231 bool hasData; 232 synchronized (mutex) 233 { 234 if (data.length >= length) 235 { 236 auto ret = data[0 .. length].dup; 237 data = data[length .. $]; 238 return ret; 239 } 240 241 hasData = data.length != 0; 242 } 243 244 if (!hasData && !isReading) 245 return null; 246 247 Fiber.yield(); 248 } 249 } 250 251 abstract void stop(); 252 abstract bool isReading(); 253 254 protected: 255 abstract void run(); 256 257 ubyte[] data; 258 Mutex mutex; 259 } 260 261 /// Creates a new FileReader using the GC reading from stdin using a platform 262 /// optimized implementation or StdFileReader if none is available. 263 /// 264 /// The created instance can then be started using the `start` method and 265 /// stopped at exit using the `stop` method. 266 /// 267 /// Examples: 268 /// --- 269 /// auto input = newStdinReader(); 270 /// input.start(); 271 /// scope (exit) 272 /// input.stop(); 273 /// --- 274 FileReader newStdinReader() 275 { 276 version (Windows) 277 return new WindowsStdinReader(); 278 else version (Posix) 279 return new PosixStdinReader(); 280 else 281 return new StdFileReader(stdin); 282 } 283 284 /// Reads a file into a given buffer with a specified maximum length. If the 285 /// file is bigger than the buffer, the buffer will be resized using the GC and 286 /// updated through the ref argument. 287 /// Params: 288 /// file = The filename of the file to read. 289 /// buffer = A GC allocated buffer that may be enlarged if it is too small. 290 /// maxLen = The maxmimum amount of bytes to read from the file. 291 /// Returns: The contents of the file up to maxLen or EOF. The data is a slice 292 /// of the buffer argument case to a `char[]`. 293 char[] readCodeWithBuffer(string file, ref ubyte[] buffer, size_t maxLen = 1024 * 50) 294 { 295 auto f = File(file, "rb"); 296 size_t len = f.rawRead(buffer).length; 297 if (f.eof) 298 return cast(char[]) buffer[0 .. len]; 299 while (buffer.length * 2 < maxLen) 300 { 301 buffer.length *= 2; 302 len += f.rawRead(buffer[len .. $]).length; 303 if (f.eof) 304 return cast(char[]) buffer[0 .. len]; 305 } 306 if (buffer.length >= maxLen) 307 return cast(char[]) buffer; 308 buffer.length = maxLen; 309 f.rawRead(buffer[len .. $]); 310 return cast(char[]) buffer; 311 }