1 module xserial.serial; 2 3 import std.system : Endian, endian; 4 import std.traits : isArray, isDynamicArray, isStaticArray, isAssociativeArray, ForeachType, KeyType, ValueType, isIntegral, isFloatingPoint, isSomeChar, isType, isCallable, isPointer, hasUDA, getUDAs; 5 import std.typecons : isTuple; 6 import std.algorithm.searching : canFind; 7 8 import xbuffer.buffer : canSwapEndianness, Buffer, BufferOverflowException; 9 import xbuffer.memory : xalloc, xfree; 10 import xbuffer.varint : isVar; 11 12 import xserial.attribute; 13 14 /** 15 * Serializes some data. 16 */ 17 ubyte[] serialize(Endian endianness, L, Endian lengthEndianness, T)(T value, Buffer buffer) { 18 serializeImpl!(endianness, L, lengthEndianness, T)(buffer, value); 19 return buffer.data!ubyte; 20 } 21 22 /// ditto 23 ubyte[] serialize(Endian endianness, L, Endian lengthEndianness, T)(T value) { 24 Buffer buffer = xalloc!Buffer(64); 25 scope(exit) xfree(buffer); 26 return serialize!(endianness, L, lengthEndianness, T)(value, buffer).dup; 27 } 28 29 /// ditto 30 ubyte[] serialize(Endian endianness, L, T)(T value, Buffer buffer) { 31 return serialize!(endianness, L, endianness, T)(value, buffer); 32 } 33 34 /// ditto 35 ubyte[] serialize(Endian endianness, L, T)(T value) { 36 return serialize!(endianness, L, endianness, T)(value); 37 } 38 39 /// ditto 40 ubyte[] serialize(Endian endianness, T)(T value, Buffer buffer) { 41 return serialize!(endianness, uint)(value, buffer); 42 } 43 44 /// ditto 45 ubyte[] serialize(Endian endianness, T)(T value) { 46 return serialize!(endianness, uint)(value); 47 } 48 49 /// ditto 50 ubyte[] serialize(T)(T value, Buffer buffer) { 51 return serialize!(endian, uint, T)(value, buffer); 52 } 53 54 /// ditto 55 ubyte[] serialize(T)(T value) { 56 return serialize!(endian, uint, T)(value); 57 } 58 59 /** 60 * Deserializes some data. 61 */ 62 T deserialize(T, Endian endianness, L, Endian lengthEndianness)(Buffer buffer) { 63 return deserializeImpl!(endianness, L, lengthEndianness, T)(buffer); 64 } 65 66 /// ditto 67 T deserialize(T, Endian endianness, L, Endian lengthEndianness)(in ubyte[] data) { 68 Buffer buffer = xalloc!Buffer(data); 69 scope(exit) xfree(buffer); 70 return deserialize!(T, endianness, L, lengthEndianness)(buffer); 71 } 72 73 /// ditto 74 T deserialize(T, Endian endianness, L)(Buffer buffer) { 75 return deserialize!(T, endianness, L, endianness)(buffer); 76 } 77 78 /// ditto 79 T deserialize(T, Endian endianness, L)(in ubyte[] data) { 80 return deserialize!(T, endianness, L, endianness)(data); 81 } 82 83 /// ditto 84 T deserialize(T, Endian endianness)(Buffer buffer) { 85 return deserialize!(T, endianness, uint)(buffer); 86 } 87 88 /// ditto 89 T deserialize(T, Endian endianness)(in ubyte[] data) { 90 return deserialize!(T, endianness, uint)(data); 91 } 92 93 /// ditto 94 T deserialize(T)(Buffer buffer) { 95 return deserialize!(T, endian, uint)(buffer); 96 } 97 98 /// ditto 99 T deserialize(T)(in ubyte[] data) { 100 return deserialize!(T, endian, uint)(data); 101 } 102 103 // ----------- 104 // common data 105 // ----------- 106 107 enum EndianType { 108 109 bigEndian = cast(int)Endian.bigEndian, 110 littleEndian = cast(int)Endian.littleEndian, 111 var, 112 113 } 114 115 template Members(T, alias Only) { 116 117 import std.typetuple : TypeTuple; 118 119 mixin({ 120 121 string ret = "alias Members = TypeTuple!("; 122 foreach(member ; __traits(allMembers, T)) { 123 static if(is(typeof(mixin("T." ~ member)))) { 124 mixin("alias M = typeof(T." ~ member ~ ");"); 125 static if( 126 isType!M && 127 !isCallable!M && 128 ( 129 hasUDA!(__traits(getMember, T, member), Include) || 130 ( 131 [__traits(derivedMembers, T)].canFind(member) && 132 !__traits(compiles, { mixin("auto test=T." ~ member ~ ";"); }) && // static members 133 !__traits(compiles, { mixin("auto test=T.init." ~ member ~ "();"); }) && // properties 134 !hasUDA!(__traits(getMember, T, member), Exclude) && 135 !hasUDA!(__traits(getMember, T, member), Only) 136 ) 137 ) 138 ){ 139 ret ~= `"` ~ member ~ `",`; 140 141 } 142 } 143 } 144 return ret ~ ");"; 145 146 }()); 147 148 } 149 150 // ------------- 151 // serialization 152 // ------------- 153 154 void serializeImpl(Endian endianness, L, Endian lengthEndianness, T)(Buffer buffer, T value) { 155 static if(isVar!L) serializeImpl!(cast(EndianType)endianness, L.Type, EndianType.var, L.Type, EndianType.var, T)(buffer, value); 156 else serializeImpl!(cast(EndianType)endianness, L, cast(EndianType)lengthEndianness, L, cast(EndianType)lengthEndianness, T)(buffer, value); 157 } 158 159 void serializeImpl(EndianType endianness, OL, EndianType ole, CL, EndianType cle, T)(Buffer buffer, T value) { 160 static if(isArray!T) { 161 static if(isDynamicArray!T) serializeLength!(cle, CL)(buffer, value.length); 162 serializeArray!(endianness, OL, ole)(buffer, value); 163 } else static if(isAssociativeArray!T) { 164 serializeLength!(cle, CL)(buffer, value.length); 165 serializeAssociativeArray!(endianness, OL, ole)(buffer, value); 166 } else static if(isTuple!T) { 167 serializeTuple!(endianness, OL, ole)(buffer, value); 168 } else static if(is(T == class) || is(T == struct) || is(T == interface)) { 169 static if(__traits(hasMember, T, "serialize") && __traits(compiles, value.serialize(buffer))) { 170 value.serialize(buffer); 171 } else { 172 serializeMembers!(endianness, OL, ole)(buffer, value); 173 } 174 } else static if(is(T : bool) || isIntegral!T || isFloatingPoint!T || isSomeChar!T) { 175 serializeNumber!endianness(buffer, value); 176 } else { 177 static assert(0, "Cannot serialize " ~ T.stringof); 178 } 179 } 180 181 void serializeNumber(EndianType endianness, T)(Buffer buffer, T value) { 182 static if(endianness == EndianType.var) { 183 static assert(isIntegral!T && T.sizeof > 1, T.stringof ~ " cannot be annotated with @Var"); 184 buffer.writeVar!T(value); 185 } else static if(endianness == EndianType.bigEndian) { 186 buffer.write!(Endian.bigEndian, T)(value); 187 } else static if(endianness == EndianType.littleEndian) { 188 buffer.write!(Endian.littleEndian, T)(value); 189 } 190 } 191 192 void serializeLength(EndianType endianness, L)(Buffer buffer, size_t length) { 193 static if(L.sizeof < size_t.sizeof) serializeNumber!(endianness, L)(buffer, cast(L)length); 194 else serializeNumber!(endianness, L)(buffer, length); 195 } 196 197 void serializeArray(EndianType endianness, OL, EndianType ole, T)(Buffer buffer, T array) if(isArray!T) { 198 static if(canSwapEndianness!(ForeachType!T) && !is(ForeachType!T == struct) && !is(ForeachType!T == class) && endianness != EndianType.var) { 199 buffer.write!(cast(Endian)endianness)(array); 200 } else { 201 foreach(value ; array) { 202 serializeImpl!(endianness, OL, ole, OL, ole)(buffer, value); 203 } 204 } 205 } 206 207 void serializeAssociativeArray(EndianType endianness, OL, EndianType ole, T)(Buffer buffer, T array) if(isAssociativeArray!T) { 208 foreach(key, value; array) { 209 serializeImpl!(endianness, OL, ole, OL, ole)(buffer, key); 210 serializeImpl!(endianness, OL, ole, OL, ole)(buffer, value); 211 } 212 } 213 214 void serializeTuple(EndianType endianness, OL, EndianType ole, T)(Buffer buffer, T tuple) if(isTuple!T) { 215 static foreach(i ; 0..tuple.fieldNames.length) { 216 serializeImpl!(endianness, OL, ole, OL, ole)(buffer, tuple[i]); 217 } 218 } 219 220 void serializeMembers(EndianType endianness, L, EndianType le, T)(Buffer __buffer, T __container) { 221 foreach(member ; Members!(T, DecodeOnly)) { 222 223 mixin("alias M = typeof(__container." ~ member ~ ");"); 224 225 static foreach(uda ; __traits(getAttributes, __traits(getMember, T, member))) { 226 static if(is(uda : Custom!C, C)) { 227 enum __custom = true; 228 uda.C.serialize(mixin("__container." ~ member), __buffer); 229 } 230 } 231 232 static if(!is(typeof(__custom))) mixin({ 233 234 static if(hasUDA!(__traits(getMember, T, member), LengthImpl)) { 235 import std.conv : to; 236 auto length = getUDAs!(__traits(getMember, T, member), LengthImpl)[0]; 237 immutable e = "L, le, " ~ length.type ~ ", " ~ (length.endianness == -1 ? "endianness" : "EndianType." ~ (cast(EndianType)length.endianness).to!string); 238 } else { 239 immutable e = "L, le, L, le"; 240 } 241 242 static if(hasUDA!(__traits(getMember, T, member), NoLength)) immutable ret = "xserial.serial.serializeArray!(endianness, L, le, M)(__buffer, __container." ~ member ~ ");"; 243 else static if(hasUDA!(__traits(getMember, T, member), Var)) immutable ret = "xserial.serial.serializeImpl!(EndianType.var, " ~ e ~ ", M)(__buffer, __container." ~ member ~ ");"; 244 else static if(hasUDA!(__traits(getMember, T, member), BigEndian)) immutable ret = "xserial.serial.serializeImpl!(EndianType.bigEndian, " ~ e ~ ", M)(__buffer, __container." ~ member ~ ");"; 245 else static if(hasUDA!(__traits(getMember, T, member), LittleEndian)) immutable ret = "xserial.serial.serializeImpl!(EndianType.littleEndian, " ~ e ~ ", M)(__buffer, __container." ~ member ~ ");"; 246 else immutable ret = "xserial.serial.serializeImpl!(endianness, " ~ e ~ ", M)(__buffer, __container." ~ member ~ ");"; 247 248 static if(!hasUDA!(__traits(getMember, T, member), Condition)) return ret; 249 else return "with(__container){if(" ~ getUDAs!(__traits(getMember, T, member), Condition)[0].condition ~ "){" ~ ret ~ "}}"; 250 251 }()); 252 253 } 254 } 255 256 // --------------- 257 // deserialization 258 // --------------- 259 260 T deserializeImpl(Endian endianness, L, Endian lengthEndianness, T)(Buffer buffer) { 261 static if(isVar!L) return deserializeImpl!(cast(EndianType)endianness, L.Type, EndianType.var, L.Type, EndianType.var, T)(buffer); 262 else return deserializeImpl!(cast(EndianType)endianness, L, cast(EndianType)lengthEndianness, L, cast(EndianType)lengthEndianness, T)(buffer); 263 } 264 265 T deserializeImpl(EndianType endianness, OL, EndianType ole, CL, EndianType cle, T)(Buffer buffer) { 266 static if(isStaticArray!T) { 267 return deserializeStaticArray!(endianness, OL, ole, T)(buffer); 268 } else static if(isDynamicArray!T) { 269 return deserializeDynamicArray!(endianness, OL, ole, T)(buffer, deserializeLength!(cle, CL)(buffer)); 270 } else static if(isAssociativeArray!T) { 271 return deserializeAssociativeArray!(endianness, OL, ole, T)(buffer, deserializeLength!(cle, CL)(buffer)); 272 } else static if(isTuple!T) { 273 return deserializeTuple!(endianness, OL, ole, T)(buffer); 274 } else static if(is(T == class) || is(T == struct)) { 275 T ret; 276 static if(is(T == class)) ret = new T(); 277 static if(__traits(hasMember, T, "deserialize") && __traits(compiles, ret.deserialize(buffer))) { 278 ret.deserialize(buffer); 279 } else { 280 deserializeMembers!(endianness, OL, ole)(buffer, &ret); 281 } 282 return ret; 283 } else static if(is(T : bool) || isIntegral!T || isFloatingPoint!T || isSomeChar!T) { 284 return deserializeNumber!(endianness, T)(buffer); 285 } else { 286 static assert(0, "Cannot deserialize " ~ T.stringof); 287 } 288 } 289 290 T deserializeNumber(EndianType endianness, T)(Buffer buffer) { 291 static if(endianness == EndianType.var) { 292 static assert(isIntegral!T && T.sizeof > 1, T.stringof ~ " cannot be annotated with @Var"); 293 return buffer.readVar!T(); 294 } else static if(endianness == EndianType.bigEndian) { 295 return buffer.read!(Endian.bigEndian, T)(); 296 } else static if(endianness == EndianType.littleEndian) { 297 return buffer.read!(Endian.littleEndian, T)(); 298 } 299 } 300 301 size_t deserializeLength(EndianType endianness, L)(Buffer buffer) { 302 static if(L.sizeof > size_t.sizeof) return cast(size_t)deserializeNumber!(endianness, L)(buffer); 303 else return deserializeNumber!(endianness, L)(buffer); 304 } 305 306 T deserializeStaticArray(EndianType endianness, OL, EndianType ole, T)(Buffer buffer) if(isStaticArray!T) { 307 T ret; 308 foreach(ref value ; ret) { 309 value = deserializeImpl!(endianness, OL, ole, OL, ole, ForeachType!T)(buffer); 310 } 311 return ret; 312 } 313 314 T deserializeDynamicArray(EndianType endianness, OL, EndianType ole, T)(Buffer buffer, size_t length) if(isDynamicArray!T) { 315 T ret; 316 foreach(i ; 0..length) { 317 ret ~= deserializeImpl!(endianness, OL, ole, OL, ole, ForeachType!T)(buffer); 318 } 319 return ret; 320 } 321 322 T deserializeAssociativeArray(EndianType endianness, OL, EndianType ole, T)(Buffer buffer, size_t length) if(isAssociativeArray!T) { 323 T ret; 324 foreach(i ; 0..length) { 325 ret[deserializeImpl!(endianness, OL, ole, OL, ole, KeyType!T)(buffer)] = deserializeImpl!(endianness, OL, ole, OL, ole, ValueType!T)(buffer); 326 } 327 return ret; 328 } 329 330 T deserializeNoLengthArray(EndianType endianness, OL, EndianType ole, T)(Buffer buffer) if(isDynamicArray!T) { 331 T ret; 332 try { 333 while(true) ret ~= deserializeImpl!(endianness, OL, ole, OL, ole, ForeachType!T)(buffer); 334 } catch(BufferOverflowException) {} 335 return ret; 336 } 337 338 T deserializeTuple(EndianType endianness, OL, EndianType ole, T)(Buffer buffer) if(isTuple!T) { 339 T ret; 340 foreach(i, U; T.Types) { 341 ret[i] = deserializeImpl!(endianness, OL, ole, OL, ole, U)(buffer); 342 } 343 return ret; 344 } 345 346 void deserializeMembers(EndianType endianness, L, EndianType le, C)(Buffer __buffer, C __container) { 347 static if(isPointer!C) alias T = typeof(*__container); 348 else alias T = C; 349 foreach(member ; Members!(T, EncodeOnly)) { 350 351 mixin("alias M = typeof(__container." ~ member ~ ");"); 352 353 static foreach(uda ; __traits(getAttributes, __traits(getMember, T, member))) { 354 static if(is(uda : Custom!C, C)) { 355 enum __custom = true; 356 mixin("__container." ~ member) = uda.C.deserialize(__buffer); 357 } 358 } 359 360 static if(!is(typeof(__custom))) mixin({ 361 362 static if(hasUDA!(__traits(getMember, T, member), LengthImpl)) { 363 import std.conv : to; 364 auto length = getUDAs!(__traits(getMember, T, member), LengthImpl)[0]; 365 immutable e = "L, le, " ~ length.type ~ ", " ~ (length.endianness == -1 ? "endianness" : "EndianType." ~ (cast(EndianType)length.endianness).to!string); 366 } else { 367 immutable e = "L, le, L, le"; 368 } 369 370 static if(hasUDA!(__traits(getMember, T, member), NoLength)) immutable ret = "__container." ~ member ~ "=xserial.serial.deserializeNoLengthArray!(endianness, L, le, M)(__buffer);"; 371 else static if(hasUDA!(__traits(getMember, T, member), Var)) immutable ret = "__container." ~ member ~ "=xserial.serial.deserializeImpl!(EndianType.var, " ~ e ~ ", M)(__buffer);"; 372 else static if(hasUDA!(__traits(getMember, T, member), BigEndian)) immutable ret = "__container." ~ member ~ "=xserial.serial.deserializeImpl!(EndianType.bigEndian, " ~ e ~ ", M)(__buffer);"; 373 else static if(hasUDA!(__traits(getMember, T, member), LittleEndian)) immutable ret = "__container." ~ member ~ "=xserial.serial.deserializeImpl!(EndianType.littleEndian, " ~ e ~ ", M)(__buffer);"; 374 else immutable ret = "__container." ~ member ~ "=xserial.serial.deserializeImpl!(endianness, " ~ e ~ ", M)(__buffer);"; 375 376 static if(!hasUDA!(__traits(getMember, T, member), Condition)) return ret; 377 else return "with(__container){if(" ~ getUDAs!(__traits(getMember, T, member), Condition)[0].condition ~ "){" ~ ret ~ "}}"; 378 379 }()); 380 381 } 382 } 383 384 // --------- 385 // unittests 386 // --------- 387 388 @("numbers") unittest { 389 390 // bools and numbers 391 392 assert(true.serialize() == [1]); 393 assert(5.serialize!(Endian.bigEndian)() == [0, 0, 0, 5]); 394 395 assert(deserialize!(int, Endian.bigEndian)([0, 0, 0, 5]) == 5); 396 397 version(LittleEndian) assert(12.serialize() == [12, 0, 0, 0]); 398 version(BigEndian) assert(12.serialize() == [0, 0, 0, 12]); 399 400 } 401 402 @("arrays") unittest { 403 404 assert([1, 2, 3].serialize!(Endian.bigEndian, uint)() == [0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3]); 405 assert([1, 2, 3].serialize().deserialize!(int[])() == [1, 2, 3]); 406 407 ushort[2] test1 = [1, 2]; 408 assert(test1.serialize!(Endian.bigEndian)() == [0, 1, 0, 2]); 409 test1 = deserialize!(ushort[2], Endian.littleEndian)([2, 0, 1, 0]); 410 assert(test1 == [2, 1]); 411 412 } 413 414 @("associative arrays") unittest { 415 416 // associative arrays 417 418 int[ushort] test; 419 test[1] = 112; 420 assert(test.serialize!(Endian.bigEndian, uint)() == [0, 0, 0, 1, 0, 1, 0, 0, 0, 112]); 421 422 test = deserialize!(int[ushort], Endian.bigEndian, ubyte)([1, 0, 0, 0, 0, 0, 55]); 423 assert(test == [ushort(0): 55]); 424 425 } 426 427 @("tuples") unittest { 428 429 import std.typecons : Tuple, tuple; 430 431 assert(tuple(1, "test").serialize!(Endian.bigEndian, ushort)() == [0, 0, 0, 1, 0, 4, 't', 'e', 's', 't']); 432 433 Tuple!(ubyte, "a", uint[], "b") test; 434 test.a = 12; 435 assert(test.serialize!(Endian.littleEndian, uint)() == [12, 0, 0, 0, 0]); 436 assert(deserialize!(typeof(test), Endian.bigEndian, ushort)([12, 0, 0]) == test); 437 438 } 439 440 @("structs and classes") unittest { 441 442 struct Test1 { 443 444 byte a, b, c; 445 446 } 447 448 Test1 test1 = Test1(1, 3, 55); 449 assert(test1.serialize() == [1, 3, 55]); 450 451 assert(deserialize!Test1([1, 3, 55]) == test1); 452 453 static struct Test2 { 454 455 int a; 456 457 void serialize(Buffer buffer) { 458 buffer.write!(Endian.bigEndian)(this.a + 1); 459 } 460 461 void deserialize(Buffer buffer) { 462 this.a = buffer.read!(Endian.bigEndian, int)() - 1; 463 } 464 465 } 466 467 assert(serialize(Test2(5)) == [0, 0, 0, 6]); 468 assert(deserialize!Test2([0, 0, 0, 6]) == Test2(5)); 469 470 static class Test3 { 471 472 ubyte a; 473 474 void serialize() {} 475 476 void deserialize() {} 477 478 } 479 480 Test3 test3 = new Test3(); 481 assert(serialize(test3) == [0]); 482 assert(deserialize!Test3([5]).a == 5); 483 484 } 485 486 @("attributes") unittest { 487 488 struct Test1 { 489 490 @BigEndian int a; 491 492 @EncodeOnly @LittleEndian ushort b; 493 494 @Condition("a==1") @Var uint c; 495 496 @DecodeOnly @Var uint d; 497 498 @Exclude ubyte e; 499 500 } 501 502 Test1 test1 = Test1(1, 2, 3, 4, 5); 503 assert(test1.serialize() == [0, 0, 0, 1, 2, 0, 3]); 504 assert(deserialize!Test1([0, 0, 0, 1, 4, 12]) == Test1(1, 0, 4, 12)); 505 506 test1.a = 0; 507 assert(test1.serialize() == [0, 0, 0, 0, 2, 0]); 508 assert(deserialize!Test1([0, 0, 0, 0, 0, 0, 0, 0]) == Test1(0, 0, 0, 0)); 509 510 struct Test2 { 511 512 ubyte[] a; 513 514 @Length!ushort ushort[] b; 515 516 @NoLength uint[] c; 517 518 } 519 520 Test2 test2 = Test2([1, 2], [3, 4], [5, 6]); 521 assert(test2.serialize!(Endian.bigEndian, uint)() == [0, 0, 0, 2, 1, 2, 0, 2, 0, 3, 0, 4, 0, 0, 0, 5, 0, 0, 0, 6]); 522 assert(deserialize!(Test2, Endian.bigEndian, uint)([0, 0, 0, 2, 1, 2, 0, 2, 0, 3, 0, 4, 0, 0, 0, 5, 0, 0, 0, 6, 1]) == test2); 523 524 struct Test3 { 525 526 @EndianLength!ushort(Endian.littleEndian) @LittleEndian ushort[] a; 527 528 @NoLength ushort[] b; 529 530 } 531 532 Test3 test3 = Test3([1, 2], [3, 4]); 533 assert(test3.serialize!(Endian.bigEndian)() == [2, 0, 1, 0, 2, 0, 0, 3, 0, 4]); 534 535 struct Test4 { 536 537 ubyte a; 538 539 @LittleEndian uint b; 540 541 } 542 543 struct Test5 { 544 545 @Length!ubyte Test4[] a; 546 547 @NoLength Test4[] b; 548 549 } 550 551 Test5 test5 = Test5([Test4(1, 2)], [Test4(1, 2), Test4(3, 4)]); 552 assert(test5.serialize() == [1, 1, 2, 0, 0, 0, 1, 2, 0, 0, 0, 3, 4, 0, 0, 0]); 553 assert(deserialize!Test5([1, 1, 2, 0, 0, 0, 1, 2, 0, 0, 0, 3, 4, 0, 0, 0]) == test5); 554 555 } 556 557 @("using buffer") unittest { 558 559 Buffer buffer = new Buffer(64); 560 561 serialize(ubyte(55), buffer); 562 assert(buffer.data.length == 1); 563 assert(buffer.data!ubyte == [55]); 564 565 assert(deserialize!ubyte(buffer) == 55); 566 assert(buffer.data.length == 0); 567 568 }