You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

69 lines
2.3 KiB

  1. /* Branchless UTF-8 decoder
  2. *
  3. * This is free and unencumbered software released into the public domain.
  4. */
  5. #ifndef UTF8_H
  6. #define UTF8_H
  7. #include <stdint.h>
  8. /* Decode the next character, C, from BUF, reporting errors in E.
  9. *
  10. * Since this is a branchless decoder, four bytes will be read from the
  11. * buffer regardless of the actual length of the next character. This
  12. * means the buffer _must_ have at least three bytes of zero padding
  13. * following the end of the data stream.
  14. *
  15. * Errors are reported in E, which will be non-zero if the parsed
  16. * character was somehow invalid: invalid byte sequence, non-canonical
  17. * encoding, or a surrogate half.
  18. *
  19. * The function returns a pointer to the next character. When an error
  20. * occurs, this pointer will be a guess that depends on the particular
  21. * error, but it will always advance at least one byte.
  22. */
  23. static void *
  24. utf8_decode(void *buf, uint32_t *c, int *e)
  25. {
  26. static const char lengths[] = {
  27. 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  28. 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 3, 3, 4, 0
  29. };
  30. static const int masks[] = {0x00, 0x7f, 0x1f, 0x0f, 0x07};
  31. static const uint32_t mins[] = {4194304, 0, 128, 2048, 65536};
  32. static const int shiftc[] = {0, 18, 12, 6, 0};
  33. static const int shifte[] = {0, 6, 4, 2, 0};
  34. unsigned char *s = buf;
  35. int len = lengths[s[0] >> 3];
  36. /* Compute the pointer to the next character early so that the next
  37. * iteration can start working on the next character. Neither Clang
  38. * nor GCC figure out this reordering on their own.
  39. */
  40. unsigned char *next = s + len + !len;
  41. /* Assume a four-byte character and load four bytes. Unused bits are
  42. * shifted out.
  43. */
  44. *c = (uint32_t)(s[0] & masks[len]) << 18;
  45. *c |= (uint32_t)(s[1] & 0x3f) << 12;
  46. *c |= (uint32_t)(s[2] & 0x3f) << 6;
  47. *c |= (uint32_t)(s[3] & 0x3f) << 0;
  48. *c >>= shiftc[len];
  49. /* Accumulate the various error conditions. */
  50. *e = (*c < mins[len]) << 6; // non-canonical encoding
  51. *e |= ((*c >> 11) == 0x1b) << 7; // surrogate half?
  52. *e |= (*c > 0x10FFFF) << 8; // out of range?
  53. *e |= (s[1] & 0xc0) >> 2;
  54. *e |= (s[2] & 0xc0) >> 4;
  55. *e |= (s[3] ) >> 6;
  56. *e ^= 0x2a; // top two bits of each tail byte correct?
  57. *e >>= shifte[len];
  58. return next;
  59. }
  60. #endif