From e20daff4eb37368a116cfcf15d02b0f4f452a5ed Mon Sep 17 00:00:00 2001 From: XMRig Date: Sun, 21 Feb 2021 22:28:15 +0700 Subject: [PATCH 01/16] v6.9.1-dev --- src/version.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/version.h b/src/version.h index 15426415..2eb246d1 100644 --- a/src/version.h +++ b/src/version.h @@ -28,7 +28,7 @@ #define APP_ID "xmrig" #define APP_NAME "XMRig" #define APP_DESC "XMRig miner" -#define APP_VERSION "6.9.0" +#define APP_VERSION "6.9.1-dev" #define APP_DOMAIN "xmrig.com" #define APP_SITE "www.xmrig.com" #define APP_COPYRIGHT "Copyright (C) 2016-2021 xmrig.com" @@ -36,7 +36,7 @@ #define APP_VER_MAJOR 6 #define APP_VER_MINOR 9 -#define APP_VER_PATCH 0 +#define APP_VER_PATCH 1 #ifdef _MSC_VER # if (_MSC_VER >= 1920) From d478d737c434c6d74748214b037487cfb48c259f Mon Sep 17 00:00:00 2001 From: SChernykh Date: Sun, 21 Feb 2021 16:33:57 +0100 Subject: [PATCH 02/16] Fixed pause logic when both pause on battery and user activity are enabled --- src/core/Miner.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/Miner.cpp b/src/core/Miner.cpp index b9e4fbad..29b4d60a 100644 --- a/src/core/Miner.cpp +++ b/src/core/Miner.cpp @@ -354,6 +354,7 @@ public: bool battery_power = false; bool user_active = false; bool enabled = true; + int32_t auto_pause = 0; bool reset = true; Controller *controller; Job job; @@ -635,7 +636,8 @@ void xmrig::Miner::onTimer(const Timer *) LOG_INFO("%s %s", Tags::miner(), pause ? pauseMessage : activeMessage); state = pause; - setEnabled(!pause); + d_ptr->auto_pause += pause ? 1 : -1; + setEnabled(d_ptr->auto_pause == 0); } }; From 4a8e7510e1ee83a7b627ee02d5f990efc2b80857 Mon Sep 17 00:00:00 2001 From: XMRig Date: Sat, 27 Feb 2021 15:29:14 +0700 Subject: [PATCH 03/16] #2123 Ignore regex exception. --- src/hw/dmi/DmiMemory.cpp | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/hw/dmi/DmiMemory.cpp b/src/hw/dmi/DmiMemory.cpp index 735af286..883c89b5 100644 --- a/src/hw/dmi/DmiMemory.cpp +++ b/src/hw/dmi/DmiMemory.cpp @@ -230,14 +230,16 @@ void xmrig::DmiMemory::setId(const char *slot, const char *bank) m_slot = slot; m_bank = bank; - std::cmatch cm; - if (std::regex_match(slot, cm, std::regex("^Channel([A-Z])[-_]DIMM(\\d+)$", std::regex_constants::icase))) { - m_id = fmt::format(kIdFormat, cm.str(1), cm.str(2)).c_str(); - } - else if (std::regex_search(bank, cm, std::regex("CHANNEL ([A-Z])$"))) { - std::cmatch cm2; - if (std::regex_match(slot, cm2, std::regex("^DIMM (\\d+)$"))) { - m_id = fmt::format(kIdFormat, cm.str(1), cm2.str(1)).c_str(); + try { + std::cmatch cm; + if (std::regex_match(slot, cm, std::regex("^Channel([A-Z])[-_]DIMM(\\d+)$", std::regex_constants::icase))) { + m_id = fmt::format(kIdFormat, cm.str(1), cm.str(2)).c_str(); } - } + else if (std::regex_search(bank, cm, std::regex("CHANNEL ([A-Z])$"))) { + std::cmatch cm2; + if (std::regex_match(slot, cm2, std::regex("^DIMM (\\d+)$"))) { + m_id = fmt::format(kIdFormat, cm.str(1), cm2.str(1)).c_str(); + } + } + } catch (...) {} } From 6f8ffb76608350da0ce04d6095fcff0c1aeeff58 Mon Sep 17 00:00:00 2001 From: XMRig Date: Mon, 1 Mar 2021 18:54:20 +0700 Subject: [PATCH 04/16] Fixed possible out of order write to log file. --- src/base/io/log/FileLogWriter.cpp | 36 ++++++++++++++++++++----------- src/base/io/log/FileLogWriter.h | 15 ++++++++++--- 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/src/base/io/log/FileLogWriter.cpp b/src/base/io/log/FileLogWriter.cpp index 921d9337..352dec20 100644 --- a/src/base/io/log/FileLogWriter.cpp +++ b/src/base/io/log/FileLogWriter.cpp @@ -1,6 +1,6 @@ /* XMRig - * Copyright 2018-2020 SChernykh - * Copyright 2016-2020 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -37,9 +37,6 @@ static void fsWriteCallback(uv_fs_t *req) } -static const char *kNewLine = "\n"; - - } // namespace xmrig @@ -50,11 +47,24 @@ bool xmrig::FileLogWriter::open(const char *fileName) return false; } - uv_fs_t req; - m_file = uv_fs_open(uv_default_loop(), &req, Env::expand(fileName), O_CREAT | O_APPEND | O_WRONLY, 0644, nullptr); + uv_fs_t req{}; + const auto path = Env::expand(fileName); + m_file = uv_fs_open(uv_default_loop(), &req, path, O_CREAT | O_WRONLY, 0644, nullptr); + + if (req.result < 0 || !isOpen()) { + uv_fs_req_cleanup(&req); + m_file = -1; + + return false; + } + uv_fs_req_cleanup(&req); - return isOpen(); + uv_fs_stat(uv_default_loop(), &req, path, nullptr); + m_pos = req.statbuf.st_size; + uv_fs_req_cleanup(&req); + + return true; } @@ -70,7 +80,8 @@ bool xmrig::FileLogWriter::write(const char *data, size_t size) auto req = new uv_fs_t; req->data = buf.base; - uv_fs_write(uv_default_loop(), req, m_file, &buf, 1, -1, fsWriteCallback); + uv_fs_write(uv_default_loop(), req, m_file, &buf, 1, m_pos, fsWriteCallback); + m_pos += size; return true; } @@ -78,9 +89,9 @@ bool xmrig::FileLogWriter::write(const char *data, size_t size) bool xmrig::FileLogWriter::writeLine(const char *data, size_t size) { - uv_buf_t buf[2] = { + const uv_buf_t buf[2] = { uv_buf_init(new char[size], size), - uv_buf_init(const_cast(kNewLine), 1) + uv_buf_init(m_endl, sizeof(m_endl) - 1) }; memcpy(buf[0].base, data, size); @@ -88,7 +99,8 @@ bool xmrig::FileLogWriter::writeLine(const char *data, size_t size) auto req = new uv_fs_t; req->data = buf[0].base; - uv_fs_write(uv_default_loop(), req, m_file, buf, 2, -1, fsWriteCallback); + uv_fs_write(uv_default_loop(), req, m_file, buf, 2, m_pos, fsWriteCallback); + m_pos += (buf[0].len + buf[1].len); return true; } diff --git a/src/base/io/log/FileLogWriter.h b/src/base/io/log/FileLogWriter.h index 5dca4c58..548d235c 100644 --- a/src/base/io/log/FileLogWriter.h +++ b/src/base/io/log/FileLogWriter.h @@ -1,6 +1,6 @@ /* XMRig - * Copyright 2018-2020 SChernykh - * Copyright 2016-2020 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -21,6 +21,7 @@ #include +#include namespace xmrig { @@ -33,13 +34,21 @@ public: FileLogWriter(const char *fileName) { open(fileName); } inline bool isOpen() const { return m_file >= 0; } + inline int64_t pos() const { return m_pos; } bool open(const char *fileName); bool write(const char *data, size_t size); bool writeLine(const char *data, size_t size); private: - int m_file = -1; +# ifdef XMRIG_OS_WIN + char m_endl[3] = "\r\n"; +# else + char m_endl[2] = "\n"; +# endif + + int m_file = -1; + int64_t m_pos = 0; }; From 0b7dfaabe00175e056429576a1d16f64b86b7e0d Mon Sep 17 00:00:00 2001 From: XMRig Date: Mon, 1 Mar 2021 19:04:03 +0700 Subject: [PATCH 05/16] Code cleanup. --- src/base/net/http/HttpContext.cpp | 10 ++++------ src/base/net/http/HttpContext.h | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/base/net/http/HttpContext.cpp b/src/base/net/http/HttpContext.cpp index 52739e4d..d56db0c8 100644 --- a/src/base/net/http/HttpContext.cpp +++ b/src/base/net/http/HttpContext.cpp @@ -1,7 +1,7 @@ /* XMRig * Copyright (c) 2014-2019 heapwolf - * Copyright (c) 2018-2020 SChernykh - * Copyright (c) 2016-2020 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -175,11 +175,9 @@ void xmrig::HttpContext::close(int status) xmrig::HttpContext *xmrig::HttpContext::get(uint64_t id) { - if (storage.count(id) == 0) { - return nullptr; - } + const auto it = storage.find(id); - return storage[id]; + return it == storage.end() ? nullptr : it->second; } diff --git a/src/base/net/http/HttpContext.h b/src/base/net/http/HttpContext.h index 4202bfaf..9c719333 100644 --- a/src/base/net/http/HttpContext.h +++ b/src/base/net/http/HttpContext.h @@ -1,7 +1,7 @@ /* XMRig * Copyright (c) 2014-2019 heapwolf - * Copyright (c) 2018-2020 SChernykh - * Copyright (c) 2016-2020 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by From 91ad6fcf3d35c803bcc7b7f33d02f3bd52d8defe Mon Sep 17 00:00:00 2001 From: SChernykh Date: Mon, 1 Mar 2021 17:46:05 +0100 Subject: [PATCH 06/16] Fixed many "new job" messages when solo mining Fix for https://github.com/xmrig/xmrig/issues/2127 --- src/base/net/stratum/DaemonClient.cpp | 26 ++++++++++++++++++++++---- src/base/net/stratum/DaemonClient.h | 2 ++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/base/net/stratum/DaemonClient.cpp b/src/base/net/stratum/DaemonClient.cpp index 554ee8a8..07cc7360 100644 --- a/src/base/net/stratum/DaemonClient.cpp +++ b/src/base/net/stratum/DaemonClient.cpp @@ -179,12 +179,30 @@ void xmrig::DaemonClient::onHttpData(const HttpData &data) return send(kGetInfo); } - if (isOutdated(Json::getUint64(doc, kHeight), Json::getString(doc, kHash))) { - getBlockTemplate(); + const uint64_t height = Json::getUint64(doc, kHeight); + const String hash = Json::getString(doc, kHash); + + if (isOutdated(height, hash)) { + // Multiple /getheight responses can come at once resulting in multiple getBlockTemplate() calls + if ((height != m_blocktemplateRequestHeight) || (hash != m_blocktemplateRequestHash)) { + m_blocktemplateRequestHeight = height; + m_blocktemplateRequestHash = hash; + getBlockTemplate(); + } } } - else if (data.url == kGetInfo && isOutdated(Json::getUint64(doc, kHeight), Json::getString(doc, "top_block_hash"))) { - getBlockTemplate(); + else if (data.url == kGetInfo) { + const uint64_t height = Json::getUint64(doc, kHeight); + const String hash = Json::getString(doc, "top_block_hash"); + + if (isOutdated(height, hash)) { + // Multiple /getinfo responses can come at once resulting in multiple getBlockTemplate() calls + if ((height != m_blocktemplateRequestHeight) || (hash != m_blocktemplateRequestHash)) { + m_blocktemplateRequestHeight = height; + m_blocktemplateRequestHash = hash; + getBlockTemplate(); + } + } } return; diff --git a/src/base/net/stratum/DaemonClient.h b/src/base/net/stratum/DaemonClient.h index 94858b50..ff4f9e04 100644 --- a/src/base/net/stratum/DaemonClient.h +++ b/src/base/net/stratum/DaemonClient.h @@ -89,6 +89,8 @@ private: String m_tlsFingerprint; String m_tlsVersion; Timer *m_timer; + uint64_t m_blocktemplateRequestHeight = 0; + String m_blocktemplateRequestHash; }; From ba3299b61b12948bc1015bff71ab334da7c4e8a3 Mon Sep 17 00:00:00 2001 From: TheGreatMcPain Date: Tue, 2 Mar 2021 01:33:25 -0600 Subject: [PATCH 07/16] Update sse2neon.h to the latest master. Fixes build on armv7. A few days after this header was introduced. Upstream updated it with armv7 versions of `_mm_aesenc_si128` which allows xmrig to build on armv7. --- src/crypto/cn/sse2neon.h | 3688 ++++++++++++++++++++++++++++++++++---- 1 file changed, 3345 insertions(+), 343 deletions(-) diff --git a/src/crypto/cn/sse2neon.h b/src/crypto/cn/sse2neon.h index 15c26efe..039697a6 100644 --- a/src/crypto/cn/sse2neon.h +++ b/src/crypto/cn/sse2neon.h @@ -20,6 +20,11 @@ // Sebastian Pop // Developer Ecosystem Engineering // Danila Kutenin +// François Turban (JishinMaster) +// Pei-Hsuan Hung +// Yang-Hao Yuan +// Syoyo Fujita +// Brecht Van Lommel /* * sse2neon is freely redistributable under the MIT License. @@ -43,13 +48,38 @@ * SOFTWARE. */ +/* Tunable configurations */ + +/* Enable precise implementation of math operations + * This would slow down the computation a bit, but gives consistent result with + * x86 SSE2. (e.g. would solve a hole or NaN pixel in the rendering result) + */ +/* _mm_min_ps and _mm_max_ps */ +#ifndef SSE2NEON_PRECISE_MINMAX +#define SSE2NEON_PRECISE_MINMAX (0) +#endif +/* _mm_rcp_ps and _mm_div_ps */ +#ifndef SSE2NEON_PRECISE_DIV +#define SSE2NEON_PRECISE_DIV (0) +#endif +/* _mm_sqrt_ps and _mm_rsqrt_ps */ +#ifndef SSE2NEON_PRECISE_SQRT +#define SSE2NEON_PRECISE_SQRT (0) +#endif + #if defined(__GNUC__) || defined(__clang__) #pragma push_macro("FORCE_INLINE") #pragma push_macro("ALIGN_STRUCT") #define FORCE_INLINE static inline __attribute__((always_inline)) #define ALIGN_STRUCT(x) __attribute__((aligned(x))) +#ifndef likely +#define likely(x) __builtin_expect(!!(x), 1) +#endif +#ifndef unlikely +#define unlikely(x) __builtin_expect(!!(x), 0) +#endif #else -#error "Macro name collisions may happens with unknown compiler" +#error "Macro name collisions may happen with unsupported compiler." #ifdef FORCE_INLINE #undef FORCE_INLINE #endif @@ -58,12 +88,48 @@ #define ALIGN_STRUCT(x) __declspec(align(x)) #endif #endif +#ifndef likely +#define likely(x) (x) +#endif +#ifndef unlikely +#define unlikely(x) (x) +#endif #include #include +/* Architecture-specific build options */ +/* FIXME: #pragma GCC push_options is only available on GCC */ +#if defined(__GNUC__) +#if defined(__arm__) && __ARM_ARCH == 7 +/* According to ARM C Language Extensions Architecture specification, + * __ARM_NEON is defined to a value indicating the Advanced SIMD (NEON) + * architecture supported. + */ +#if !defined(__ARM_NEON) || !defined(__ARM_NEON__) +#error "You must enable NEON instructions (e.g. -mfpu=neon) to use SSE2NEON." +#endif +#if !defined(__clang__) +#pragma GCC push_options +#pragma GCC target("fpu=neon") +#endif +#elif defined(__aarch64__) +#if !defined(__clang__) +#pragma GCC push_options +#pragma GCC target("+simd") +#endif +#else +#error "Unsupported target. Must be either ARMv7-A+NEON or ARMv8-A." +#endif +#endif + #include +/* Rounding functions require either Aarch64 instructions or libm failback */ +#if !defined(__aarch64__) +#include +#endif + /* "__has_builtin" can be used to query support for built-in functions * provided by gcc/clang and other compilers that support it. */ @@ -89,6 +155,18 @@ #define _MM_SHUFFLE(fp3, fp2, fp1, fp0) \ (((fp3) << 6) | ((fp2) << 4) | ((fp1) << 2) | ((fp0))) +/* Rounding mode macros. */ +#define _MM_FROUND_TO_NEAREST_INT 0x00 +#define _MM_FROUND_TO_NEG_INF 0x01 +#define _MM_FROUND_TO_POS_INF 0x02 +#define _MM_FROUND_TO_ZERO 0x03 +#define _MM_FROUND_CUR_DIRECTION 0x04 +#define _MM_FROUND_NO_EXC 0x08 +#define _MM_ROUND_NEAREST 0x0000 +#define _MM_ROUND_DOWN 0x2000 +#define _MM_ROUND_UP 0x4000 +#define _MM_ROUND_TOWARD_ZERO 0x6000 + /* indicate immediate constant argument in a given range */ #define __constrange(a, b) const @@ -98,7 +176,7 @@ * a suffix, it contains floats. An integer vector type can contain any type * of integer, from chars to shorts to unsigned long longs. */ -typedef float32x2_t __m64; +typedef int64x1_t __m64; typedef float32x4_t __m128; /* 128-bit vector containing 4 floats */ // On ARM 32-bit architecture, the float64x2_t is not supported. // The data type __m128d should be represented in a different way for related @@ -108,7 +186,6 @@ typedef float64x2_t __m128d; /* 128-bit vector containing 2 doubles */ #else typedef float32x4_t __m128d; #endif -typedef int64x1_t __m64i; typedef int64x2_t __m128i; /* 128-bit vector containing integers */ /* type-safe casting between types */ @@ -151,6 +228,9 @@ typedef int64x2_t __m128i; /* 128-bit vector containing integers */ #define vreinterpretq_m128i_u32(x) vreinterpretq_s64_u32(x) #define vreinterpretq_m128i_u64(x) vreinterpretq_s64_u64(x) +#define vreinterpretq_f32_m128i(x) vreinterpretq_f32_s64(x) +#define vreinterpretq_f64_m128i(x) vreinterpretq_f64_s64(x) + #define vreinterpretq_s8_m128i(x) vreinterpretq_s8_s64(x) #define vreinterpretq_s16_m128i(x) vreinterpretq_s16_s64(x) #define vreinterpretq_s32_m128i(x) vreinterpretq_s32_s64(x) @@ -161,25 +241,63 @@ typedef int64x2_t __m128i; /* 128-bit vector containing integers */ #define vreinterpretq_u32_m128i(x) vreinterpretq_u32_s64(x) #define vreinterpretq_u64_m128i(x) vreinterpretq_u64_s64(x) -#define vreinterpret_m64i_s8(x) vreinterpret_s64_s8(x) -#define vreinterpret_m64i_s16(x) vreinterpret_s64_s16(x) -#define vreinterpret_m64i_s32(x) vreinterpret_s64_s32(x) -#define vreinterpret_m64i_s64(x) (x) +#define vreinterpret_m64_s8(x) vreinterpret_s64_s8(x) +#define vreinterpret_m64_s16(x) vreinterpret_s64_s16(x) +#define vreinterpret_m64_s32(x) vreinterpret_s64_s32(x) +#define vreinterpret_m64_s64(x) (x) -#define vreinterpret_m64i_u8(x) vreinterpret_s64_u8(x) -#define vreinterpret_m64i_u16(x) vreinterpret_s64_u16(x) -#define vreinterpret_m64i_u32(x) vreinterpret_s64_u32(x) -#define vreinterpret_m64i_u64(x) vreinterpret_s64_u64(x) +#define vreinterpret_m64_u8(x) vreinterpret_s64_u8(x) +#define vreinterpret_m64_u16(x) vreinterpret_s64_u16(x) +#define vreinterpret_m64_u32(x) vreinterpret_s64_u32(x) +#define vreinterpret_m64_u64(x) vreinterpret_s64_u64(x) -#define vreinterpret_u8_m64i(x) vreinterpret_u8_s64(x) -#define vreinterpret_u16_m64i(x) vreinterpret_u16_s64(x) -#define vreinterpret_u32_m64i(x) vreinterpret_u32_s64(x) -#define vreinterpret_u64_m64i(x) vreinterpret_u64_s64(x) +#define vreinterpret_m64_f16(x) vreinterpret_s64_f16(x) +#define vreinterpret_m64_f32(x) vreinterpret_s64_f32(x) +#define vreinterpret_m64_f64(x) vreinterpret_s64_f64(x) -#define vreinterpret_s8_m64i(x) vreinterpret_s8_s64(x) -#define vreinterpret_s16_m64i(x) vreinterpret_s16_s64(x) -#define vreinterpret_s32_m64i(x) vreinterpret_s32_s64(x) -#define vreinterpret_s64_m64i(x) (x) +#define vreinterpret_u8_m64(x) vreinterpret_u8_s64(x) +#define vreinterpret_u16_m64(x) vreinterpret_u16_s64(x) +#define vreinterpret_u32_m64(x) vreinterpret_u32_s64(x) +#define vreinterpret_u64_m64(x) vreinterpret_u64_s64(x) + +#define vreinterpret_s8_m64(x) vreinterpret_s8_s64(x) +#define vreinterpret_s16_m64(x) vreinterpret_s16_s64(x) +#define vreinterpret_s32_m64(x) vreinterpret_s32_s64(x) +#define vreinterpret_s64_m64(x) (x) + +#define vreinterpret_f32_m64(x) vreinterpret_f32_s64(x) + +#if defined(__aarch64__) +#define vreinterpretq_m128d_s32(x) vreinterpretq_f64_s32(x) +#define vreinterpretq_m128d_s64(x) vreinterpretq_f64_s64(x) + +#define vreinterpretq_m128d_u64(x) vreinterpretq_f64_u64(x) + +#define vreinterpretq_m128d_f32(x) vreinterpretq_f64_f32(x) +#define vreinterpretq_m128d_f64(x) (x) + +#define vreinterpretq_s64_m128d(x) vreinterpretq_s64_f64(x) + +#define vreinterpretq_u64_m128d(x) vreinterpretq_u64_f64(x) + +#define vreinterpretq_f64_m128d(x) (x) +#define vreinterpretq_f32_m128d(x) vreinterpretq_f32_f64(x) +#else +#define vreinterpretq_m128d_s32(x) vreinterpretq_f32_s32(x) +#define vreinterpretq_m128d_s64(x) vreinterpretq_f32_s64(x) + +#define vreinterpretq_m128d_u32(x) vreinterpretq_f32_u32(x) +#define vreinterpretq_m128d_u64(x) vreinterpretq_f32_u64(x) + +#define vreinterpretq_m128d_f32(x) (x) + +#define vreinterpretq_s64_m128d(x) vreinterpretq_s64_f32(x) + +#define vreinterpretq_u32_m128d(x) vreinterpretq_u32_f32(x) +#define vreinterpretq_u64_m128d(x) vreinterpretq_u64_f32(x) + +#define vreinterpretq_f32_m128d(x) (x) +#endif // A struct is defined in this header file called 'SIMDVec' which can be used // by applications which attempt to access the contents of an _m128 struct @@ -219,13 +337,16 @@ typedef union ALIGN_STRUCT(16) SIMDVec { // casting using SIMDVec #define vreinterpretq_nth_u64_m128i(x, n) (((SIMDVec *) &x)->m128_u64[n]) #define vreinterpretq_nth_u32_m128i(x, n) (((SIMDVec *) &x)->m128_u32[n]) +#define vreinterpretq_nth_u8_m128i(x, n) (((SIMDVec *) &x)->m128_u8[n]) /* Backwards compatibility for compilers with lack of specific type support */ // Older gcc does not define vld1q_u8_x4 type -#if defined(__GNUC__) && !defined(__clang__) -#if __GNUC__ <= 9 -FORCE_INLINE uint8x16x4_t vld1q_u8_x4(const uint8_t *p) +#if defined(__GNUC__) && !defined(__clang__) && \ + ((__GNUC__ == 10 && (__GNUC_MINOR__ <= 1)) || \ + (__GNUC__ == 9 && (__GNUC_MINOR__ <= 3)) || \ + (__GNUC__ == 8 && (__GNUC_MINOR__ <= 4)) || __GNUC__ <= 7) +FORCE_INLINE uint8x16x4_t _sse2neon_vld1q_u8_x4(const uint8_t *p) { uint8x16x4_t ret; ret.val[0] = vld1q_u8(p + 0); @@ -234,7 +355,12 @@ FORCE_INLINE uint8x16x4_t vld1q_u8_x4(const uint8_t *p) ret.val[3] = vld1q_u8(p + 48); return ret; } -#endif +#else +// Wraps vld1q_u8_x4 +FORCE_INLINE uint8x16x4_t _sse2neon_vld1q_u8_x4(const uint8_t *p) +{ + return vld1q_u8_x4(p); +} #endif /* Function Naming Conventions @@ -336,13 +462,137 @@ FORCE_INLINE void _mm_prefetch(const void *p, int i) __builtin_prefetch(p); } -// extracts the lower order floating point value from the parameter : -// https://msdn.microsoft.com/en-us/library/bb514059%28v=vs.120%29.aspx?f=255&MSPPError=-2147217396 +// Pause the processor. This is typically used in spin-wait loops and depending +// on the x86 processor typical values are in the 40-100 cycle range. The +// 'yield' instruction isn't a good fit beacuse it's effectively a nop on most +// Arm cores. Experience with several databases has shown has shown an 'isb' is +// a reasonable approximation. +FORCE_INLINE void _mm_pause() +{ + __asm__ __volatile__("isb\n"); +} + +// Copy the lower single-precision (32-bit) floating-point element of a to dst. +// +// dst[31:0] := a[31:0] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtss_f32 FORCE_INLINE float _mm_cvtss_f32(__m128 a) { return vgetq_lane_f32(vreinterpretq_f32_m128(a), 0); } +// Convert the lower single-precision (32-bit) floating-point element in b to a +// double-precision (64-bit) floating-point element, store the result in the +// lower element of dst, and copy the upper element from a to the upper element +// of dst. +// +// dst[63:0] := Convert_FP32_To_FP64(b[31:0]) +// dst[127:64] := a[127:64] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtss_sd +FORCE_INLINE __m128d _mm_cvtss_sd(__m128d a, __m128 b) +{ + double d = (double) vgetq_lane_f32(vreinterpretq_f32_m128(b), 0); +#if defined(__aarch64__) + return vreinterpretq_m128d_f64( + vsetq_lane_f64(d, vreinterpretq_f64_m128d(a), 0)); +#else + return vreinterpretq_m128d_s64( + vsetq_lane_s64(*(int64_t *) &d, vreinterpretq_s64_m128d(a), 0)); +#endif +} + +// Convert the lower single-precision (32-bit) floating-point element in a to a +// 32-bit integer, and store the result in dst. +// +// dst[31:0] := Convert_FP32_To_Int32(a[31:0]) +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtss_si32 +#define _mm_cvtss_si32(a) _mm_cvt_ss2si(a) + +// Convert the lower single-precision (32-bit) floating-point element in a to a +// 64-bit integer, and store the result in dst. +// +// dst[63:0] := Convert_FP32_To_Int64(a[31:0]) +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtss_si64 +FORCE_INLINE int _mm_cvtss_si64(__m128 a) +{ +#if defined(__aarch64__) + return vgetq_lane_s64( + vreinterpretq_s64_s32(vcvtnq_s32_f32(vreinterpretq_f32_m128(a))), 0); +#else + float32_t data = vgetq_lane_f32(vreinterpretq_f32_m128(a), 0); + float32_t diff = data - floor(data); + if (diff > 0.5) + return (int64_t) ceil(data); + if (unlikely(diff == 0.5)) { + int64_t f = (int64_t) floor(data); + int64_t c = (int64_t) ceil(data); + return c & 1 ? f : c; + } + return (int64_t) floor(data); +#endif +} + +// Convert packed single-precision (32-bit) floating-point elements in a to +// packed 32-bit integers with truncation, and store the results in dst. +// +// FOR j := 0 to 1 +// i := 32*j +// dst[i+31:i] := Convert_FP32_To_Int32_Truncate(a[i+31:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_ps2pi +FORCE_INLINE __m64 _mm_cvtt_ps2pi(__m128 a) +{ + return vreinterpret_m64_s32( + vget_low_s32(vcvtq_s32_f32(vreinterpretq_f32_m128(a)))); +} + +// Convert the lower single-precision (32-bit) floating-point element in a to a +// 32-bit integer with truncation, and store the result in dst. +// +// dst[31:0] := Convert_FP32_To_Int32_Truncate(a[31:0]) +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_ss2si +FORCE_INLINE int _mm_cvtt_ss2si(__m128 a) +{ + return vgetq_lane_s32(vcvtq_s32_f32(vreinterpretq_f32_m128(a)), 0); +} + +// Convert packed single-precision (32-bit) floating-point elements in a to +// packed 32-bit integers with truncation, and store the results in dst. +// +// FOR j := 0 to 1 +// i := 32*j +// dst[i+31:i] := Convert_FP32_To_Int32_Truncate(a[i+31:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttps_pi32 +#define _mm_cvttps_pi32(a) _mm_cvtt_ps2pi(a) + +// Convert the lower single-precision (32-bit) floating-point element in a to a +// 32-bit integer with truncation, and store the result in dst. +// +// dst[31:0] := Convert_FP32_To_Int32_Truncate(a[31:0]) +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttss_si32 +#define _mm_cvttss_si32(a) _mm_cvtt_ss2si(a) + +// Convert the lower single-precision (32-bit) floating-point element in a to a +// 64-bit integer with truncation, and store the result in dst. +// +// dst[63:0] := Convert_FP32_To_Int64_Truncate(a[31:0]) +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttss_si64 +FORCE_INLINE int64_t _mm_cvttss_si64(__m128 a) +{ + return vgetq_lane_s64( + vmovl_s32(vget_low_s32(vcvtq_s32_f32(vreinterpretq_f32_m128(a)))), 0); +} + // Sets the 128-bit value to zero // https://msdn.microsoft.com/en-us/library/vstudio/ys7dw0kh(v=vs.100).aspx FORCE_INLINE __m128i _mm_setzero_si128(void) @@ -357,6 +607,17 @@ FORCE_INLINE __m128 _mm_setzero_ps(void) return vreinterpretq_m128_f32(vdupq_n_f32(0)); } +// Return vector of type __m128d with all elements set to zero. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_setzero_pd +FORCE_INLINE __m128d _mm_setzero_pd(void) +{ +#if defined(__aarch64__) + return vreinterpretq_m128d_f64(vdupq_n_f64(0)); +#else + return vreinterpretq_m128d_f32(vdupq_n_f32(0)); +#endif +} + // Sets the four single-precision, floating-point values to w. // // r0 := r1 := r2 := r3 := w @@ -384,7 +645,7 @@ FORCE_INLINE __m128 _mm_set_ps(float w, float z, float y, float x) // Copy single-precision (32-bit) floating-point element a to the lower element // of dst, and zero the upper 3 elements. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_set_ss&expand=4901,4895,4901 +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_set_ss FORCE_INLINE __m128 _mm_set_ss(float a) { float ALIGN_STRUCT(16) data[4] = {a, 0, 0, 0}; @@ -428,6 +689,13 @@ FORCE_INLINE __m128i _mm_setr_epi32(int i3, int i2, int i1, int i0) return vreinterpretq_m128i_s32(vld1q_s32(data)); } +// Set packed 64-bit integers in dst with the supplied values in reverse order. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_setr_epi64 +FORCE_INLINE __m128i _mm_setr_epi64(__m64 e1, __m64 e0) +{ + return vreinterpretq_m128i_s64(vcombine_s64(e1, e0)); +} + // Sets the 16 signed 8-bit integer values to b. // // r0 := b @@ -441,6 +709,18 @@ FORCE_INLINE __m128i _mm_set1_epi8(signed char w) return vreinterpretq_m128i_s8(vdupq_n_s8(w)); } +// Broadcast double-precision (64-bit) floating-point value a to all elements of +// dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_set1_pd +FORCE_INLINE __m128d _mm_set1_pd(double d) +{ +#if defined(__aarch64__) + return vreinterpretq_m128d_f64(vdupq_n_f64(d)); +#else + return vreinterpretq_m128d_s64(vdupq_n_s64(*(int64_t *) &d)); +#endif +} + // Sets the 8 signed 16-bit integer values to w. // // r0 := w @@ -538,13 +818,13 @@ FORCE_INLINE __m128i _mm_set1_epi32(int _i) // Sets the 2 signed 64-bit integer values to i. // https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/whtfzhzk(v=vs.100) -FORCE_INLINE __m128i _mm_set1_epi64(int64_t _i) +FORCE_INLINE __m128i _mm_set1_epi64(__m64 _i) { - return vreinterpretq_m128i_s64(vdupq_n_s64(_i)); + return vreinterpretq_m128i_s64(vdupq_n_s64((int64_t) _i)); } // Sets the 2 signed 64-bit integer values to i. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_set1_epi64x&expand=4961 +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_set1_epi64x FORCE_INLINE __m128i _mm_set1_epi64x(int64_t _i) { return vreinterpretq_m128i_s64(vdupq_n_s64(_i)); @@ -563,10 +843,52 @@ FORCE_INLINE __m128i _mm_set_epi32(int i3, int i2, int i1, int i0) // https://msdn.microsoft.com/en-us/library/dk2sdw0h(v=vs.120).aspx FORCE_INLINE __m128i _mm_set_epi64x(int64_t i1, int64_t i2) { - int64_t ALIGN_STRUCT(16) data[2] = {i2, i1}; - return vreinterpretq_m128i_s64(vld1q_s64(data)); + return vreinterpretq_m128i_s64( + vcombine_s64(vcreate_s64(i2), vcreate_s64(i1))); } +// Returns the __m128i structure with its two 64-bit integer values +// initialized to the values of the two 64-bit integers passed in. +// https://msdn.microsoft.com/en-us/library/dk2sdw0h(v=vs.120).aspx +FORCE_INLINE __m128i _mm_set_epi64(__m64 i1, __m64 i2) +{ + return _mm_set_epi64x((int64_t) i1, (int64_t) i2); +} + +// Set packed double-precision (64-bit) floating-point elements in dst with the +// supplied values. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_set_pd +FORCE_INLINE __m128d _mm_set_pd(double e1, double e0) +{ + double ALIGN_STRUCT(16) data[2] = {e0, e1}; +#if defined(__aarch64__) + return vreinterpretq_m128d_f64(vld1q_f64((float64_t *) data)); +#else + return vreinterpretq_m128d_f32(vld1q_f32((float32_t *) data)); +#endif +} + +// Set packed double-precision (64-bit) floating-point elements in dst with the +// supplied values in reverse order. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_setr_pd +FORCE_INLINE __m128d _mm_setr_pd(double e1, double e0) +{ + return _mm_set_pd(e0, e1); +} + +// Copy double-precision (64-bit) floating-point element a to the lower element +// of dst, and zero the upper element. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_set_sd +FORCE_INLINE __m128d _mm_set_sd(double a) +{ + return _mm_set_pd(0, a); +} + +// Broadcast double-precision (64-bit) floating-point value a to all elements of +// dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_set_pd1 +#define _mm_set_pd1 _mm_set1_pd + // Stores four single-precision, floating-point values. // https://msdn.microsoft.com/en-us/library/vstudio/s3h4ay6y(v=vs.100).aspx FORCE_INLINE void _mm_store_ps(float *p, __m128 a) @@ -574,6 +896,51 @@ FORCE_INLINE void _mm_store_ps(float *p, __m128 a) vst1q_f32(p, vreinterpretq_f32_m128(a)); } +// Store the lower single-precision (32-bit) floating-point element from a into +// 4 contiguous elements in memory. mem_addr must be aligned on a 16-byte +// boundary or a general-protection exception may be generated. +// +// MEM[mem_addr+31:mem_addr] := a[31:0] +// MEM[mem_addr+63:mem_addr+32] := a[31:0] +// MEM[mem_addr+95:mem_addr+64] := a[31:0] +// MEM[mem_addr+127:mem_addr+96] := a[31:0] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_store_ps1 +FORCE_INLINE void _mm_store_ps1(float *p, __m128 a) +{ + float32_t a0 = vgetq_lane_f32(vreinterpretq_f32_m128(a), 0); + vst1q_f32(p, vdupq_n_f32(a0)); +} + +// Store the lower single-precision (32-bit) floating-point element from a into +// 4 contiguous elements in memory. mem_addr must be aligned on a 16-byte +// boundary or a general-protection exception may be generated. +// +// MEM[mem_addr+31:mem_addr] := a[31:0] +// MEM[mem_addr+63:mem_addr+32] := a[31:0] +// MEM[mem_addr+95:mem_addr+64] := a[31:0] +// MEM[mem_addr+127:mem_addr+96] := a[31:0] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_store1_ps +#define _mm_store1_ps _mm_store_ps1 + +// Store 4 single-precision (32-bit) floating-point elements from a into memory +// in reverse order. mem_addr must be aligned on a 16-byte boundary or a +// general-protection exception may be generated. +// +// MEM[mem_addr+31:mem_addr] := a[127:96] +// MEM[mem_addr+63:mem_addr+32] := a[95:64] +// MEM[mem_addr+95:mem_addr+64] := a[63:32] +// MEM[mem_addr+127:mem_addr+96] := a[31:0] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_storer_ps +FORCE_INLINE void _mm_storer_ps(float *p, __m128 a) +{ + float32x4_t tmp = vrev64q_f32(vreinterpretq_f32_m128(a)); + float32x4_t rev = vextq_f32(tmp, tmp, 2); + vst1q_f32(p, rev); +} + // Stores four single-precision, floating-point values. // https://msdn.microsoft.com/en-us/library/44e30x22(v=vs.100).aspx FORCE_INLINE void _mm_storeu_ps(float *p, __m128 a) @@ -602,6 +969,107 @@ FORCE_INLINE void _mm_store_ss(float *p, __m128 a) vst1q_lane_f32(p, vreinterpretq_f32_m128(a), 0); } +// Store 128-bits (composed of 2 packed double-precision (64-bit) floating-point +// elements) from a into memory. mem_addr must be aligned on a 16-byte boundary +// or a general-protection exception may be generated. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_store_pd +FORCE_INLINE void _mm_store_pd(double *mem_addr, __m128d a) +{ +#if defined(__aarch64__) + vst1q_f64((float64_t *) mem_addr, vreinterpretq_f64_m128d(a)); +#else + vst1q_f32((float32_t *) mem_addr, vreinterpretq_f32_m128d(a)); +#endif +} + +// Store the upper double-precision (64-bit) floating-point element from a into +// memory. +// +// MEM[mem_addr+63:mem_addr] := a[127:64] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_storeh_pd +FORCE_INLINE void _mm_storeh_pd(double *mem_addr, __m128d a) +{ +#if defined(__aarch64__) + vst1_f64((float64_t *) mem_addr, vget_high_f64(vreinterpretq_f64_m128d(a))); +#else + vst1_f32((float32_t *) mem_addr, vget_high_f32(vreinterpretq_f32_m128d(a))); +#endif +} + +// Store the lower double-precision (64-bit) floating-point element from a into +// memory. +// +// MEM[mem_addr+63:mem_addr] := a[63:0] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_storel_pd +FORCE_INLINE void _mm_storel_pd(double *mem_addr, __m128d a) +{ +#if defined(__aarch64__) + vst1_f64((float64_t *) mem_addr, vget_low_f64(vreinterpretq_f64_m128d(a))); +#else + vst1_f32((float32_t *) mem_addr, vget_low_f32(vreinterpretq_f32_m128d(a))); +#endif +} + +// Store 2 double-precision (64-bit) floating-point elements from a into memory +// in reverse order. mem_addr must be aligned on a 16-byte boundary or a +// general-protection exception may be generated. +// +// MEM[mem_addr+63:mem_addr] := a[127:64] +// MEM[mem_addr+127:mem_addr+64] := a[63:0] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_storer_pd +FORCE_INLINE void _mm_storer_pd(double *mem_addr, __m128d a) +{ + float32x4_t f = vreinterpretq_f32_m128d(a); + _mm_store_pd(mem_addr, vreinterpretq_m128d_f32(vextq_f32(f, f, 2))); +} + +// Store the lower double-precision (64-bit) floating-point element from a into +// 2 contiguous elements in memory. mem_addr must be aligned on a 16-byte +// boundary or a general-protection exception may be generated. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_store_pd1 +FORCE_INLINE void _mm_store_pd1(double *mem_addr, __m128d a) +{ +#if defined(__aarch64__) + float64x1_t a_low = vget_low_f64(vreinterpretq_f64_m128d(a)); + vst1q_f64((float64_t *) mem_addr, + vreinterpretq_f64_m128d(vcombine_f64(a_low, a_low))); +#else + float32x2_t a_low = vget_low_f32(vreinterpretq_f32_m128d(a)); + vst1q_f32((float32_t *) mem_addr, + vreinterpretq_f32_m128d(vcombine_f32(a_low, a_low))); +#endif +} + +// Store the lower double-precision (64-bit) floating-point element from a into +// memory. mem_addr does not need to be aligned on any particular boundary. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_store_sd +FORCE_INLINE void _mm_store_sd(double *mem_addr, __m128d a) +{ +#if defined(__aarch64__) + vst1_f64((float64_t *) mem_addr, vget_low_f64(vreinterpretq_f64_m128d(a))); +#else + vst1_u64((uint64_t *) mem_addr, vget_low_u64(vreinterpretq_u64_m128d(a))); +#endif +} + +// Store the lower double-precision (64-bit) floating-point element from a into +// 2 contiguous elements in memory. mem_addr must be aligned on a 16-byte +// boundary or a general-protection exception may be generated. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=9,526,5601&text=_mm_store1_pd +#define _mm_store1_pd _mm_store_pd1 + +// Store 128-bits (composed of 2 packed double-precision (64-bit) floating-point +// elements) from a into memory. mem_addr does not need to be aligned on any +// particular boundary. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_storeu_pd +FORCE_INLINE void _mm_storeu_pd(double *mem_addr, __m128d a) +{ + _mm_store_pd(mem_addr, a); +} + // Reads the lower 64 bits of b and stores them into the lower 64 bits of a. // https://msdn.microsoft.com/en-us/library/hhwf428f%28v=vs.90%29.aspx FORCE_INLINE void _mm_storel_epi64(__m128i *a, __m128i b) @@ -620,7 +1088,7 @@ FORCE_INLINE void _mm_storel_epi64(__m128i *a, __m128i b) // https://msdn.microsoft.com/en-us/library/h54t98ks(v=vs.90).aspx FORCE_INLINE void _mm_storel_pi(__m64 *p, __m128 a) { - *p = vget_low_f32(a); + *p = vreinterpret_m64_f32(vget_low_f32(a)); } // Stores the upper two single-precision, floating-point values of a to the @@ -632,7 +1100,7 @@ FORCE_INLINE void _mm_storel_pi(__m64 *p, __m128 a) // https://msdn.microsoft.com/en-us/library/a7525fs8(v%3dvs.90).aspx FORCE_INLINE void _mm_storeh_pi(__m64 *p, __m128 a) { - *p = vget_high_f32(a); + *p = vreinterpret_m64_f32(vget_high_f32(a)); } // Loads a single single-precision, floating-point value, copying it into all @@ -642,6 +1110,16 @@ FORCE_INLINE __m128 _mm_load1_ps(const float *p) { return vreinterpretq_m128_f32(vld1q_dup_f32(p)); } + +// Load a single-precision (32-bit) floating-point element from memory into all +// elements of dst. +// +// dst[31:0] := MEM[mem_addr+31:mem_addr] +// dst[63:32] := MEM[mem_addr+31:mem_addr] +// dst[95:64] := MEM[mem_addr+31:mem_addr] +// dst[127:96] := MEM[mem_addr+31:mem_addr] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_load_ps1 #define _mm_load_ps1 _mm_load1_ps // Sets the lower two single-precision, floating-point values with 64 @@ -661,6 +1139,22 @@ FORCE_INLINE __m128 _mm_loadl_pi(__m128 a, __m64 const *p) vcombine_f32(vld1_f32((const float32_t *) p), vget_high_f32(a))); } +// Load 4 single-precision (32-bit) floating-point elements from memory into dst +// in reverse order. mem_addr must be aligned on a 16-byte boundary or a +// general-protection exception may be generated. +// +// dst[31:0] := MEM[mem_addr+127:mem_addr+96] +// dst[63:32] := MEM[mem_addr+95:mem_addr+64] +// dst[95:64] := MEM[mem_addr+63:mem_addr+32] +// dst[127:96] := MEM[mem_addr+31:mem_addr] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadr_ps +FORCE_INLINE __m128 _mm_loadr_ps(const float *p) +{ + float32x4_t v = vrev64q_f32(vld1q_f32(p)); + return vreinterpretq_m128_f32(vextq_f32(v, v, 2)); +} + // Sets the upper two single-precision, floating-point values with 64 // bits of data loaded from the address p; the lower two values are passed // through from a. @@ -693,21 +1187,73 @@ FORCE_INLINE __m128 _mm_loadu_ps(const float *p) return vreinterpretq_m128_f32(vld1q_f32(p)); } -// Loads a double-precision, floating-point value. -// The upper double-precision, floating-point is set to zero. The address p does -// not need to be 16-byte aligned. -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/574w9fdd(v%3dvs.100) +// Load unaligned 16-bit integer from memory into the first element of dst. +// +// dst[15:0] := MEM[mem_addr+15:mem_addr] +// dst[MAX:16] := 0 +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadu_si16 +FORCE_INLINE __m128i _mm_loadu_si16(const void *p) +{ + return vreinterpretq_m128i_s16( + vsetq_lane_s16(*(const int16_t *) p, vdupq_n_s16(0), 0)); +} + +// Load unaligned 64-bit integer from memory into the first element of dst. +// +// dst[63:0] := MEM[mem_addr+63:mem_addr] +// dst[MAX:64] := 0 +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadu_si64 +FORCE_INLINE __m128i _mm_loadu_si64(const void *p) +{ + return vreinterpretq_m128i_s64( + vcombine_s64(vld1_s64((const int64_t *) p), vdup_n_s64(0))); +} + +// Load a double-precision (64-bit) floating-point element from memory into the +// lower of dst, and zero the upper element. mem_addr does not need to be +// aligned on any particular boundary. +// +// dst[63:0] := MEM[mem_addr+63:mem_addr] +// dst[127:64] := 0 +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_load_sd FORCE_INLINE __m128d _mm_load_sd(const double *p) { #if defined(__aarch64__) - return vsetq_lane_f64(*p, vdupq_n_f64(0), 0); + return vreinterpretq_m128d_f64(vsetq_lane_f64(*p, vdupq_n_f64(0), 0)); #else const float *fp = (const float *) p; float ALIGN_STRUCT(16) data[4] = {fp[0], fp[1], 0, 0}; - return vld1q_f32(data); + return vreinterpretq_m128d_f32(vld1q_f32(data)); #endif } +// Loads two double-precision from 16-byte aligned memory, floating-point +// values. +// +// dst[127:0] := MEM[mem_addr+127:mem_addr] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_load_pd +FORCE_INLINE __m128d _mm_load_pd(const double *p) +{ +#if defined(__aarch64__) + return vreinterpretq_m128d_f64(vld1q_f64(p)); +#else + const float *fp = (const float *) p; + float ALIGN_STRUCT(16) data[4] = {fp[0], fp[1], fp[2], fp[3]}; + return vreinterpretq_m128d_f32(vld1q_f32(data)); +#endif +} + +// Loads two double-precision from unaligned memory, floating-point values. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadu_pd +FORCE_INLINE __m128d _mm_loadu_pd(const double *p) +{ + return _mm_load_pd(p); +} + // Loads an single - precision, floating - point value into the low word and // clears the upper three words. // https://msdn.microsoft.com/en-us/library/548bb9h4%28v=vs.90%29.aspx @@ -716,6 +1262,8 @@ FORCE_INLINE __m128 _mm_load_ss(const float *p) return vreinterpretq_m128_f32(vsetq_lane_f32(*p, vdupq_n_f32(0), 0)); } +// Load 64-bit integer from memory into the first element of dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadl_epi64 FORCE_INLINE __m128i _mm_loadl_epi64(__m128i const *p) { /* Load the lower 64 bits of the value pointed to by p into the @@ -725,16 +1273,99 @@ FORCE_INLINE __m128i _mm_loadl_epi64(__m128i const *p) vcombine_s32(vld1_s32((int32_t const *) p), vcreate_s32(0))); } -/* Logic/Binary operations */ - -// Compares for inequality. -// https://msdn.microsoft.com/en-us/library/sf44thbx(v=vs.100).aspx -FORCE_INLINE __m128 _mm_cmpneq_ps(__m128 a, __m128 b) +// Load a double-precision (64-bit) floating-point element from memory into the +// lower element of dst, and copy the upper element from a to dst. mem_addr does +// not need to be aligned on any particular boundary. +// +// dst[63:0] := MEM[mem_addr+63:mem_addr] +// dst[127:64] := a[127:64] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadl_pd +FORCE_INLINE __m128d _mm_loadl_pd(__m128d a, const double *p) { - return vreinterpretq_m128_u32(vmvnq_u32( - vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)))); +#if defined(__aarch64__) + return vreinterpretq_m128d_f64( + vcombine_f64(vld1_f64(p), vget_high_f64(vreinterpretq_f64_m128d(a)))); +#else + return vreinterpretq_m128d_f32( + vcombine_f32(vld1_f32((const float *) p), + vget_high_f32(vreinterpretq_f32_m128d(a)))); +#endif } +// Load 2 double-precision (64-bit) floating-point elements from memory into dst +// in reverse order. mem_addr must be aligned on a 16-byte boundary or a +// general-protection exception may be generated. +// +// dst[63:0] := MEM[mem_addr+127:mem_addr+64] +// dst[127:64] := MEM[mem_addr+63:mem_addr] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadr_pd +FORCE_INLINE __m128d _mm_loadr_pd(const double *p) +{ +#if defined(__aarch64__) + float64x2_t v = vld1q_f64(p); + return vreinterpretq_m128d_f64(vextq_f64(v, v, 1)); +#else + int64x2_t v = vld1q_s64((const int64_t *) p); + return vreinterpretq_m128d_s64(vextq_s64(v, v, 1)); +#endif +} + +// Sets the low word to the single-precision, floating-point value of b +// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/35hdzazd(v=vs.100) +FORCE_INLINE __m128 _mm_move_ss(__m128 a, __m128 b) +{ + return vreinterpretq_m128_f32( + vsetq_lane_f32(vgetq_lane_f32(vreinterpretq_f32_m128(b), 0), + vreinterpretq_f32_m128(a), 0)); +} + +// Move the lower double-precision (64-bit) floating-point element from b to the +// lower element of dst, and copy the upper element from a to the upper element +// of dst. +// +// dst[63:0] := b[63:0] +// dst[127:64] := a[127:64] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_move_sd +FORCE_INLINE __m128d _mm_move_sd(__m128d a, __m128d b) +{ + return vreinterpretq_m128d_f32( + vcombine_f32(vget_low_f32(vreinterpretq_f32_m128d(b)), + vget_high_f32(vreinterpretq_f32_m128d(a)))); +} + +// Copy the lower 64-bit integer in a to the lower element of dst, and zero the +// upper element. +// +// dst[63:0] := a[63:0] +// dst[127:64] := 0 +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_move_epi64 +FORCE_INLINE __m128i _mm_move_epi64(__m128i a) +{ + return vreinterpretq_m128i_s64( + vsetq_lane_s64(0, vreinterpretq_s64_m128i(a), 1)); +} + +// Return vector of type __m128 with undefined elements. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_undefined_ps +FORCE_INLINE __m128 _mm_undefined_ps(void) +{ +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#endif + __m128 a; + return a; +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif +} + +/* Logic/Binary operations */ + // Computes the bitwise AND-NOT of the four single-precision, floating-point // values of a and b. // @@ -751,6 +1382,22 @@ FORCE_INLINE __m128 _mm_andnot_ps(__m128 a, __m128 b) vreinterpretq_s32_m128(a))); // *NOTE* argument swap } +// Compute the bitwise NOT of packed double-precision (64-bit) floating-point +// elements in a and then AND with b, and store the results in dst. +// +// FOR j := 0 to 1 +// i := j*64 +// dst[i+63:i] := ((NOT a[i+63:i]) AND b[i+63:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_andnot_pd +FORCE_INLINE __m128d _mm_andnot_pd(__m128d a, __m128d b) +{ + // *NOTE* argument swap + return vreinterpretq_m128d_s64( + vbicq_s64(vreinterpretq_s64_m128d(b), vreinterpretq_s64_m128d(a))); +} + // Computes the bitwise AND of the 128-bit value in b and the bitwise NOT of the // 128-bit value in a. // @@ -791,6 +1438,21 @@ FORCE_INLINE __m128 _mm_and_ps(__m128 a, __m128 b) vandq_s32(vreinterpretq_s32_m128(a), vreinterpretq_s32_m128(b))); } +// Compute the bitwise AND of packed double-precision (64-bit) floating-point +// elements in a and b, and store the results in dst. +// +// FOR j := 0 to 1 +// i := j*64 +// dst[i+63:i] := a[i+63:i] AND b[i+63:i] +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_and_pd +FORCE_INLINE __m128d _mm_and_pd(__m128d a, __m128d b) +{ + return vreinterpretq_m128d_s64( + vandq_s64(vreinterpretq_s64_m128d(a), vreinterpretq_s64_m128d(b))); +} + // Computes the bitwise OR of the four single-precision, floating-point values // of a and b. // https://msdn.microsoft.com/en-us/library/vstudio/7ctdsyy0(v=vs.100).aspx @@ -809,6 +1471,30 @@ FORCE_INLINE __m128 _mm_xor_ps(__m128 a, __m128 b) veorq_s32(vreinterpretq_s32_m128(a), vreinterpretq_s32_m128(b))); } +// Compute the bitwise XOR of packed double-precision (64-bit) floating-point +// elements in a and b, and store the results in dst. +// +// FOR j := 0 to 1 +// i := j*64 +// dst[i+63:i] := a[i+63:i] XOR b[i+63:i] +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_xor_pd +FORCE_INLINE __m128d _mm_xor_pd(__m128d a, __m128d b) +{ + return vreinterpretq_m128d_s64( + veorq_s64(vreinterpretq_s64_m128d(a), vreinterpretq_s64_m128d(b))); +} + +// Compute the bitwise OR of packed double-precision (64-bit) floating-point +// elements in a and b, and store the results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_or_pd +FORCE_INLINE __m128d _mm_or_pd(__m128d a, __m128d b) +{ + return vreinterpretq_m128d_s64( + vorrq_s64(vreinterpretq_s64_m128d(a), vreinterpretq_s64_m128d(b))); +} + // Computes the bitwise OR of the 128-bit value in a and the 128-bit value in b. // // r := a | b @@ -828,6 +1514,52 @@ FORCE_INLINE __m128i _mm_xor_si128(__m128i a, __m128i b) veorq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); } +// Duplicate the low double-precision (64-bit) floating-point element from a, +// and store the results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_movedup_pd +FORCE_INLINE __m128d _mm_movedup_pd(__m128d a) +{ +#if (__aarch64__) + return vreinterpretq_m128d_f64( + vdupq_laneq_f64(vreinterpretq_f64_m128d(a), 0)); +#else + return vreinterpretq_m128d_u64( + vdupq_n_u64(vgetq_lane_u64(vreinterpretq_u64_m128d(a), 0))); +#endif +} + +// Duplicate odd-indexed single-precision (32-bit) floating-point elements +// from a, and store the results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_movehdup_ps +FORCE_INLINE __m128 _mm_movehdup_ps(__m128 a) +{ +#if __has_builtin(__builtin_shufflevector) + return vreinterpretq_m128_f32(__builtin_shufflevector( + vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a), 1, 1, 3, 3)); +#else + float32_t a1 = vgetq_lane_f32(vreinterpretq_f32_m128(a), 1); + float32_t a3 = vgetq_lane_f32(vreinterpretq_f32_m128(a), 3); + float ALIGN_STRUCT(16) data[4] = {a1, a1, a3, a3}; + return vreinterpretq_m128_f32(vld1q_f32(data)); +#endif +} + +// Duplicate even-indexed single-precision (32-bit) floating-point elements +// from a, and store the results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_moveldup_ps +FORCE_INLINE __m128 _mm_moveldup_ps(__m128 a) +{ +#if __has_builtin(__builtin_shufflevector) + return vreinterpretq_m128_f32(__builtin_shufflevector( + vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a), 0, 0, 2, 2)); +#else + float32_t a0 = vgetq_lane_f32(vreinterpretq_f32_m128(a), 0); + float32_t a2 = vgetq_lane_f32(vreinterpretq_f32_m128(a), 2); + float ALIGN_STRUCT(16) data[4] = {a0, a0, a2, a2}; + return vreinterpretq_m128_f32(vld1q_f32(data)); +#endif +} + // Moves the upper two values of B into the lower two values of A. // // r3 := a3 @@ -854,21 +1586,171 @@ FORCE_INLINE __m128 _mm_movelh_ps(__m128 __A, __m128 __B) return vreinterpretq_m128_f32(vcombine_f32(a10, b10)); } +// Create mask from the most significant bit of each 8-bit element in a, and +// store the result in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_movemask_pi8 +FORCE_INLINE int _mm_movemask_pi8(__m64 a) +{ + uint8x8_t input = vreinterpret_u8_m64(a); +#if defined(__aarch64__) + static const int8x8_t shift = {0, 1, 2, 3, 4, 5, 6, 7}; + uint8x8_t tmp = vshr_n_u8(input, 7); + return vaddv_u8(vshl_u8(tmp, shift)); +#else + // Refer the implementation of `_mm_movemask_epi8` + uint16x4_t high_bits = vreinterpret_u16_u8(vshr_n_u8(input, 7)); + uint32x2_t paired16 = + vreinterpret_u32_u16(vsra_n_u16(high_bits, high_bits, 7)); + uint8x8_t paired32 = + vreinterpret_u8_u32(vsra_n_u32(paired16, paired16, 14)); + return vget_lane_u8(paired32, 0) | ((int) vget_lane_u8(paired32, 4) << 4); +#endif +} + +// Compute the absolute value of packed signed 32-bit integers in a, and store +// the unsigned results in dst. +// +// FOR j := 0 to 3 +// i := j*32 +// dst[i+31:i] := ABS(a[i+31:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_abs_epi32 FORCE_INLINE __m128i _mm_abs_epi32(__m128i a) { return vreinterpretq_m128i_s32(vabsq_s32(vreinterpretq_s32_m128i(a))); } +// Compute the absolute value of packed signed 16-bit integers in a, and store +// the unsigned results in dst. +// +// FOR j := 0 to 7 +// i := j*16 +// dst[i+15:i] := ABS(a[i+15:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_abs_epi16 FORCE_INLINE __m128i _mm_abs_epi16(__m128i a) { return vreinterpretq_m128i_s16(vabsq_s16(vreinterpretq_s16_m128i(a))); } +// Compute the absolute value of packed signed 8-bit integers in a, and store +// the unsigned results in dst. +// +// FOR j := 0 to 15 +// i := j*8 +// dst[i+7:i] := ABS(a[i+7:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_abs_epi8 FORCE_INLINE __m128i _mm_abs_epi8(__m128i a) { return vreinterpretq_m128i_s8(vabsq_s8(vreinterpretq_s8_m128i(a))); } +// Compute the absolute value of packed signed 32-bit integers in a, and store +// the unsigned results in dst. +// +// FOR j := 0 to 1 +// i := j*32 +// dst[i+31:i] := ABS(a[i+31:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_abs_pi32 +FORCE_INLINE __m64 _mm_abs_pi32(__m64 a) +{ + return vreinterpret_m64_s32(vabs_s32(vreinterpret_s32_m64(a))); +} + +// Compute the absolute value of packed signed 16-bit integers in a, and store +// the unsigned results in dst. +// +// FOR j := 0 to 3 +// i := j*16 +// dst[i+15:i] := ABS(a[i+15:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_abs_pi16 +FORCE_INLINE __m64 _mm_abs_pi16(__m64 a) +{ + return vreinterpret_m64_s16(vabs_s16(vreinterpret_s16_m64(a))); +} + +// Compute the absolute value of packed signed 8-bit integers in a, and store +// the unsigned results in dst. +// +// FOR j := 0 to 7 +// i := j*8 +// dst[i+7:i] := ABS(a[i+7:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_abs_pi8 +FORCE_INLINE __m64 _mm_abs_pi8(__m64 a) +{ + return vreinterpret_m64_s8(vabs_s8(vreinterpret_s8_m64(a))); +} + +// Concatenate 16-byte blocks in a and b into a 32-byte temporary result, shift +// the result right by imm8 bytes, and store the low 16 bytes in dst. +// +// tmp[255:0] := ((a[127:0] << 128)[255:0] OR b[127:0]) >> (imm8*8) +// dst[127:0] := tmp[127:0] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_alignr_epi8 +#define _mm_alignr_epi8(a, b, imm) \ + __extension__({ \ + __m128i ret; \ + if (unlikely((imm) >= 32)) { \ + ret = _mm_setzero_si128(); \ + } else { \ + uint8x16_t tmp_low, tmp_high; \ + if (imm >= 16) { \ + const int idx = imm - 16; \ + tmp_low = vreinterpretq_u8_m128i(a); \ + tmp_high = vdupq_n_u8(0); \ + ret = \ + vreinterpretq_m128i_u8(vextq_u8(tmp_low, tmp_high, idx)); \ + } else { \ + const int idx = imm; \ + tmp_low = vreinterpretq_u8_m128i(b); \ + tmp_high = vreinterpretq_u8_m128i(a); \ + ret = \ + vreinterpretq_m128i_u8(vextq_u8(tmp_low, tmp_high, idx)); \ + } \ + } \ + ret; \ + }) + +// Concatenate 8-byte blocks in a and b into a 16-byte temporary result, shift +// the result right by imm8 bytes, and store the low 8 bytes in dst. +// +// tmp[127:0] := ((a[63:0] << 64)[127:0] OR b[63:0]) >> (imm8*8) +// dst[63:0] := tmp[63:0] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_alignr_pi8 +#define _mm_alignr_pi8(a, b, imm) \ + __extension__({ \ + __m64 ret; \ + if (unlikely((imm) >= 16)) { \ + ret = vreinterpret_m64_s8(vdup_n_s8(0)); \ + } else { \ + uint8x8_t tmp_low, tmp_high; \ + if (imm >= 8) { \ + const int idx = imm - 8; \ + tmp_low = vreinterpret_u8_m64(a); \ + tmp_high = vdup_n_u8(0); \ + ret = vreinterpret_m64_u8(vext_u8(tmp_low, tmp_high, idx)); \ + } else { \ + const int idx = imm; \ + tmp_low = vreinterpret_u8_m64(b); \ + tmp_high = vreinterpret_u8_m64(a); \ + ret = vreinterpret_m64_u8(vext_u8(tmp_low, tmp_high, idx)); \ + } \ + } \ + ret; \ + }) + // Takes the upper 64 bits of a and places it in the low end of the result // Takes the lower 64 bits of b and places it into the high end of the result. FORCE_INLINE __m128 _mm_shuffle_ps_1032(__m128 a, __m128 b) @@ -1007,20 +1889,17 @@ FORCE_INLINE __m128 _mm_shuffle_ps_2032(__m128 a, __m128 b) // NEON does not support a general purpose permute intrinsic // Selects four specific single-precision, floating-point values from a and b, // based on the mask i. +// +// C equivalent: +// __m128 _mm_shuffle_ps_default(__m128 a, __m128 b, +// __constrange(0, 255) int imm) { +// __m128 ret; +// ret[0] = a[imm & 0x3]; ret[1] = a[(imm >> 2) & 0x3]; +// ret[2] = b[(imm >> 4) & 0x03]; ret[3] = b[(imm >> 6) & 0x03]; +// return ret; +// } +// // https://msdn.microsoft.com/en-us/library/vstudio/5f0858x0(v=vs.100).aspx -#if 0 /* C version */ -FORCE_INLINE __m128 _mm_shuffle_ps_default(__m128 a, - __m128 b, - __constrange(0, 255) int imm) -{ - __m128 ret; - ret[0] = a[imm & 0x3]; - ret[1] = a[(imm >> 2) & 0x3]; - ret[2] = b[(imm >> 4) & 0x03]; - ret[3] = b[(imm >> 6) & 0x03]; - return ret; -} -#endif #define _mm_shuffle_ps_default(a, b, imm) \ __extension__({ \ float32x4_t ret; \ @@ -1198,7 +2077,7 @@ FORCE_INLINE __m128i _mm_shuffle_epi_3332(__m128i a) // Shuffle packed 8-bit integers in a according to shuffle control mask in the // corresponding 8-bit element of b, and store the results in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shuffle_epi8&expand=5146 +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shuffle_epi8 FORCE_INLINE __m128i _mm_shuffle_epi8(__m128i a, __m128i b) { int8x16_t tbl = vreinterpretq_s8_m128i(a); // input a @@ -1211,9 +2090,9 @@ FORCE_INLINE __m128i _mm_shuffle_epi8(__m128i a, __m128i b) int8x16_t ret; // %e and %f represent the even and odd D registers // respectively. - __asm__( - " vtbl.8 %e[ret], {%e[tbl], %f[tbl]}, %e[idx]\n" - " vtbl.8 %f[ret], {%e[tbl], %f[tbl]}, %f[idx]\n" + __asm__ __volatile__( + "vtbl.8 %e[ret], {%e[tbl], %f[tbl]}, %e[idx]\n" + "vtbl.8 %f[ret], {%e[tbl], %f[tbl]}, %f[idx]\n" : [ret] "=&w"(ret) : [tbl] "w"(tbl), [idx] "w"(idx_masked)); return vreinterpretq_m128i_s8(ret); @@ -1226,18 +2105,14 @@ FORCE_INLINE __m128i _mm_shuffle_epi8(__m128i a, __m128i b) #endif } -#if 0 /* C version */ -FORCE_INLINE __m128i _mm_shuffle_epi32_default(__m128i a, - __constrange(0, 255) int imm) -{ - __m128i ret; - ret[0] = a[imm & 0x3]; - ret[1] = a[(imm >> 2) & 0x3]; - ret[2] = a[(imm >> 4) & 0x03]; - ret[3] = a[(imm >> 6) & 0x03]; - return ret; -} -#endif +// C equivalent: +// __m128i _mm_shuffle_epi32_default(__m128i a, +// __constrange(0, 255) int imm) { +// __m128i ret; +// ret[0] = a[imm & 0x3]; ret[1] = a[(imm >> 2) & 0x3]; +// ret[2] = a[(imm >> 4) & 0x03]; ret[3] = a[(imm >> 6) & 0x03]; +// return ret; +// } #define _mm_shuffle_epi32_default(a, imm) \ __extension__({ \ int32x4_t ret; \ @@ -1410,6 +2285,25 @@ FORCE_INLINE __m128i _mm_shuffle_epi32_default(__m128i a, #define _mm_shufflehi_epi16(a, imm) _mm_shufflehi_epi16_function((a), (imm)) #endif +// Shuffle double-precision (64-bit) floating-point elements using the control +// in imm8, and store the results in dst. +// +// dst[63:0] := (imm8[0] == 0) ? a[63:0] : a[127:64] +// dst[127:64] := (imm8[1] == 0) ? b[63:0] : b[127:64] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shuffle_pd +#if __has_builtin(__builtin_shufflevector) +#define _mm_shuffle_pd(a, b, imm8) \ + vreinterpretq_m128d_s64(__builtin_shufflevector( \ + vreinterpretq_s64_m128d(a), vreinterpretq_s64_m128d(b), imm8 & 0x1, \ + ((imm8 & 0x2) >> 1) + 2)) +#else +#define _mm_shuffle_pd(a, b, imm8) \ + _mm_castsi128_pd(_mm_set_epi64x( \ + vgetq_lane_s64(vreinterpretq_s64_m128d(b), (imm8 & 0x2) >> 1), \ + vgetq_lane_s64(vreinterpretq_s64_m128d(a), imm8 & 0x1))) +#endif + // Blend packed 16-bit integers from a and b using control mask imm8, and store // the results in dst. // @@ -1462,27 +2356,13 @@ FORCE_INLINE __m128i _mm_blendv_epi8(__m128i _a, __m128i _b, __m128i _mask) /* Shifts */ -// Shifts the 4 signed 32-bit integers in a right by count bits while shifting -// in the sign bit. -// -// r0 := a0 >> count -// r1 := a1 >> count -// r2 := a2 >> count -// r3 := a3 >> count immediate -FORCE_INLINE __m128i _mm_srai_epi32(__m128i a, int count) -{ - return (__m128i) vshlq_s32((int32x4_t) a, vdupq_n_s32(-count)); -} -// Shifts the 8 signed 16-bit integers in a right by count bits while shifting -// in the sign bit. -// -// r0 := a0 >> count -// r1 := a1 >> count -// ... -// r7 := a7 >> count -FORCE_INLINE __m128i _mm_srai_epi16(__m128i a, int count) +// Shift packed 16-bit integers in a right by imm while shifting in sign +// bits, and store the results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srai_epi16 +FORCE_INLINE __m128i _mm_srai_epi16(__m128i a, int imm) { + const int count = (imm & ~15) ? 15 : imm; return (__m128i) vshlq_s16((int16x8_t) a, vdupq_n_s16(-count)); } @@ -1498,9 +2378,10 @@ FORCE_INLINE __m128i _mm_srai_epi16(__m128i a, int count) #define _mm_slli_epi16(a, imm) \ __extension__({ \ __m128i ret; \ - if ((imm) <= 0) { \ + if (unlikely((imm)) <= 0) { \ ret = a; \ - } else if ((imm) > 31) { \ + } \ + if (unlikely((imm) > 15)) { \ ret = _mm_setzero_si128(); \ } else { \ ret = vreinterpretq_m128i_s16( \ @@ -1513,112 +2394,141 @@ FORCE_INLINE __m128i _mm_srai_epi16(__m128i a, int count) // shifting in zeros. : // https://msdn.microsoft.com/en-us/library/z2k3bbtb%28v=vs.90%29.aspx // FORCE_INLINE __m128i _mm_slli_epi32(__m128i a, __constrange(0,255) int imm) -#define _mm_slli_epi32(a, imm) \ - __extension__({ \ - __m128i ret; \ - if ((imm) <= 0) { \ - ret = a; \ - } else if ((imm) > 31) { \ - ret = _mm_setzero_si128(); \ - } else { \ - ret = vreinterpretq_m128i_s32( \ - vshlq_n_s32(vreinterpretq_s32_m128i(a), (imm))); \ - } \ - ret; \ - }) +FORCE_INLINE __m128i _mm_slli_epi32(__m128i a, int imm) +{ + if (unlikely(imm <= 0)) /* TODO: add constant range macro: [0, 255] */ + return a; + if (unlikely(imm > 31)) + return _mm_setzero_si128(); + return vreinterpretq_m128i_s32( + vshlq_s32(vreinterpretq_s32_m128i(a), vdupq_n_s32(imm))); +} // Shift packed 64-bit integers in a left by imm8 while shifting in zeros, and // store the results in dst. -#define _mm_slli_epi64(a, imm) \ - __extension__({ \ - __m128i ret; \ - if ((imm) <= 0) { \ - ret = a; \ - } else if ((imm) > 63) { \ - ret = _mm_setzero_si128(); \ - } else { \ - ret = vreinterpretq_m128i_s64( \ - vshlq_n_s64(vreinterpretq_s64_m128i(a), (imm))); \ - } \ - ret; \ +FORCE_INLINE __m128i _mm_slli_epi64(__m128i a, int imm) +{ + if (unlikely(imm <= 0)) /* TODO: add constant range macro: [0, 255] */ + return a; + if (unlikely(imm > 63)) + return _mm_setzero_si128(); + return vreinterpretq_m128i_s64( + vshlq_s64(vreinterpretq_s64_m128i(a), vdupq_n_s64(imm))); +} + +// Shift packed 16-bit integers in a right by imm8 while shifting in zeros, and +// store the results in dst. +// +// FOR j := 0 to 7 +// i := j*16 +// IF imm8[7:0] > 15 +// dst[i+15:i] := 0 +// ELSE +// dst[i+15:i] := ZeroExtend16(a[i+15:i] >> imm8[7:0]) +// FI +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srli_epi16 +#define _mm_srli_epi16(a, imm) \ + __extension__({ \ + __m128i ret; \ + if (unlikely(imm) == 0) { \ + ret = a; \ + } \ + if (likely(0 < (imm) && (imm) < 16)) { \ + ret = vreinterpretq_m128i_u16( \ + vshlq_u16(vreinterpretq_u16_m128i(a), vdupq_n_s16(-imm))); \ + } else { \ + ret = _mm_setzero_si128(); \ + } \ + ret; \ }) -// Shifts the 8 signed or unsigned 16-bit integers in a right by count bits -// while shifting in zeros. +// Shift packed 32-bit integers in a right by imm8 while shifting in zeros, and +// store the results in dst. // -// r0 := srl(a0, count) -// r1 := srl(a1, count) -// ... -// r7 := srl(a7, count) +// FOR j := 0 to 3 +// i := j*32 +// IF imm8[7:0] > 31 +// dst[i+31:i] := 0 +// ELSE +// dst[i+31:i] := ZeroExtend32(a[i+31:i] >> imm8[7:0]) +// FI +// ENDFOR // -// https://msdn.microsoft.com/en-us/library/6tcwd38t(v=vs.90).aspx -#define _mm_srli_epi16(a, imm) \ - __extension__({ \ - __m128i ret; \ - if ((imm) <= 0) { \ - ret = a; \ - } else if ((imm) > 31) { \ - ret = _mm_setzero_si128(); \ - } else { \ - ret = vreinterpretq_m128i_u16( \ - vshrq_n_u16(vreinterpretq_u16_m128i(a), (imm))); \ - } \ - ret; \ - }) - -// Shifts the 4 signed or unsigned 32-bit integers in a right by count bits -// while shifting in zeros. -// https://msdn.microsoft.com/en-us/library/w486zcfa(v=vs.100).aspx FORCE_INLINE -// __m128i _mm_srli_epi32(__m128i a, __constrange(0,255) int imm) -#define _mm_srli_epi32(a, imm) \ - __extension__({ \ - __m128i ret; \ - if ((imm) <= 0) { \ - ret = a; \ - } else if ((imm) > 31) { \ - ret = _mm_setzero_si128(); \ - } else { \ - ret = vreinterpretq_m128i_u32( \ - vshrq_n_u32(vreinterpretq_u32_m128i(a), (imm))); \ - } \ - ret; \ +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srli_epi32 +// FORCE_INLINE __m128i _mm_srli_epi32(__m128i a, __constrange(0,255) int imm) +#define _mm_srli_epi32(a, imm) \ + __extension__({ \ + __m128i ret; \ + if (unlikely((imm) == 0)) { \ + ret = a; \ + } \ + if (likely(0 < (imm) && (imm) < 32)) { \ + ret = vreinterpretq_m128i_u32( \ + vshlq_u32(vreinterpretq_u32_m128i(a), vdupq_n_s32(-imm))); \ + } else { \ + ret = _mm_setzero_si128(); \ + } \ + ret; \ }) // Shift packed 64-bit integers in a right by imm8 while shifting in zeros, and // store the results in dst. -#define _mm_srli_epi64(a, imm) \ - __extension__({ \ - __m128i ret; \ - if ((imm) <= 0) { \ - ret = a; \ - } else if ((imm) > 63) { \ - ret = _mm_setzero_si128(); \ - } else { \ - ret = vreinterpretq_m128i_u64( \ - vshrq_n_u64(vreinterpretq_u64_m128i(a), (imm))); \ - } \ - ret; \ +// +// FOR j := 0 to 1 +// i := j*64 +// IF imm8[7:0] > 63 +// dst[i+63:i] := 0 +// ELSE +// dst[i+63:i] := ZeroExtend64(a[i+63:i] >> imm8[7:0]) +// FI +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srli_epi64 +#define _mm_srli_epi64(a, imm) \ + __extension__({ \ + __m128i ret; \ + if (unlikely((imm) == 0)) { \ + ret = a; \ + } \ + if (likely(0 < (imm) && (imm) < 64)) { \ + ret = vreinterpretq_m128i_u64( \ + vshlq_u64(vreinterpretq_u64_m128i(a), vdupq_n_s64(-imm))); \ + } else { \ + ret = _mm_setzero_si128(); \ + } \ + ret; \ }) -// Shifts the 4 signed 32 - bit integers in a right by count bits while shifting -// in the sign bit. -// https://msdn.microsoft.com/en-us/library/z1939387(v=vs.100).aspx +// Shift packed 32-bit integers in a right by imm8 while shifting in sign bits, +// and store the results in dst. +// +// FOR j := 0 to 3 +// i := j*32 +// IF imm8[7:0] > 31 +// dst[i+31:i] := (a[i+31] ? 0xFFFFFFFF : 0x0) +// ELSE +// dst[i+31:i] := SignExtend32(a[i+31:i] >> imm8[7:0]) +// FI +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srai_epi32 // FORCE_INLINE __m128i _mm_srai_epi32(__m128i a, __constrange(0,255) int imm) -#define _mm_srai_epi32(a, imm) \ - __extension__({ \ - __m128i ret; \ - if ((imm) <= 0) { \ - ret = a; \ - } else if ((imm) > 31) { \ - ret = vreinterpretq_m128i_s32( \ - vshrq_n_s32(vreinterpretq_s32_m128i(a), 16)); \ - ret = vreinterpretq_m128i_s32( \ - vshrq_n_s32(vreinterpretq_s32_m128i(ret), 16)); \ - } else { \ - ret = vreinterpretq_m128i_s32( \ - vshrq_n_s32(vreinterpretq_s32_m128i(a), (imm))); \ - } \ - ret; \ +#define _mm_srai_epi32(a, imm) \ + __extension__({ \ + __m128i ret; \ + if (unlikely((imm) == 0)) { \ + ret = a; \ + } \ + if (likely(0 < (imm) && (imm) < 32)) { \ + ret = vreinterpretq_m128i_s32( \ + vshlq_s32(vreinterpretq_s32_m128i(a), vdupq_n_s32(-imm))); \ + } else { \ + ret = vreinterpretq_m128i_s32( \ + vshrq_n_s32(vreinterpretq_s32_m128i(a), 31)); \ + } \ + ret; \ }) // Shifts the 128 - bit value in a right by imm bytes while shifting in @@ -1631,9 +2541,10 @@ FORCE_INLINE __m128i _mm_srai_epi16(__m128i a, int count) #define _mm_srli_si128(a, imm) \ __extension__({ \ __m128i ret; \ - if ((imm) <= 0) { \ + if (unlikely((imm) <= 0)) { \ ret = a; \ - } else if ((imm) > 15) { \ + } \ + if (unlikely((imm) > 15)) { \ ret = _mm_setzero_si128(); \ } else { \ ret = vreinterpretq_m128i_s8( \ @@ -1652,9 +2563,10 @@ FORCE_INLINE __m128i _mm_srai_epi16(__m128i a, int count) #define _mm_slli_si128(a, imm) \ __extension__({ \ __m128i ret; \ - if ((imm) <= 0) { \ + if (unlikely((imm) <= 0)) { \ ret = a; \ - } else if ((imm) > 15) { \ + } \ + if (unlikely((imm) > 15)) { \ ret = _mm_setzero_si128(); \ } else { \ ret = vreinterpretq_m128i_s8(vextq_s8( \ @@ -1674,8 +2586,8 @@ FORCE_INLINE __m128i _mm_srai_epi16(__m128i a, int count) // https://msdn.microsoft.com/en-us/library/c79w388h(v%3dvs.90).aspx FORCE_INLINE __m128i _mm_sll_epi16(__m128i a, __m128i count) { - uint64_t c = ((SIMDVec *) &count)->m128_u64[0]; - if (c > 15) + uint64_t c = vreinterpretq_nth_u64_m128i(count, 0); + if (unlikely(c > 15)) return _mm_setzero_si128(); int16x8_t vc = vdupq_n_s16((int16_t) c); @@ -1693,8 +2605,8 @@ FORCE_INLINE __m128i _mm_sll_epi16(__m128i a, __m128i count) // https://msdn.microsoft.com/en-us/library/6fe5a6s9(v%3dvs.90).aspx FORCE_INLINE __m128i _mm_sll_epi32(__m128i a, __m128i count) { - uint64_t c = ((SIMDVec *) &count)->m128_u64[0]; - if (c > 31) + uint64_t c = vreinterpretq_nth_u64_m128i(count, 0); + if (unlikely(c > 31)) return _mm_setzero_si128(); int32x4_t vc = vdupq_n_s32((int32_t) c); @@ -1710,8 +2622,8 @@ FORCE_INLINE __m128i _mm_sll_epi32(__m128i a, __m128i count) // https://msdn.microsoft.com/en-us/library/6ta9dffd(v%3dvs.90).aspx FORCE_INLINE __m128i _mm_sll_epi64(__m128i a, __m128i count) { - uint64_t c = ((SIMDVec *) &count)->m128_u64[0]; - if (c > 63) + uint64_t c = vreinterpretq_nth_u64_m128i(count, 0); + if (unlikely(c > 63)) return _mm_setzero_si128(); int64x2_t vc = vdupq_n_s64((int64_t) c); @@ -1729,8 +2641,8 @@ FORCE_INLINE __m128i _mm_sll_epi64(__m128i a, __m128i count) // https://msdn.microsoft.com/en-us/library/wd5ax830(v%3dvs.90).aspx FORCE_INLINE __m128i _mm_srl_epi16(__m128i a, __m128i count) { - uint64_t c = ((SIMDVec *) &count)->m128_u64[0]; - if (c > 15) + uint64_t c = vreinterpretq_nth_u64_m128i(count, 0); + if (unlikely(c > 15)) return _mm_setzero_si128(); int16x8_t vc = vdupq_n_s16(-(int16_t) c); @@ -1748,8 +2660,8 @@ FORCE_INLINE __m128i _mm_srl_epi16(__m128i a, __m128i count) // https://msdn.microsoft.com/en-us/library/a9cbttf4(v%3dvs.90).aspx FORCE_INLINE __m128i _mm_srl_epi32(__m128i a, __m128i count) { - uint64_t c = ((SIMDVec *) &count)->m128_u64[0]; - if (c > 31) + uint64_t c = vreinterpretq_nth_u64_m128i(count, 0); + if (unlikely(c > 31)) return _mm_setzero_si128(); int32x4_t vc = vdupq_n_s32(-(int32_t) c); @@ -1765,8 +2677,8 @@ FORCE_INLINE __m128i _mm_srl_epi32(__m128i a, __m128i count) // https://msdn.microsoft.com/en-us/library/yf6cf9k8(v%3dvs.90).aspx FORCE_INLINE __m128i _mm_srl_epi64(__m128i a, __m128i count) { - uint64_t c = ((SIMDVec *) &count)->m128_u64[0]; - if (c > 63) + uint64_t c = vreinterpretq_nth_u64_m128i(count, 0); + if (unlikely(c > 63)) return _mm_setzero_si128(); int64x2_t vc = vdupq_n_s64(-(int64_t) c); @@ -1779,19 +2691,6 @@ FORCE_INLINE __m128i _mm_srl_epi64(__m128i a, __m128i count) // https://msdn.microsoft.com/en-us/library/vstudio/s090c8fk(v=vs.100).aspx FORCE_INLINE int _mm_movemask_epi8(__m128i a) { -#if defined(__aarch64__) - uint8x16_t input = vreinterpretq_u8_m128i(a); - const int8_t ALIGN_STRUCT(16) - xr[16] = {-7, -6, -5, -4, -3, -2, -1, 0, -7, -6, -5, -4, -3, -2, -1, 0}; - const uint8x16_t mask_and = vdupq_n_u8(0x80); - const int8x16_t mask_shift = vld1q_s8(xr); - const uint8x16_t mask_result = - vshlq_u8(vandq_u8(input, mask_and), mask_shift); - uint8x8_t lo = vget_low_u8(mask_result); - uint8x8_t hi = vget_high_u8(mask_result); - - return vaddv_u8(lo) + (vaddv_u8(hi) << 8); -#else // Use increasingly wide shifts+adds to collect the sign bits // together. // Since the widening shifts would be rather confusing to follow in little @@ -1868,7 +2767,29 @@ FORCE_INLINE int _mm_movemask_epi8(__m128i a) // d2 // Note: Little endian would return the correct value 4b (01001011) instead. return vgetq_lane_u8(paired64, 0) | ((int) vgetq_lane_u8(paired64, 8) << 8); -#endif +} + +// Copy the lower 64-bit integer in a to dst. +// +// dst[63:0] := a[63:0] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_movepi64_pi64 +FORCE_INLINE __m64 _mm_movepi64_pi64(__m128i a) +{ + return vreinterpret_m64_s64(vget_low_s64(vreinterpretq_s64_m128i(a))); +} + +// Copy the 64-bit integer a to the lower element of dst, and zero the upper +// element. +// +// dst[63:0] := a[63:0] +// dst[127:64] := 0 +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_movpi64_epi64 +FORCE_INLINE __m128i _mm_movpi64_epi64(__m64 a) +{ + return vreinterpretq_m128i_s64( + vcombine_s64(vreinterpret_s64_m64(a), vdup_n_s64(0))); } // NEON does not provide this method @@ -1879,10 +2800,9 @@ FORCE_INLINE int _mm_movemask_ps(__m128 a) { uint32x4_t input = vreinterpretq_u32_m128(a); #if defined(__aarch64__) - static const int32x4_t shift = {-31, -30, -29, -28}; - static const uint32x4_t highbit = {0x80000000, 0x80000000, 0x80000000, - 0x80000000}; - return vaddvq_u32(vshlq_u32(vandq_u32(input, highbit), shift)); + static const int32x4_t shift = {0, 1, 2, 3}; + uint32x4_t tmp = vshrq_n_u32(input, 31); + return vaddvq_u32(vshlq_u32(tmp, shift)); #else // Uses the exact same method as _mm_movemask_epi8, see that for details. // Shift out everything but the sign bits with a 32-bit unsigned shift @@ -1896,9 +2816,18 @@ FORCE_INLINE int _mm_movemask_ps(__m128 a) #endif } +// Compute the bitwise NOT of a and then AND with a 128-bit vector containing +// all 1's, and return 1 if the result is zero, otherwise return 0. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_test_all_ones +FORCE_INLINE int _mm_test_all_ones(__m128i a) +{ + return (uint64_t)(vgetq_lane_s64(a, 0) & vgetq_lane_s64(a, 1)) == + ~(uint64_t) 0; +} + // Compute the bitwise AND of 128 bits (representing integer data) in a and // mask, and return 1 if the result is zero, otherwise return 0. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_test_all_zeros&expand=5871 +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_test_all_zeros FORCE_INLINE int _mm_test_all_zeros(__m128i a, __m128i mask) { int64x2_t a_and_mask = @@ -1923,6 +2852,20 @@ FORCE_INLINE __m128 _mm_sub_ps(__m128 a, __m128 b) vsubq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); } +// Subtract the lower single-precision (32-bit) floating-point element in b from +// the lower single-precision (32-bit) floating-point element in a, store the +// result in the lower element of dst, and copy the upper 3 packed elements from +// a to the upper elements of dst. +// +// dst[31:0] := a[31:0] - b[31:0] +// dst[127:32] := a[127:32] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sub_ss +FORCE_INLINE __m128 _mm_sub_ss(__m128 a, __m128 b) +{ + return _mm_move_ss(a, _mm_sub_ps(a, b)); +} + // Subtract 2 packed 64-bit integers in b from 2 packed 64-bit integers in a, // and store the results in dst. // r0 := a0 - b0 @@ -1948,18 +2891,35 @@ FORCE_INLINE __m128i _mm_sub_epi32(__m128i a, __m128i b) vsubq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); } +// Subtract packed 16-bit integers in b from packed 16-bit integers in a, and +// store the results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sub_epi16 FORCE_INLINE __m128i _mm_sub_epi16(__m128i a, __m128i b) { return vreinterpretq_m128i_s16( vsubq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); } +// Subtract packed 8-bit integers in b from packed 8-bit integers in a, and +// store the results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sub_epi8 FORCE_INLINE __m128i _mm_sub_epi8(__m128i a, __m128i b) { return vreinterpretq_m128i_s8( vsubq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); } +// Subtract 64-bit integer b from 64-bit integer a, and store the result in dst. +// +// dst[63:0] := a[63:0] - b[63:0] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sub_si64 +FORCE_INLINE __m64 _mm_sub_si64(__m64 a, __m64 b) +{ + return vreinterpret_m64_s64( + vsub_s64(vreinterpret_s64_m64(a), vreinterpret_s64_m64(b))); +} + // Subtracts the 8 unsigned 16-bit integers of bfrom the 8 unsigned 16-bit // integers of a and saturates.. // https://technet.microsoft.com/en-us/subscriptions/index/f44y0s19(v=vs.90).aspx @@ -2014,6 +2974,44 @@ FORCE_INLINE __m128i _mm_subs_epi16(__m128i a, __m128i b) vqsubq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); } +// Subtract packed double-precision (64-bit) floating-point elements in b from +// packed double-precision (64-bit) floating-point elements in a, and store the +// results in dst. +// +// FOR j := 0 to 1 +// i := j*64 +// dst[i+63:i] := a[i+63:i] - b[i+63:i] +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_sub_pd +FORCE_INLINE __m128d _mm_sub_pd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) + return vreinterpretq_m128d_f64( + vsubq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#else + double *da = (double *) &a; + double *db = (double *) &b; + double c[2]; + c[0] = da[0] - db[0]; + c[1] = da[1] - db[1]; + return vld1q_f32((float32_t *) c); +#endif +} + +// Subtract the lower double-precision (64-bit) floating-point element in b from +// the lower double-precision (64-bit) floating-point element in a, store the +// result in the lower element of dst, and copy the upper element from a to the +// upper element of dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sub_sd +FORCE_INLINE __m128d _mm_sub_sd(__m128d a, __m128d b) +{ + return _mm_move_sd(a, _mm_sub_pd(a, b)); +} + +// Add packed unsigned 16-bit integers in a and b using saturation, and store +// the results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_adds_epu16 FORCE_INLINE __m128i _mm_adds_epu16(__m128i a, __m128i b) { return vreinterpretq_m128i_u16( @@ -2039,18 +3037,23 @@ FORCE_INLINE __m128i _mm_sign_epi8(__m128i _a, __m128i _b) int8x16_t a = vreinterpretq_s8_m128i(_a); int8x16_t b = vreinterpretq_s8_m128i(_b); - int8x16_t zero = vdupq_n_s8(0); // signed shift right: faster than vclt // (b < 0) ? 0xFF : 0 uint8x16_t ltMask = vreinterpretq_u8_s8(vshrq_n_s8(b, 7)); + // (b == 0) ? 0xFF : 0 - int8x16_t zeroMask = vreinterpretq_s8_u8(vceqq_s8(b, zero)); - // -a - int8x16_t neg = vnegq_s8(a); - // bitwise select either a or neg based on ltMask - int8x16_t masked = vbslq_s8(ltMask, a, neg); +#if defined(__aarch64__) + int8x16_t zeroMask = vreinterpretq_s8_u8(vceqzq_s8(b)); +#else + int8x16_t zeroMask = vreinterpretq_s8_u8(vceqq_s8(b, vdupq_n_s8(0))); +#endif + + // bitwise select either a or nagative 'a' (vnegq_s8(a) return nagative 'a') + // based on ltMask + int8x16_t masked = vbslq_s8(ltMask, vnegq_s8(a), a); // res = masked & (~zeroMask) int8x16_t res = vbicq_s8(masked, zeroMask); + return vreinterpretq_m128i_s8(res); } @@ -2073,16 +3076,19 @@ FORCE_INLINE __m128i _mm_sign_epi16(__m128i _a, __m128i _b) int16x8_t a = vreinterpretq_s16_m128i(_a); int16x8_t b = vreinterpretq_s16_m128i(_b); - int16x8_t zero = vdupq_n_s16(0); // signed shift right: faster than vclt // (b < 0) ? 0xFFFF : 0 uint16x8_t ltMask = vreinterpretq_u16_s16(vshrq_n_s16(b, 15)); // (b == 0) ? 0xFFFF : 0 - int16x8_t zeroMask = vreinterpretq_s16_u16(vceqq_s16(b, zero)); - // -a - int16x8_t neg = vnegq_s16(a); - // bitwise select either a or neg based on ltMask - int16x8_t masked = vbslq_s16(ltMask, a, neg); +#if defined(__aarch64__) + int16x8_t zeroMask = vreinterpretq_s16_u16(vceqzq_s16(b)); +#else + int16x8_t zeroMask = vreinterpretq_s16_u16(vceqq_s16(b, vdupq_n_s16(0))); +#endif + + // bitwise select either a or negative 'a' (vnegq_s16(a) equals to negative + // 'a') based on ltMask + int16x8_t masked = vbslq_s16(ltMask, vnegq_s16(a), a); // res = masked & (~zeroMask) int16x8_t res = vbicq_s16(masked, zeroMask); return vreinterpretq_m128i_s16(res); @@ -2107,21 +3113,248 @@ FORCE_INLINE __m128i _mm_sign_epi32(__m128i _a, __m128i _b) int32x4_t a = vreinterpretq_s32_m128i(_a); int32x4_t b = vreinterpretq_s32_m128i(_b); - int32x4_t zero = vdupq_n_s32(0); // signed shift right: faster than vclt // (b < 0) ? 0xFFFFFFFF : 0 uint32x4_t ltMask = vreinterpretq_u32_s32(vshrq_n_s32(b, 31)); + // (b == 0) ? 0xFFFFFFFF : 0 - int32x4_t zeroMask = vreinterpretq_s32_u32(vceqq_s32(b, zero)); - // neg = -a - int32x4_t neg = vnegq_s32(a); - // bitwise select either a or neg based on ltMask - int32x4_t masked = vbslq_s32(ltMask, a, neg); +#if defined(__aarch64__) + int32x4_t zeroMask = vreinterpretq_s32_u32(vceqzq_s32(b)); +#else + int32x4_t zeroMask = vreinterpretq_s32_u32(vceqq_s32(b, vdupq_n_s32(0))); +#endif + + // bitwise select either a or negative 'a' (vnegq_s32(a) equals to negative + // 'a') based on ltMask + int32x4_t masked = vbslq_s32(ltMask, vnegq_s32(a), a); // res = masked & (~zeroMask) int32x4_t res = vbicq_s32(masked, zeroMask); return vreinterpretq_m128i_s32(res); } +// Negate packed 16-bit integers in a when the corresponding signed 16-bit +// integer in b is negative, and store the results in dst. Element in dst are +// zeroed out when the corresponding element in b is zero. +// +// FOR j := 0 to 3 +// i := j*16 +// IF b[i+15:i] < 0 +// dst[i+15:i] := -(a[i+15:i]) +// ELSE IF b[i+15:i] == 0 +// dst[i+15:i] := 0 +// ELSE +// dst[i+15:i] := a[i+15:i] +// FI +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sign_pi16 +FORCE_INLINE __m64 _mm_sign_pi16(__m64 _a, __m64 _b) +{ + int16x4_t a = vreinterpret_s16_m64(_a); + int16x4_t b = vreinterpret_s16_m64(_b); + + // signed shift right: faster than vclt + // (b < 0) ? 0xFFFF : 0 + uint16x4_t ltMask = vreinterpret_u16_s16(vshr_n_s16(b, 15)); + + // (b == 0) ? 0xFFFF : 0 +#if defined(__aarch64__) + int16x4_t zeroMask = vreinterpret_s16_u16(vceqz_s16(b)); +#else + int16x4_t zeroMask = vreinterpret_s16_u16(vceq_s16(b, vdup_n_s16(0))); +#endif + + // bitwise select either a or nagative 'a' (vneg_s16(a) return nagative 'a') + // based on ltMask + int16x4_t masked = vbsl_s16(ltMask, vneg_s16(a), a); + // res = masked & (~zeroMask) + int16x4_t res = vbic_s16(masked, zeroMask); + + return vreinterpret_m64_s16(res); +} + +// Negate packed 32-bit integers in a when the corresponding signed 32-bit +// integer in b is negative, and store the results in dst. Element in dst are +// zeroed out when the corresponding element in b is zero. +// +// FOR j := 0 to 1 +// i := j*32 +// IF b[i+31:i] < 0 +// dst[i+31:i] := -(a[i+31:i]) +// ELSE IF b[i+31:i] == 0 +// dst[i+31:i] := 0 +// ELSE +// dst[i+31:i] := a[i+31:i] +// FI +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sign_pi32 +FORCE_INLINE __m64 _mm_sign_pi32(__m64 _a, __m64 _b) +{ + int32x2_t a = vreinterpret_s32_m64(_a); + int32x2_t b = vreinterpret_s32_m64(_b); + + // signed shift right: faster than vclt + // (b < 0) ? 0xFFFFFFFF : 0 + uint32x2_t ltMask = vreinterpret_u32_s32(vshr_n_s32(b, 31)); + + // (b == 0) ? 0xFFFFFFFF : 0 +#if defined(__aarch64__) + int32x2_t zeroMask = vreinterpret_s32_u32(vceqz_s32(b)); +#else + int32x2_t zeroMask = vreinterpret_s32_u32(vceq_s32(b, vdup_n_s32(0))); +#endif + + // bitwise select either a or nagative 'a' (vneg_s32(a) return nagative 'a') + // based on ltMask + int32x2_t masked = vbsl_s32(ltMask, vneg_s32(a), a); + // res = masked & (~zeroMask) + int32x2_t res = vbic_s32(masked, zeroMask); + + return vreinterpret_m64_s32(res); +} + +// Negate packed 8-bit integers in a when the corresponding signed 8-bit integer +// in b is negative, and store the results in dst. Element in dst are zeroed out +// when the corresponding element in b is zero. +// +// FOR j := 0 to 7 +// i := j*8 +// IF b[i+7:i] < 0 +// dst[i+7:i] := -(a[i+7:i]) +// ELSE IF b[i+7:i] == 0 +// dst[i+7:i] := 0 +// ELSE +// dst[i+7:i] := a[i+7:i] +// FI +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sign_pi8 +FORCE_INLINE __m64 _mm_sign_pi8(__m64 _a, __m64 _b) +{ + int8x8_t a = vreinterpret_s8_m64(_a); + int8x8_t b = vreinterpret_s8_m64(_b); + + // signed shift right: faster than vclt + // (b < 0) ? 0xFF : 0 + uint8x8_t ltMask = vreinterpret_u8_s8(vshr_n_s8(b, 7)); + + // (b == 0) ? 0xFF : 0 +#if defined(__aarch64__) + int8x8_t zeroMask = vreinterpret_s8_u8(vceqz_s8(b)); +#else + int8x8_t zeroMask = vreinterpret_s8_u8(vceq_s8(b, vdup_n_s8(0))); +#endif + + // bitwise select either a or nagative 'a' (vneg_s8(a) return nagative 'a') + // based on ltMask + int8x8_t masked = vbsl_s8(ltMask, vneg_s8(a), a); + // res = masked & (~zeroMask) + int8x8_t res = vbic_s8(masked, zeroMask); + + return vreinterpret_m64_s8(res); +} + +// Average packed unsigned 16-bit integers in a and b, and store the results in +// dst. +// +// FOR j := 0 to 3 +// i := j*16 +// dst[i+15:i] := (a[i+15:i] + b[i+15:i] + 1) >> 1 +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_avg_pu16 +FORCE_INLINE __m64 _mm_avg_pu16(__m64 a, __m64 b) +{ + return vreinterpret_m64_u16( + vrhadd_u16(vreinterpret_u16_m64(a), vreinterpret_u16_m64(b))); +} + +// Average packed unsigned 8-bit integers in a and b, and store the results in +// dst. +// +// FOR j := 0 to 7 +// i := j*8 +// dst[i+7:i] := (a[i+7:i] + b[i+7:i] + 1) >> 1 +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_avg_pu8 +FORCE_INLINE __m64 _mm_avg_pu8(__m64 a, __m64 b) +{ + return vreinterpret_m64_u8( + vrhadd_u8(vreinterpret_u8_m64(a), vreinterpret_u8_m64(b))); +} + +// Average packed unsigned 8-bit integers in a and b, and store the results in +// dst. +// +// FOR j := 0 to 7 +// i := j*8 +// dst[i+7:i] := (a[i+7:i] + b[i+7:i] + 1) >> 1 +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_m_pavgb +#define _m_pavgb(a, b) _mm_avg_pu8(a, b) + +// Average packed unsigned 16-bit integers in a and b, and store the results in +// dst. +// +// FOR j := 0 to 3 +// i := j*16 +// dst[i+15:i] := (a[i+15:i] + b[i+15:i] + 1) >> 1 +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_m_pavgw +#define _m_pavgw(a, b) _mm_avg_pu16(a, b) + +// Extract a 16-bit integer from a, selected with imm8, and store the result in +// the lower element of dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_m_pextrw +#define _m_pextrw(a, imm) _mm_extract_pi16(a, imm) + +// Copy a to dst, and insert the 16-bit integer i into dst at the location +// specified by imm8. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=m_pinsrw +#define _m_pinsrw(a, i, imm) _mm_insert_pi16(a, i, imm) + +// Compare packed signed 16-bit integers in a and b, and store packed maximum +// values in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_m_pmaxsw +#define _m_pmaxsw(a, b) _mm_max_pi16(a, b) + +// Compare packed unsigned 8-bit integers in a and b, and store packed maximum +// values in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_m_pmaxub +#define _m_pmaxub(a, b) _mm_max_pu8(a, b) + +// Compare packed signed 16-bit integers in a and b, and store packed minimum +// values in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_m_pminsw +#define _m_pminsw(a, b) _mm_min_pi16(a, b) + +// Compare packed unsigned 8-bit integers in a and b, and store packed minimum +// values in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_m_pminub +#define _m_pminub(a, b) _mm_min_pu8(a, b) + +// Create mask from the most significant bit of each 8-bit element in a, and +// store the result in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_m_pmovmskb +#define _m_pmovmskb(a) _mm_movemask_pi8(a) + +// Multiply the packed unsigned 16-bit integers in a and b, producing +// intermediate 32-bit integers, and store the high 16 bits of the intermediate +// integers in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_m_pmulhuw +#define _m_pmulhuw(a, b) _mm_mulhi_pu16(a, b) + +// Compute the absolute differences of packed unsigned 8-bit integers in a and +// b, then horizontally sum each consecutive 8 differences to produce four +// unsigned 16-bit integers, and pack these unsigned 16-bit integers in the low +// 16 bits of dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=m_psadbw +#define _m_psadbw(a, b) _mm_sad_pu8(a, b) + // Computes the average of the 16 unsigned 8-bit integers in a and the 16 // unsigned 8-bit integers in b and rounds. // @@ -2137,6 +3370,16 @@ FORCE_INLINE __m128i _mm_avg_epu8(__m128i a, __m128i b) vrhaddq_u8(vreinterpretq_u8_m128i(a), vreinterpretq_u8_m128i(b))); } +// Shift a left by imm8 bytes while shifting in zeros, and store the results in +// dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_bslli_si128 +#define _mm_bslli_si128(a, imm) _mm_slli_si128(a, imm) + +// Shift a right by imm8 bytes while shifting in zeros, and store the results in +// dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_bsrli_si128 +#define _mm_bsrli_si128(a, imm) _mm_srli_si128(a, imm) + // Computes the average of the 8 unsigned 16-bit integers in a and the 8 // unsigned 16-bit integers in b and rounds. // @@ -2166,6 +3409,57 @@ FORCE_INLINE __m128 _mm_add_ps(__m128 a, __m128 b) vaddq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); } +// Add packed double-precision (64-bit) floating-point elements in a and b, and +// store the results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_add_pd +FORCE_INLINE __m128d _mm_add_pd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) + return vreinterpretq_m128d_f64( + vaddq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#else + double *da = (double *) &a; + double *db = (double *) &b; + double c[2]; + c[0] = da[0] + db[0]; + c[1] = da[1] + db[1]; + return vld1q_f32((float32_t *) c); +#endif +} + +// Add the lower double-precision (64-bit) floating-point element in a and b, +// store the result in the lower element of dst, and copy the upper element from +// a to the upper element of dst. +// +// dst[63:0] := a[63:0] + b[63:0] +// dst[127:64] := a[127:64] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_add_sd +FORCE_INLINE __m128d _mm_add_sd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) + return _mm_move_sd(a, _mm_add_pd(a, b)); +#else + double *da = (double *) &a; + double *db = (double *) &b; + double c[2]; + c[0] = da[0] + db[0]; + c[1] = da[1]; + return vld1q_f32((float32_t *) c); +#endif +} + +// Add 64-bit integers a and b, and store the result in dst. +// +// dst[63:0] := a[63:0] + b[63:0] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_add_si64 +FORCE_INLINE __m64 _mm_add_si64(__m64 a, __m64 b) +{ + return vreinterpret_m64_s64( + vadd_s64(vreinterpret_s64_m64(a), vreinterpret_s64_m64(b))); +} + // adds the scalar single-precision floating point values of a and b. // https://msdn.microsoft.com/en-us/library/be94x2y6(v=vs.100).aspx FORCE_INLINE __m128 _mm_add_ss(__m128 a, __m128 b) @@ -2233,6 +3527,21 @@ FORCE_INLINE __m128i _mm_adds_epi16(__m128i a, __m128i b) vqaddq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); } +// Add packed signed 8-bit integers in a and b using saturation, and store the +// results in dst. +// +// FOR j := 0 to 15 +// i := j*8 +// dst[i+7:i] := Saturate8( a[i+7:i] + b[i+7:i] ) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_adds_epi8 +FORCE_INLINE __m128i _mm_adds_epi8(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_s8( + vqaddq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); +} + // Adds the 16 unsigned 8-bit integers in a to the 16 unsigned 8-bit integers in // b and saturates.. // https://msdn.microsoft.com/en-us/library/9hahyddy(v=vs.100).aspx @@ -2266,6 +3575,19 @@ FORCE_INLINE __m128i _mm_mullo_epi32(__m128i a, __m128i b) vmulq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); } +// Multiply the packed unsigned 16-bit integers in a and b, producing +// intermediate 32-bit integers, and store the high 16 bits of the intermediate +// integers in dst. +// +// FOR j := 0 to 3 +// i := j*16 +// tmp[31:0] := a[i+15:i] * b[i+15:i] +// dst[i+15:i] := tmp[31:16] +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_m_pmulhuw +#define _m_pmulhuw(a, b) _mm_mulhi_pu16(a, b) + // Multiplies the four single-precision, floating-point values of a and b. // // r0 := a0 * b0 @@ -2280,6 +3602,46 @@ FORCE_INLINE __m128 _mm_mul_ps(__m128 a, __m128 b) vmulq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); } +// Multiply packed double-precision (64-bit) floating-point elements in a and b, +// and store the results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mul_pd +FORCE_INLINE __m128d _mm_mul_pd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) + return vreinterpretq_m128d_f64( + vmulq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#else + double *da = (double *) &a; + double *db = (double *) &b; + double c[2]; + c[0] = da[0] * db[0]; + c[1] = da[1] * db[1]; + return vld1q_f32((float32_t *) c); +#endif +} + +// Multiply the lower double-precision (64-bit) floating-point element in a and +// b, store the result in the lower element of dst, and copy the upper element +// from a to the upper element of dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mul_sd +FORCE_INLINE __m128d _mm_mul_sd(__m128d a, __m128d b) +{ + return _mm_move_sd(a, _mm_mul_pd(a, b)); +} + +// Multiply the lower single-precision (32-bit) floating-point element in a and +// b, store the result in the lower element of dst, and copy the upper 3 packed +// elements from a to the upper elements of dst. +// +// dst[31:0] := a[31:0] * b[31:0] +// dst[127:32] := a[127:32] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mul_ss +FORCE_INLINE __m128 _mm_mul_ss(__m128 a, __m128 b) +{ + return _mm_move_ss(a, _mm_mul_ps(a, b)); +} + // Multiply the low unsigned 32-bit integers from each packed 64-bit element in // a and b, and store the unsigned 64-bit results in dst. // @@ -2293,6 +3655,18 @@ FORCE_INLINE __m128i _mm_mul_epu32(__m128i a, __m128i b) return vreinterpretq_m128i_u64(vmull_u32(a_lo, b_lo)); } +// Multiply the low unsigned 32-bit integers from a and b, and store the +// unsigned 64-bit result in dst. +// +// dst[63:0] := a[31:0] * b[31:0] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mul_su32 +FORCE_INLINE __m64 _mm_mul_su32(__m64 a, __m64 b) +{ + return vreinterpret_m64_u64(vget_low_u64( + vmull_u32(vreinterpret_u32_m64(a), vreinterpret_u32_m64(b)))); +} + // Multiply the low signed 32-bit integers from each packed 64-bit element in // a and b, and store the signed 64-bit results in dst. // @@ -2327,6 +3701,21 @@ FORCE_INLINE __m128i _mm_madd_epi16(__m128i a, __m128i b) return vreinterpretq_m128i_s32(vcombine_s32(low_sum, high_sum)); } +// Conditionally store 8-bit integer elements from a into memory using mask +// (elements are not stored when the highest bit is not set in the corresponding +// element) and a non-temporal memory hint. mem_addr does not need to be aligned +// on any particular boundary. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskmoveu_si128 +FORCE_INLINE void _mm_maskmoveu_si128(__m128i a, __m128i mask, char *mem_addr) +{ + int8x16_t shr_mask = vshrq_n_s8(vreinterpretq_s8_m128i(mask), 7); + __m128 b = _mm_load_ps((const float *) mem_addr); + int8x16_t masked = + vbslq_s8(vreinterpretq_u8_s8(shr_mask), vreinterpretq_s8_m128i(a), + vreinterpretq_s8_m128(b)); + vst1q_s8((int8_t *) mem_addr, masked); +} + // Multiply packed signed 16-bit integers in a and b, producing intermediate // signed 32-bit integers. Shift right by 15 bits while rounding up, and store // the packed 16-bit integers in dst. @@ -2368,6 +3757,16 @@ FORCE_INLINE __m128i _mm_mulhrs_epi16(__m128i a, __m128i b) // ENDFOR FORCE_INLINE __m128i _mm_maddubs_epi16(__m128i _a, __m128i _b) { +#if defined(__aarch64__) + uint8x16_t a = vreinterpretq_u8_m128i(_a); + int8x16_t b = vreinterpretq_s8_m128i(_b); + int16x8_t tl = vmulq_s16(vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(a))), + vmovl_s8(vget_low_s8(b))); + int16x8_t th = vmulq_s16(vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(a))), + vmovl_s8(vget_high_s8(b))); + return vreinterpretq_m128i_s16( + vqaddq_s16(vuzp1q_s16(tl, th), vuzp2q_s16(tl, th))); +#else // This would be much simpler if x86 would choose to zero extend OR sign // extend, not both. This could probably be optimized better. uint16x8_t a = vreinterpretq_u16_m128i(_a); @@ -2387,24 +3786,56 @@ FORCE_INLINE __m128i _mm_maddubs_epi16(__m128i _a, __m128i _b) // saturated add return vreinterpretq_m128i_s16(vqaddq_s16(prod1, prod2)); +#endif } -// Computes the absolute difference of the 16 unsigned 8-bit integers from a -// and the 16 unsigned 8-bit integers from b. +// Computes the fused multiple add product of 32-bit floating point numbers. // // Return Value -// Sums the upper 8 differences and lower 8 differences and packs the -// resulting 2 unsigned 16-bit integers into the upper and lower 64-bit -// elements. -// -// r0 := abs(a0 - b0) + abs(a1 - b1) +...+ abs(a7 - b7) -// r1 := 0x0 -// r2 := 0x0 -// r3 := 0x0 -// r4 := abs(a8 - b8) + abs(a9 - b9) +...+ abs(a15 - b15) -// r5 := 0x0 -// r6 := 0x0 -// r7 := 0x0 +// Multiplies A and B, and adds C to the temporary result before returning it. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_fmadd +FORCE_INLINE __m128 _mm_fmadd_ps(__m128 a, __m128 b, __m128 c) +{ +#if defined(__aarch64__) + return vreinterpretq_m128_f32(vfmaq_f32(vreinterpretq_f32_m128(c), + vreinterpretq_f32_m128(b), + vreinterpretq_f32_m128(a))); +#else + return _mm_add_ps(_mm_mul_ps(a, b), c); +#endif +} + +// Alternatively add and subtract packed single-precision (32-bit) +// floating-point elements in a to/from packed elements in b, and store the +// results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=addsub_ps +FORCE_INLINE __m128 _mm_addsub_ps(__m128 a, __m128 b) +{ + __m128 mask = {-1.0f, 1.0f, -1.0f, 1.0f}; + return _mm_fmadd_ps(b, mask, a); +} + +// Horizontally add adjacent pairs of double-precision (64-bit) floating-point +// elements in a and b, and pack the results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_hadd_pd +FORCE_INLINE __m128d _mm_hadd_pd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) + return vreinterpretq_m128d_f64( + vpaddq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#else + double *da = (double *) &a; + double *db = (double *) &b; + double c[] = {da[0] + da[1], db[0] + db[1]}; + return vreinterpretq_m128d_u64(vld1q_u64((uint64_t *) c)); +#endif +} + +// Compute the absolute differences of packed unsigned 8-bit integers in a and +// b, then horizontally sum each consecutive 8 differences to produce two +// unsigned 16-bit integers, and pack these unsigned 16-bit integers in the low +// 16 bits of 64-bit elements in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sad_epu8 FORCE_INLINE __m128i _mm_sad_epu8(__m128i a, __m128i b) { uint16x8_t t = vpaddlq_u8(vabdq_u8((uint8x16_t) a, (uint8x16_t) b)); @@ -2414,6 +3845,34 @@ FORCE_INLINE __m128i _mm_sad_epu8(__m128i a, __m128i b) return (__m128i) vsetq_lane_u16(r4, r, 4); } +// Compute the absolute differences of packed unsigned 8-bit integers in a and +// b, then horizontally sum each consecutive 8 differences to produce four +// unsigned 16-bit integers, and pack these unsigned 16-bit integers in the low +// 16 bits of dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sad_pu8 +FORCE_INLINE __m64 _mm_sad_pu8(__m64 a, __m64 b) +{ + uint16x4_t t = + vpaddl_u8(vabd_u8(vreinterpret_u8_m64(a), vreinterpret_u8_m64(b))); + uint16_t r0 = t[0] + t[1] + t[2] + t[3]; + return vreinterpret_m64_u16(vset_lane_u16(r0, vdup_n_u16(0), 0)); +} + +// Compute the absolute differences of packed unsigned 8-bit integers in a and +// b, then horizontally sum each consecutive 8 differences to produce four +// unsigned 16-bit integers, and pack these unsigned 16-bit integers in the low +// 16 bits of dst. +// +// FOR j := 0 to 7 +// i := j*8 +// tmp[i+7:i] := ABS(a[i+7:i] - b[i+7:i]) +// ENDFOR +// dst[15:0] := tmp[7:0] + tmp[15:8] + tmp[23:16] + tmp[31:24] + tmp[39:32] + +// tmp[47:40] + tmp[55:48] + tmp[63:56] dst[63:16] := 0 +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_m_psadbw +#define _m_psadbw(a, b) _mm_sad_pu8(a, b) + // Divides the four single-precision, floating-point values of a and b. // // r0 := a0 / b0 @@ -2424,10 +3883,18 @@ FORCE_INLINE __m128i _mm_sad_epu8(__m128i a, __m128i b) // https://msdn.microsoft.com/en-us/library/edaw8147(v=vs.100).aspx FORCE_INLINE __m128 _mm_div_ps(__m128 a, __m128 b) { - float32x4_t recip0 = vrecpeq_f32(vreinterpretq_f32_m128(b)); - float32x4_t recip1 = - vmulq_f32(recip0, vrecpsq_f32(recip0, vreinterpretq_f32_m128(b))); - return vreinterpretq_m128_f32(vmulq_f32(vreinterpretq_f32_m128(a), recip1)); +#if defined(__aarch64__) && !SSE2NEON_PRECISE_DIV + return vreinterpretq_m128_f32( + vdivq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); +#else + float32x4_t recip = vrecpeq_f32(vreinterpretq_f32_m128(b)); + recip = vmulq_f32(recip, vrecpsq_f32(recip, vreinterpretq_f32_m128(b))); +#if SSE2NEON_PRECISE_DIV + // Additional Netwon-Raphson iteration for accuracy + recip = vmulq_f32(recip, vrecpsq_f32(recip, vreinterpretq_f32_m128(b))); +#endif + return vreinterpretq_m128_f32(vmulq_f32(vreinterpretq_f32_m128(a), recip)); +#endif } // Divides the scalar single-precision floating point value of a by b. @@ -2440,16 +3907,76 @@ FORCE_INLINE __m128 _mm_div_ss(__m128 a, __m128 b) vsetq_lane_f32(value, vreinterpretq_f32_m128(a), 0)); } -// Computes the approximations of reciprocals of the four single-precision, -// floating-point values of a. -// https://msdn.microsoft.com/en-us/library/vstudio/796k1tty(v=vs.100).aspx +// Divide packed double-precision (64-bit) floating-point elements in a by +// packed elements in b, and store the results in dst. +// +// FOR j := 0 to 1 +// i := 64*j +// dst[i+63:i] := a[i+63:i] / b[i+63:i] +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_div_pd +FORCE_INLINE __m128d _mm_div_pd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) + return vreinterpretq_m128d_f64( + vdivq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#else + double *da = (double *) &a; + double *db = (double *) &b; + double c[2]; + c[0] = da[0] / db[0]; + c[1] = da[1] / db[1]; + return vld1q_f32((float32_t *) c); +#endif +} + +// Divide the lower double-precision (64-bit) floating-point element in a by the +// lower double-precision (64-bit) floating-point element in b, store the result +// in the lower element of dst, and copy the upper element from a to the upper +// element of dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_div_sd +FORCE_INLINE __m128d _mm_div_sd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) + float64x2_t tmp = + vdivq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b)); + return vreinterpretq_m128d_f64( + vsetq_lane_f64(vgetq_lane_f64(vreinterpretq_f64_m128d(a), 1), tmp, 1)); +#else + return _mm_move_sd(a, _mm_div_pd(a, b)); +#endif +} + +// Compute the approximate reciprocal of packed single-precision (32-bit) +// floating-point elements in a, and store the results in dst. The maximum +// relative error for this approximation is less than 1.5*2^-12. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_rcp_ps FORCE_INLINE __m128 _mm_rcp_ps(__m128 in) { float32x4_t recip = vrecpeq_f32(vreinterpretq_f32_m128(in)); recip = vmulq_f32(recip, vrecpsq_f32(recip, vreinterpretq_f32_m128(in))); +#if SSE2NEON_PRECISE_DIV + // Additional Netwon-Raphson iteration for accuracy + recip = vmulq_f32(recip, vrecpsq_f32(recip, vreinterpretq_f32_m128(in))); +#endif return vreinterpretq_m128_f32(recip); } +// Compute the approximate reciprocal of the lower single-precision (32-bit) +// floating-point element in a, store the result in the lower element of dst, +// and copy the upper 3 packed elements from a to the upper elements of dst. The +// maximum relative error for this approximation is less than 1.5*2^-12. +// +// dst[31:0] := (1.0 / a[31:0]) +// dst[127:32] := a[127:32] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_rcp_ss +FORCE_INLINE __m128 _mm_rcp_ss(__m128 a) +{ + return _mm_move_ss(a, _mm_rcp_ps(a)); +} + // Computes the approximations of square roots of the four single-precision, // floating-point values of a. First computes reciprocal square roots and then // reciprocals of the four values. @@ -2462,10 +3989,34 @@ FORCE_INLINE __m128 _mm_rcp_ps(__m128 in) // https://msdn.microsoft.com/en-us/library/vstudio/8z67bwwk(v=vs.100).aspx FORCE_INLINE __m128 _mm_sqrt_ps(__m128 in) { +#if SSE2NEON_PRECISE_SQRT + float32x4_t recip = vrsqrteq_f32(vreinterpretq_f32_m128(in)); + + // Test for vrsqrteq_f32(0) -> positive infinity case. + // Change to zero, so that s * 1/sqrt(s) result is zero too. + const uint32x4_t pos_inf = vdupq_n_u32(0x7F800000); + const uint32x4_t div_by_zero = + vceqq_u32(pos_inf, vreinterpretq_u32_f32(recip)); + recip = vreinterpretq_f32_u32( + vandq_u32(vmvnq_u32(div_by_zero), vreinterpretq_u32_f32(recip))); + + // Additional Netwon-Raphson iteration for accuracy + recip = vmulq_f32( + vrsqrtsq_f32(vmulq_f32(recip, recip), vreinterpretq_f32_m128(in)), + recip); + recip = vmulq_f32( + vrsqrtsq_f32(vmulq_f32(recip, recip), vreinterpretq_f32_m128(in)), + recip); + + // sqrt(s) = s * 1/sqrt(s) + return vreinterpretq_m128_f32(vmulq_f32(vreinterpretq_f32_m128(in), recip)); +#elif defined(__aarch64__) + return vreinterpretq_m128_f32(vsqrtq_f32(vreinterpretq_f32_m128(in))); +#else float32x4_t recipsq = vrsqrteq_f32(vreinterpretq_f32_m128(in)); float32x4_t sq = vrecpeq_f32(recipsq); - // ??? use step versions of both sqrt and recip for better accuracy? return vreinterpretq_m128_f32(sq); +#endif } // Computes the approximation of the square root of the scalar single-precision @@ -2484,34 +4035,167 @@ FORCE_INLINE __m128 _mm_sqrt_ss(__m128 in) // https://msdn.microsoft.com/en-us/library/22hfsh53(v=vs.100).aspx FORCE_INLINE __m128 _mm_rsqrt_ps(__m128 in) { - return vreinterpretq_m128_f32(vrsqrteq_f32(vreinterpretq_f32_m128(in))); + float32x4_t out = vrsqrteq_f32(vreinterpretq_f32_m128(in)); +#if SSE2NEON_PRECISE_SQRT + // Additional Netwon-Raphson iteration for accuracy + out = vmulq_f32( + out, vrsqrtsq_f32(vmulq_f32(vreinterpretq_f32_m128(in), out), out)); + out = vmulq_f32( + out, vrsqrtsq_f32(vmulq_f32(vreinterpretq_f32_m128(in), out), out)); +#endif + return vreinterpretq_m128_f32(out); } +// Compute the approximate reciprocal square root of the lower single-precision +// (32-bit) floating-point element in a, store the result in the lower element +// of dst, and copy the upper 3 packed elements from a to the upper elements of +// dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_rsqrt_ss +FORCE_INLINE __m128 _mm_rsqrt_ss(__m128 in) +{ + return vsetq_lane_f32(vgetq_lane_f32(_mm_rsqrt_ps(in), 0), in, 0); +} + +// Compare packed signed 16-bit integers in a and b, and store packed maximum +// values in dst. +// +// FOR j := 0 to 3 +// i := j*16 +// dst[i+15:i] := MAX(a[i+15:i], b[i+15:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_max_pi16 +FORCE_INLINE __m64 _mm_max_pi16(__m64 a, __m64 b) +{ + return vreinterpret_m64_s16( + vmax_s16(vreinterpret_s16_m64(a), vreinterpret_s16_m64(b))); +} + +// Compare packed signed 16-bit integers in a and b, and store packed maximum +// values in dst. +// +// FOR j := 0 to 3 +// i := j*16 +// dst[i+15:i] := MAX(a[i+15:i], b[i+15:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_max_pi16 +#define _m_pmaxsw(a, b) _mm_max_pi16(a, b) + // Computes the maximums of the four single-precision, floating-point values of // a and b. // https://msdn.microsoft.com/en-us/library/vstudio/ff5d607a(v=vs.100).aspx FORCE_INLINE __m128 _mm_max_ps(__m128 a, __m128 b) { +#if SSE2NEON_PRECISE_MINMAX + float32x4_t _a = vreinterpretq_f32_m128(a); + float32x4_t _b = vreinterpretq_f32_m128(b); + return vbslq_f32(vcltq_f32(_b, _a), _a, _b); +#else return vreinterpretq_m128_f32( vmaxq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); +#endif } +// Compare packed unsigned 8-bit integers in a and b, and store packed maximum +// values in dst. +// +// FOR j := 0 to 7 +// i := j*8 +// dst[i+7:i] := MAX(a[i+7:i], b[i+7:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_max_pu8 +FORCE_INLINE __m64 _mm_max_pu8(__m64 a, __m64 b) +{ + return vreinterpret_m64_u8( + vmax_u8(vreinterpret_u8_m64(a), vreinterpret_u8_m64(b))); +} + +// Compare packed unsigned 8-bit integers in a and b, and store packed maximum +// values in dst. +// +// FOR j := 0 to 7 +// i := j*8 +// dst[i+7:i] := MAX(a[i+7:i], b[i+7:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_max_pu8 +#define _m_pmaxub(a, b) _mm_max_pu8(a, b) + +// Compare packed signed 16-bit integers in a and b, and store packed minimum +// values in dst. +// +// FOR j := 0 to 3 +// i := j*16 +// dst[i+15:i] := MIN(a[i+15:i], b[i+15:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_min_pi16 +FORCE_INLINE __m64 _mm_min_pi16(__m64 a, __m64 b) +{ + return vreinterpret_m64_s16( + vmin_s16(vreinterpret_s16_m64(a), vreinterpret_s16_m64(b))); +} + +// Compare packed signed 16-bit integers in a and b, and store packed minimum +// values in dst. +// +// FOR j := 0 to 3 +// i := j*16 +// dst[i+15:i] := MIN(a[i+15:i], b[i+15:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_min_pi16 +#define _m_pminsw(a, b) _mm_min_pi16(a, b) + // Computes the minima of the four single-precision, floating-point values of a // and b. // https://msdn.microsoft.com/en-us/library/vstudio/wh13kadz(v=vs.100).aspx FORCE_INLINE __m128 _mm_min_ps(__m128 a, __m128 b) { +#if SSE2NEON_PRECISE_MINMAX + float32x4_t _a = vreinterpretq_f32_m128(a); + float32x4_t _b = vreinterpretq_f32_m128(b); + return vbslq_f32(vcltq_f32(_a, _b), _a, _b); +#else return vreinterpretq_m128_f32( vminq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); +#endif } +// Compare packed unsigned 8-bit integers in a and b, and store packed minimum +// values in dst. +// +// FOR j := 0 to 7 +// i := j*8 +// dst[i+7:i] := MIN(a[i+7:i], b[i+7:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_min_pu8 +FORCE_INLINE __m64 _mm_min_pu8(__m64 a, __m64 b) +{ + return vreinterpret_m64_u8( + vmin_u8(vreinterpret_u8_m64(a), vreinterpret_u8_m64(b))); +} + +// Compare packed unsigned 8-bit integers in a and b, and store packed minimum +// values in dst. +// +// FOR j := 0 to 7 +// i := j*8 +// dst[i+7:i] := MIN(a[i+7:i], b[i+7:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_min_pu8 +#define _m_pminub(a, b) _mm_min_pu8(a, b) + // Computes the maximum of the two lower scalar single-precision floating point // values of a and b. // https://msdn.microsoft.com/en-us/library/s6db5esz(v=vs.100).aspx FORCE_INLINE __m128 _mm_max_ss(__m128 a, __m128 b) { - float32_t value = vgetq_lane_f32( - vmaxq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)), 0); + float32_t value = vgetq_lane_f32(_mm_max_ps(a, b), 0); return vreinterpretq_m128_f32( vsetq_lane_f32(value, vreinterpretq_f32_m128(a), 0)); } @@ -2521,8 +4205,7 @@ FORCE_INLINE __m128 _mm_max_ss(__m128 a, __m128 b) // https://msdn.microsoft.com/en-us/library/0a9y7xaa(v=vs.100).aspx FORCE_INLINE __m128 _mm_min_ss(__m128 a, __m128 b) { - float32_t value = vgetq_lane_f32( - vminq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)), 0); + float32_t value = vgetq_lane_f32(_mm_min_ps(a, b), 0); return vreinterpretq_m128_f32( vsetq_lane_f32(value, vreinterpretq_f32_m128(a), 0)); } @@ -2536,6 +4219,27 @@ FORCE_INLINE __m128i _mm_max_epu8(__m128i a, __m128i b) vmaxq_u8(vreinterpretq_u8_m128i(a), vreinterpretq_u8_m128i(b))); } +// Compare packed double-precision (64-bit) floating-point elements in a and b, +// and store packed maximum values in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_max_pd +FORCE_INLINE __m128d _mm_max_pd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) + return vreinterpretq_m128d_f64( + vmaxq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#else + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t b1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(b)); + uint64_t d[2]; + d[0] = (*(double *) &a0) > (*(double *) &b0) ? a0 : b0; + d[1] = (*(double *) &a1) > (*(double *) &b1) ? a1 : b1; + + return vreinterpretq_m128d_u64(vld1q_u64(d)); +#endif +} + // Computes the pairwise minima of the 16 unsigned 8-bit integers from a and the // 16 unsigned 8-bit integers from b. // https://msdn.microsoft.com/ko-kr/library/17k8cf58(v=vs.100).aspxx @@ -2554,6 +4258,42 @@ FORCE_INLINE __m128i _mm_min_epi16(__m128i a, __m128i b) vminq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); } +// Compare packed signed 8-bit integers in a and b, and store packed maximum +// values in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_max_epi8 +FORCE_INLINE __m128i _mm_max_epi8(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_s8( + vmaxq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); +} + +// Compare packed unsigned 16-bit integers in a and b, and store packed maximum +// values in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_max_epu16 +FORCE_INLINE __m128i _mm_max_epu16(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_u16( + vmaxq_u16(vreinterpretq_u16_m128i(a), vreinterpretq_u16_m128i(b))); +} + +// Compare packed signed 8-bit integers in a and b, and store packed minimum +// values in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_min_epi8 +FORCE_INLINE __m128i _mm_min_epi8(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_s8( + vminq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); +} + +// Compare packed unsigned 16-bit integers in a and b, and store packed minimum +// values in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_min_epu16 +FORCE_INLINE __m128i _mm_min_epu16(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_u16( + vminq_u16(vreinterpretq_u16_m128i(a), vreinterpretq_u16_m128i(b))); +} + // Computes the pairwise maxima of the 8 signed 16-bit integers from a and the 8 // signed 16-bit integers from b. // https://msdn.microsoft.com/en-us/LIBRary/3x060h7c(v=vs.100).aspx @@ -2596,6 +4336,34 @@ FORCE_INLINE __m128i _mm_min_epi32(__m128i a, __m128i b) vminq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); } +// Compare packed unsigned 32-bit integers in a and b, and store packed maximum +// values in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_max_epu32 +FORCE_INLINE __m128i _mm_max_epu32(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_u32( + vmaxq_u32(vreinterpretq_u32_m128i(a), vreinterpretq_u32_m128i(b))); +} + +// Compare packed unsigned 32-bit integers in a and b, and store packed minimum +// values in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_max_epu32 +FORCE_INLINE __m128i _mm_min_epu32(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_u32( + vminq_u32(vreinterpretq_u32_m128i(a), vreinterpretq_u32_m128i(b))); +} + +// Multiply the packed unsigned 16-bit integers in a and b, producing +// intermediate 32-bit integers, and store the high 16 bits of the intermediate +// integers in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mulhi_pu16 +FORCE_INLINE __m64 _mm_mulhi_pu16(__m64 a, __m64 b) +{ + return vreinterpret_m64_u16(vshrn_n_u32( + vmull_u16(vreinterpret_u16_m64(a), vreinterpret_u16_m64(b)), 16)); +} + // Multiplies the 8 signed 16-bit integers from a by the 8 signed 16-bit // integers from b. // @@ -2622,6 +4390,31 @@ FORCE_INLINE __m128i _mm_mulhi_epi16(__m128i a, __m128i b) return vreinterpretq_m128i_u16(r.val[1]); } +// Multiply the packed unsigned 16-bit integers in a and b, producing +// intermediate 32-bit integers, and store the high 16 bits of the intermediate +// integers in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mulhi_epu16 +FORCE_INLINE __m128i _mm_mulhi_epu16(__m128i a, __m128i b) +{ + uint16x4_t a3210 = vget_low_u16(vreinterpretq_u16_m128i(a)); + uint16x4_t b3210 = vget_low_u16(vreinterpretq_u16_m128i(b)); + uint32x4_t ab3210 = vmull_u16(a3210, b3210); +#if defined(__aarch64__) + uint32x4_t ab7654 = + vmull_high_u16(vreinterpretq_u16_m128i(a), vreinterpretq_u16_m128i(b)); + uint16x8_t r = vuzp2q_u16(vreinterpretq_u16_u32(ab3210), + vreinterpretq_u16_u32(ab7654)); + return vreinterpretq_m128i_u16(r); +#else + uint16x4_t a7654 = vget_high_u16(vreinterpretq_u16_m128i(a)); + uint16x4_t b7654 = vget_high_u16(vreinterpretq_u16_m128i(b)); + uint32x4_t ab7654 = vmull_u16(a7654, b7654); + uint16x8x2_t r = + vuzpq_u16(vreinterpretq_u16_u32(ab3210), vreinterpretq_u16_u32(ab7654)); + return vreinterpretq_m128i_u16(r.val[1]); +#endif +} + // Computes pairwise add of each argument as single-precision, floating-point // values a and b. // https://msdn.microsoft.com/en-us/library/yd9wecaa.aspx @@ -2655,6 +4448,40 @@ FORCE_INLINE __m128i _mm_hadd_epi16(__m128i _a, __m128i _b) #endif } +// Horizontally substract adjacent pairs of single-precision (32-bit) +// floating-point elements in a and b, and pack the results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_hsub_ps +FORCE_INLINE __m128 _mm_hsub_ps(__m128 _a, __m128 _b) +{ +#if defined(__aarch64__) + return vreinterpretq_m128_f32(vsubq_f32( + vuzp1q_f32(vreinterpretq_f32_m128(_a), vreinterpretq_f32_m128(_b)), + vuzp2q_f32(vreinterpretq_f32_m128(_a), vreinterpretq_f32_m128(_b)))); +#else + float32x4x2_t c = + vuzpq_f32(vreinterpretq_f32_m128(_a), vreinterpretq_f32_m128(_b)); + return vreinterpretq_m128_f32(vsubq_f32(c.val[0], c.val[1])); +#endif +} + +// Horizontally add adjacent pairs of 16-bit integers in a and b, and pack the +// signed 16-bit results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_hadd_pi16 +FORCE_INLINE __m64 _mm_hadd_pi16(__m64 a, __m64 b) +{ + return vreinterpret_m64_s16( + vpadd_s16(vreinterpret_s16_m64(a), vreinterpret_s16_m64(b))); +} + +// Horizontally add adjacent pairs of 32-bit integers in a and b, and pack the +// signed 32-bit results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_hadd_pi32 +FORCE_INLINE __m64 _mm_hadd_pi32(__m64 a, __m64 b) +{ + return vreinterpret_m64_s32( + vpadd_s32(vreinterpret_s32_m64(a), vreinterpret_s32_m64(b))); +} + // Computes pairwise difference of each argument as a 16-bit signed or unsigned // integer values a and b. FORCE_INLINE __m128i _mm_hsub_epi16(__m128i _a, __m128i _b) @@ -2674,6 +4501,12 @@ FORCE_INLINE __m128i _mm_hsub_epi16(__m128i _a, __m128i _b) // integer values a and b. FORCE_INLINE __m128i _mm_hadds_epi16(__m128i _a, __m128i _b) { +#if defined(__aarch64__) + int16x8_t a = vreinterpretq_s16_m128i(_a); + int16x8_t b = vreinterpretq_s16_m128i(_b); + return vreinterpretq_s64_s16( + vqaddq_s16(vuzp1q_s16(a, b), vuzp2q_s16(a, b))); +#else int32x4_t a = vreinterpretq_s32_m128i(_a); int32x4_t b = vreinterpretq_s32_m128i(_b); // Interleave using vshrn/vmovn @@ -2683,12 +4516,20 @@ FORCE_INLINE __m128i _mm_hadds_epi16(__m128i _a, __m128i _b) int16x8_t ab1357 = vcombine_s16(vshrn_n_s32(a, 16), vshrn_n_s32(b, 16)); // Saturated add return vreinterpretq_m128i_s16(vqaddq_s16(ab0246, ab1357)); +#endif } // Computes saturated pairwise difference of each argument as a 16-bit signed // integer values a and b. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_hsubs_epi16 FORCE_INLINE __m128i _mm_hsubs_epi16(__m128i _a, __m128i _b) { +#if defined(__aarch64__) + int16x8_t a = vreinterpretq_s16_m128i(_a); + int16x8_t b = vreinterpretq_s16_m128i(_b); + return vreinterpretq_s64_s16( + vqsubq_s16(vuzp1q_s16(a, b), vuzp2q_s16(a, b))); +#else int32x4_t a = vreinterpretq_s32_m128i(_a); int32x4_t b = vreinterpretq_s32_m128i(_b); // Interleave using vshrn/vmovn @@ -2698,6 +4539,7 @@ FORCE_INLINE __m128i _mm_hsubs_epi16(__m128i _a, __m128i _b) int16x8_t ab1357 = vcombine_s16(vshrn_n_s32(a, 16), vshrn_n_s32(b, 16)); // Saturated subtract return vreinterpretq_m128i_s16(vqsubq_s16(ab0246, ab1357)); +#endif } // Computes pairwise add of each argument as a 32-bit signed or unsigned integer @@ -2726,6 +4568,60 @@ FORCE_INLINE __m128i _mm_hsub_epi32(__m128i _a, __m128i _b) return vreinterpretq_m128i_s32(vsubq_s32(ab02, ab13)); } +// Kahan summation for accurate summation of floating-point numbers. +// http://blog.zachbjornson.com/2019/08/11/fast-float-summation.html +FORCE_INLINE void _sse2neon_kadd_f32(float *sum, float *c, float y) +{ + y -= *c; + float t = *sum + y; + *c = (t - *sum) - y; + *sum = t; +} + +// Conditionally multiply the packed single-precision (32-bit) floating-point +// elements in a and b using the high 4 bits in imm8, sum the four products, +// and conditionally store the sum in dst using the low 4 bits of imm. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_dp_ps +FORCE_INLINE __m128 _mm_dp_ps(__m128 a, __m128 b, const int imm) +{ +#if defined(__aarch64__) + /* shortcuts */ + if (imm == 0xFF) { + return _mm_set1_ps(vaddvq_f32(_mm_mul_ps(a, b))); + } + if (imm == 0x7F) { + float32x4_t m = _mm_mul_ps(a, b); + m[3] = 0; + return _mm_set1_ps(vaddvq_f32(m)); + } +#endif + + float s = 0, c = 0; + float32x4_t f32a = vreinterpretq_f32_m128(a); + float32x4_t f32b = vreinterpretq_f32_m128(b); + + /* To improve the accuracy of floating-point summation, Kahan algorithm + * is used for each operation. + */ + if (imm & (1 << 4)) + _sse2neon_kadd_f32(&s, &c, f32a[0] * f32b[0]); + if (imm & (1 << 5)) + _sse2neon_kadd_f32(&s, &c, f32a[1] * f32b[1]); + if (imm & (1 << 6)) + _sse2neon_kadd_f32(&s, &c, f32a[2] * f32b[2]); + if (imm & (1 << 7)) + _sse2neon_kadd_f32(&s, &c, f32a[3] * f32b[3]); + s += c; + + float32x4_t res = { + (imm & 0x1) ? s : 0, + (imm & 0x2) ? s : 0, + (imm & 0x4) ? s : 0, + (imm & 0x8) ? s : 0, + }; + return vreinterpretq_m128_f32(res); +} + /* Compare operations */ // Compares for less than @@ -2736,6 +4632,13 @@ FORCE_INLINE __m128 _mm_cmplt_ps(__m128 a, __m128 b) vcltq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); } +// Compares for less than +// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/fy94wye7(v=vs.100) +FORCE_INLINE __m128 _mm_cmplt_ss(__m128 a, __m128 b) +{ + return _mm_move_ss(a, _mm_cmplt_ps(a, b)); +} + // Compares for greater than. // // r0 := (a0 > b0) ? 0xffffffff : 0x0 @@ -2750,6 +4653,13 @@ FORCE_INLINE __m128 _mm_cmpgt_ps(__m128 a, __m128 b) vcgtq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); } +// Compares for greater than. +// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/1xyyyy9e(v=vs.100) +FORCE_INLINE __m128 _mm_cmpgt_ss(__m128 a, __m128 b) +{ + return _mm_move_ss(a, _mm_cmpgt_ps(a, b)); +} + // Compares for greater than or equal. // https://msdn.microsoft.com/en-us/library/vstudio/fs813y2t(v=vs.100).aspx FORCE_INLINE __m128 _mm_cmpge_ps(__m128 a, __m128 b) @@ -2758,6 +4668,13 @@ FORCE_INLINE __m128 _mm_cmpge_ps(__m128 a, __m128 b) vcgeq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); } +// Compares for greater than or equal. +// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/kesh3ddc(v=vs.100) +FORCE_INLINE __m128 _mm_cmpge_ss(__m128 a, __m128 b) +{ + return _mm_move_ss(a, _mm_cmpge_ps(a, b)); +} + // Compares for less than or equal. // // r0 := (a0 <= b0) ? 0xffffffff : 0x0 @@ -2772,6 +4689,13 @@ FORCE_INLINE __m128 _mm_cmple_ps(__m128 a, __m128 b) vcleq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); } +// Compares for less than or equal. +// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/a7x0hbhw(v=vs.100) +FORCE_INLINE __m128 _mm_cmple_ss(__m128 a, __m128 b) +{ + return _mm_move_ss(a, _mm_cmple_ps(a, b)); +} + // Compares for equality. // https://msdn.microsoft.com/en-us/library/vstudio/36aectz5(v=vs.100).aspx FORCE_INLINE __m128 _mm_cmpeq_ps(__m128 a, __m128 b) @@ -2780,6 +4704,84 @@ FORCE_INLINE __m128 _mm_cmpeq_ps(__m128 a, __m128 b) vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); } +// Compares for equality. +// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/k423z28e(v=vs.100) +FORCE_INLINE __m128 _mm_cmpeq_ss(__m128 a, __m128 b) +{ + return _mm_move_ss(a, _mm_cmpeq_ps(a, b)); +} + +// Compares for inequality. +// https://msdn.microsoft.com/en-us/library/sf44thbx(v=vs.100).aspx +FORCE_INLINE __m128 _mm_cmpneq_ps(__m128 a, __m128 b) +{ + return vreinterpretq_m128_u32(vmvnq_u32( + vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)))); +} + +// Compares for inequality. +// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/ekya8fh4(v=vs.100) +FORCE_INLINE __m128 _mm_cmpneq_ss(__m128 a, __m128 b) +{ + return _mm_move_ss(a, _mm_cmpneq_ps(a, b)); +} + +// Compares for not greater than or equal. +// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/wsexys62(v=vs.100) +FORCE_INLINE __m128 _mm_cmpnge_ps(__m128 a, __m128 b) +{ + return _mm_cmplt_ps(a, b); +} + +// Compares for not greater than or equal. +// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/fk2y80s8(v=vs.100) +FORCE_INLINE __m128 _mm_cmpnge_ss(__m128 a, __m128 b) +{ + return _mm_cmplt_ss(a, b); +} + +// Compares for not greater than. +// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/d0xh7w0s(v=vs.100) +FORCE_INLINE __m128 _mm_cmpngt_ps(__m128 a, __m128 b) +{ + return _mm_cmple_ps(a, b); +} + +// Compares for not greater than. +// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/z7x9ydwh(v=vs.100) +FORCE_INLINE __m128 _mm_cmpngt_ss(__m128 a, __m128 b) +{ + return _mm_cmple_ss(a, b); +} + +// Compares for not less than or equal. +// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/6a330kxw(v=vs.100) +FORCE_INLINE __m128 _mm_cmpnle_ps(__m128 a, __m128 b) +{ + return _mm_cmpgt_ps(a, b); +} + +// Compares for not less than or equal. +// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/z7x9ydwh(v=vs.100) +FORCE_INLINE __m128 _mm_cmpnle_ss(__m128 a, __m128 b) +{ + return _mm_cmpgt_ss(a, b); +} + +// Compares for not less than. +// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/4686bbdw(v=vs.100) +FORCE_INLINE __m128 _mm_cmpnlt_ps(__m128 a, __m128 b) +{ + return _mm_cmpge_ps(a, b); +} + +// Compares for not less than. +// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/56b9z2wf(v=vs.100) +FORCE_INLINE __m128 _mm_cmpnlt_ss(__m128 a, __m128 b) +{ + return _mm_cmpge_ss(a, b); +} + // Compares the 16 signed or unsigned 8-bit integers in a and the 16 signed or // unsigned 8-bit integers in b for equality. // https://msdn.microsoft.com/en-us/library/windows/desktop/bz5xk21a(v=vs.90).aspx @@ -2789,6 +4791,23 @@ FORCE_INLINE __m128i _mm_cmpeq_epi8(__m128i a, __m128i b) vceqq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); } +// Compare packed double-precision (64-bit) floating-point elements in a and b +// for equality, and store the results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpeq_pd +FORCE_INLINE __m128d _mm_cmpeq_pd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) + return vreinterpretq_m128d_u64( + vceqq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#else + // (a == b) -> (a_lo == b_lo) && (a_hi == b_hi) + uint32x4_t cmp = + vceqq_u32(vreinterpretq_u32_m128d(a), vreinterpretq_u32_m128d(b)); + uint32x4_t swapped = vrev64q_u32(cmp); + return vreinterpretq_m128d_u32(vandq_u32(cmp, swapped)); +#endif +} + // Compares the 8 signed or unsigned 16-bit integers in a and the 8 signed or // unsigned 16-bit integers in b for equality. // https://msdn.microsoft.com/en-us/library/2ay060te(v=vs.100).aspx @@ -2832,6 +4851,27 @@ FORCE_INLINE __m128i _mm_cmplt_epi8(__m128i a, __m128i b) vcltq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); } +// Compare packed double-precision (64-bit) floating-point elements in a and b +// for less-than, and store the results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmplt_pd +FORCE_INLINE __m128d _mm_cmplt_pd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) + return vreinterpretq_m128d_u64( + vcltq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#else + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t b1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(b)); + uint64_t d[2]; + d[0] = (*(double *) &a0) < (*(double *) &b0) ? ~UINT64_C(0) : UINT64_C(0); + d[1] = (*(double *) &a1) < (*(double *) &b1) ? ~UINT64_C(0) : UINT64_C(0); + + return vreinterpretq_m128d_u64(vld1q_u64(d)); +#endif +} + // Compares the 16 signed 8-bit integers in a and the 16 signed 8-bit integers // in b for greater than. // @@ -2951,6 +4991,31 @@ FORCE_INLINE __m128 _mm_cmpord_ps(__m128 a, __m128 b) return vreinterpretq_m128_u32(vandq_u32(ceqaa, ceqbb)); } +// Compares for ordered. +// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/343t62da(v=vs.100) +FORCE_INLINE __m128 _mm_cmpord_ss(__m128 a, __m128 b) +{ + return _mm_move_ss(a, _mm_cmpord_ps(a, b)); +} + +// Compares for unordered. +// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/khy6fk1t(v=vs.100) +FORCE_INLINE __m128 _mm_cmpunord_ps(__m128 a, __m128 b) +{ + uint32x4_t f32a = + vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a)); + uint32x4_t f32b = + vceqq_f32(vreinterpretq_f32_m128(b), vreinterpretq_f32_m128(b)); + return vreinterpretq_m128_u32(vmvnq_u32(vandq_u32(f32a, f32b))); +} + +// Compares for unordered. +// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/2as2387b(v=vs.100) +FORCE_INLINE __m128 _mm_cmpunord_ss(__m128 a, __m128 b) +{ + return _mm_move_ss(a, _mm_cmpunord_ps(a, b)); +} + // Compares the lower single-precision floating point scalar values of a and b // using a less than operation. : // https://msdn.microsoft.com/en-us/library/2kwe606b(v=vs.90).aspx Important @@ -3055,15 +5120,192 @@ FORCE_INLINE int _mm_comineq_ss(__m128 a, __m128 b) // according to the documentation, these intrinsics behave the same as the // non-'u' versions. We'll just alias them here. -#define _mm_ucomilt_ss _mm_comilt_ss -#define _mm_ucomile_ss _mm_comile_ss -#define _mm_ucomigt_ss _mm_comigt_ss -#define _mm_ucomige_ss _mm_comige_ss #define _mm_ucomieq_ss _mm_comieq_ss +#define _mm_ucomige_ss _mm_comige_ss +#define _mm_ucomigt_ss _mm_comigt_ss +#define _mm_ucomile_ss _mm_comile_ss +#define _mm_ucomilt_ss _mm_comilt_ss #define _mm_ucomineq_ss _mm_comineq_ss /* Conversions */ +// Convert packed signed 32-bit integers in b to packed single-precision +// (32-bit) floating-point elements, store the results in the lower 2 elements +// of dst, and copy the upper 2 packed elements from a to the upper elements of +// dst. +// +// dst[31:0] := Convert_Int32_To_FP32(b[31:0]) +// dst[63:32] := Convert_Int32_To_FP32(b[63:32]) +// dst[95:64] := a[95:64] +// dst[127:96] := a[127:96] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_pi2ps +FORCE_INLINE __m128 _mm_cvt_pi2ps(__m128 a, __m64 b) +{ + return vreinterpretq_m128_f32( + vcombine_f32(vcvt_f32_s32(vreinterpret_s32_m64(b)), + vget_high_f32(vreinterpretq_f32_m128(a)))); +} + +// Convert the signed 32-bit integer b to a single-precision (32-bit) +// floating-point element, store the result in the lower element of dst, and +// copy the upper 3 packed elements from a to the upper elements of dst. +// +// dst[31:0] := Convert_Int32_To_FP32(b[31:0]) +// dst[127:32] := a[127:32] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_si2ss +FORCE_INLINE __m128 _mm_cvt_si2ss(__m128 a, int b) +{ + return vreinterpretq_m128_f32( + vsetq_lane_f32((float) b, vreinterpretq_f32_m128(a), 0)); +} + +// Convert the signed 32-bit integer b to a single-precision (32-bit) +// floating-point element, store the result in the lower element of dst, and +// copy the upper 3 packed elements from a to the upper elements of dst. +// +// dst[31:0] := Convert_Int32_To_FP32(b[31:0]) +// dst[127:32] := a[127:32] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsi32_ss +#define _mm_cvtsi32_ss(a, b) _mm_cvt_si2ss(a, b) + +// Convert the signed 64-bit integer b to a single-precision (32-bit) +// floating-point element, store the result in the lower element of dst, and +// copy the upper 3 packed elements from a to the upper elements of dst. +// +// dst[31:0] := Convert_Int64_To_FP32(b[63:0]) +// dst[127:32] := a[127:32] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsi64_ss +FORCE_INLINE __m128 _mm_cvtsi64_ss(__m128 a, int64_t b) +{ + return vreinterpretq_m128_f32( + vsetq_lane_f32((float) b, vreinterpretq_f32_m128(a), 0)); +} + +// Convert the lower single-precision (32-bit) floating-point element in a to a +// 32-bit integer, and store the result in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_ss2si +FORCE_INLINE int _mm_cvt_ss2si(__m128 a) +{ +#if defined(__aarch64__) + return vgetq_lane_s32(vcvtnq_s32_f32(vreinterpretq_f32_m128(a)), 0); +#else + float32_t data = vgetq_lane_f32(vreinterpretq_f32_m128(a), 0); + float32_t diff = data - floor(data); + if (diff > 0.5) + return (int32_t) ceil(data); + if (unlikely(diff == 0.5)) { + int32_t f = (int32_t) floor(data); + int32_t c = (int32_t) ceil(data); + return c & 1 ? f : c; + } + return (int32_t) floor(data); +#endif +} + +// Convert packed 16-bit integers in a to packed single-precision (32-bit) +// floating-point elements, and store the results in dst. +// +// FOR j := 0 to 3 +// i := j*16 +// m := j*32 +// dst[m+31:m] := Convert_Int16_To_FP32(a[i+15:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtpi16_ps +FORCE_INLINE __m128 _mm_cvtpi16_ps(__m64 a) +{ + return vreinterpretq_m128_f32( + vcvtq_f32_s32(vmovl_s16(vreinterpret_s16_m64(a)))); +} + +// Convert packed 32-bit integers in b to packed single-precision (32-bit) +// floating-point elements, store the results in the lower 2 elements of dst, +// and copy the upper 2 packed elements from a to the upper elements of dst. +// +// dst[31:0] := Convert_Int32_To_FP32(b[31:0]) +// dst[63:32] := Convert_Int32_To_FP32(b[63:32]) +// dst[95:64] := a[95:64] +// dst[127:96] := a[127:96] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtpi32_ps +FORCE_INLINE __m128 _mm_cvtpi32_ps(__m128 a, __m64 b) +{ + return vreinterpretq_m128_f32( + vcombine_f32(vcvt_f32_s32(vreinterpret_s32_m64(b)), + vget_high_f32(vreinterpretq_f32_m128(a)))); +} + +// Convert packed signed 32-bit integers in a to packed single-precision +// (32-bit) floating-point elements, store the results in the lower 2 elements +// of dst, then covert the packed signed 32-bit integers in b to +// single-precision (32-bit) floating-point element, and store the results in +// the upper 2 elements of dst. +// +// dst[31:0] := Convert_Int32_To_FP32(a[31:0]) +// dst[63:32] := Convert_Int32_To_FP32(a[63:32]) +// dst[95:64] := Convert_Int32_To_FP32(b[31:0]) +// dst[127:96] := Convert_Int32_To_FP32(b[63:32]) +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtpi32x2_ps +FORCE_INLINE __m128 _mm_cvtpi32x2_ps(__m64 a, __m64 b) +{ + return vreinterpretq_m128_f32(vcvtq_f32_s32( + vcombine_s32(vreinterpret_s32_m64(a), vreinterpret_s32_m64(b)))); +} + +// Convert the lower packed 8-bit integers in a to packed single-precision +// (32-bit) floating-point elements, and store the results in dst. +// +// FOR j := 0 to 3 +// i := j*8 +// m := j*32 +// dst[m+31:m] := Convert_Int8_To_FP32(a[i+7:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtpi8_ps +FORCE_INLINE __m128 _mm_cvtpi8_ps(__m64 a) +{ + return vreinterpretq_m128_f32(vcvtq_f32_s32( + vmovl_s16(vget_low_s16(vmovl_s8(vreinterpret_s8_m64(a)))))); +} + +// Convert packed unsigned 16-bit integers in a to packed single-precision +// (32-bit) floating-point elements, and store the results in dst. +// +// FOR j := 0 to 3 +// i := j*16 +// m := j*32 +// dst[m+31:m] := Convert_UInt16_To_FP32(a[i+15:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtpu16_ps +FORCE_INLINE __m128 _mm_cvtpu16_ps(__m64 a) +{ + return vreinterpretq_m128_f32( + vcvtq_f32_u32(vmovl_u16(vreinterpret_u16_m64(a)))); +} + +// Convert the lower packed unsigned 8-bit integers in a to packed +// single-precision (32-bit) floating-point elements, and store the results in +// dst. +// +// FOR j := 0 to 3 +// i := j*8 +// m := j*32 +// dst[m+31:m] := Convert_UInt8_To_FP32(a[i+7:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtpu8_ps +FORCE_INLINE __m128 _mm_cvtpu8_ps(__m64 a) +{ + return vreinterpretq_m128_f32(vcvtq_f32_u32( + vmovl_u16(vget_low_u16(vmovl_u8(vreinterpret_u8_m64(a)))))); +} + // Converts the four single-precision, floating-point values of a to signed // 32-bit integer values using truncate. // https://msdn.microsoft.com/en-us/library/vstudio/1h005y6x(v=vs.100).aspx @@ -3072,6 +5314,30 @@ FORCE_INLINE __m128i _mm_cvttps_epi32(__m128 a) return vreinterpretq_m128i_s32(vcvtq_s32_f32(vreinterpretq_f32_m128(a))); } +// Convert the lower double-precision (64-bit) floating-point element in a to a +// 64-bit integer with truncation, and store the result in dst. +// +// dst[63:0] := Convert_FP64_To_Int64_Truncate(a[63:0]) +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttsd_si64 +FORCE_INLINE int64_t _mm_cvttsd_si64(__m128d a) +{ +#if defined(__aarch64__) + return vgetq_lane_s64(vcvtq_s64_f64(vreinterpretq_f64_m128d(a)), 0); +#else + double ret = *((double *) &a); + return (int64_t) ret; +#endif +} + +// Convert the lower double-precision (64-bit) floating-point element in a to a +// 64-bit integer with truncation, and store the result in dst. +// +// dst[63:0] := Convert_FP64_To_Int64_Truncate(a[63:0]) +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttsd_si64x +#define _mm_cvttsd_si64x(a) _mm_cvttsd_si64(a) + // Converts the four signed 32-bit integer values of a to single-precision, // floating-point values // https://msdn.microsoft.com/en-us/library/vstudio/36bwxcx5(v=vs.100).aspx @@ -3080,6 +5346,28 @@ FORCE_INLINE __m128 _mm_cvtepi32_ps(__m128i a) return vreinterpretq_m128_f32(vcvtq_f32_s32(vreinterpretq_s32_m128i(a))); } +// Convert packed signed 32-bit integers in a to packed double-precision +// (64-bit) floating-point elements, and store the results in dst. +// +// FOR j := 0 to 1 +// i := j*32 +// m := j*64 +// dst[m+63:m] := Convert_Int32_To_FP64(a[i+31:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtepi32_pd +FORCE_INLINE __m128d _mm_cvtepi32_pd(__m128i a) +{ +#if defined(__aarch64__) + return vreinterpretq_m128d_f64( + vcvtq_f64_s64(vmovl_s32(vget_low_s32(vreinterpretq_s32_m128i(a))))); +#else + double a0 = (double) vgetq_lane_s32(vreinterpretq_s32_m128i(a), 0); + double a1 = (double) vgetq_lane_s32(vreinterpretq_s32_m128i(a), 1); + return _mm_set_pd(a1, a0); +#endif +} + // Converts the four unsigned 8-bit integers in the lower 16 bits to four // unsigned 32-bit integers. FORCE_INLINE __m128i _mm_cvtepu8_epi16(__m128i a) @@ -3228,20 +5516,45 @@ FORCE_INLINE __m128i _mm_cvtps_epi32(__m128 a) #endif } -// Moves the least significant 32 bits of a to a 32-bit integer. -// https://msdn.microsoft.com/en-us/library/5z7a9642%28v=vs.90%29.aspx +// Convert packed single-precision (32-bit) floating-point elements in a to +// packed 16-bit integers, and store the results in dst. Note: this intrinsic +// will generate 0x7FFF, rather than 0x8000, for input values between 0x7FFF and +// 0x7FFFFFFF. +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtps_pi16 +FORCE_INLINE __m64 _mm_cvtps_pi16(__m128 a) +{ + return vreinterpret_m64_s16( + vmovn_s32(vreinterpretq_s32_m128i(_mm_cvtps_epi32(a)))); +} + +// Copy the lower 32-bit integer in a to dst. +// +// dst[31:0] := a[31:0] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsi128_si32 FORCE_INLINE int _mm_cvtsi128_si32(__m128i a) { return vgetq_lane_s32(vreinterpretq_s32_m128i(a), 0); } -// Extracts the low order 64-bit integer from the parameter. -// https://msdn.microsoft.com/en-us/library/bb531384(v=vs.120).aspx -FORCE_INLINE uint64_t _mm_cvtsi128_si64(__m128i a) +// Copy the lower 64-bit integer in a to dst. +// +// dst[63:0] := a[63:0] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsi128_si64 +FORCE_INLINE int64_t _mm_cvtsi128_si64(__m128i a) { return vgetq_lane_s64(vreinterpretq_s64_m128i(a), 0); } +// Copy the lower 64-bit integer in a to dst. +// +// dst[63:0] := a[63:0] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsi128_si64x +#define _mm_cvtsi128_si64x(a) _mm_cvtsi128_si64(a) + // Moves 32-bit integer a to the least significant 32 bits of an __m128 object, // zero extending the upper bits. // @@ -3266,6 +5579,14 @@ FORCE_INLINE __m128i _mm_cvtsi64_si128(int64_t a) return vreinterpretq_m128i_s64(vsetq_lane_s64(a, vdupq_n_s64(0), 0)); } +// Cast vector of type __m128 to type __m128d. This intrinsic is only used for +// compilation and does not generate any instructions, thus it has zero latency. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_castps_pd +FORCE_INLINE __m128d _mm_castps_pd(__m128 a) +{ + return vreinterpretq_m128d_s32(vreinterpretq_s32_m128(a)); +} + // Applies a type cast to reinterpret four 32-bit floating point values passed // in as a 128-bit parameter as packed 32-bit integers. // https://msdn.microsoft.com/en-us/library/bb514099.aspx @@ -3274,6 +5595,18 @@ FORCE_INLINE __m128i _mm_castps_si128(__m128 a) return vreinterpretq_m128i_s32(vreinterpretq_s32_m128(a)); } +// Cast vector of type __m128i to type __m128d. This intrinsic is only used for +// compilation and does not generate any instructions, thus it has zero latency. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_castsi128_pd +FORCE_INLINE __m128d _mm_castsi128_pd(__m128i a) +{ +#if defined(__aarch64__) + return vreinterpretq_m128d_f64(vreinterpretq_f64_m128i(a)); +#else + return vreinterpretq_m128d_f32(vreinterpretq_f32_m128i(a)); +#endif +} + // Applies a type cast to reinterpret four 32-bit integers passed in as a // 128-bit parameter as packed 32-bit floating point values. // https://msdn.microsoft.com/en-us/library/bb514029.aspx @@ -3289,6 +5622,68 @@ FORCE_INLINE __m128i _mm_load_si128(const __m128i *p) return vreinterpretq_m128i_s32(vld1q_s32((const int32_t *) p)); } +// Load a double-precision (64-bit) floating-point element from memory into both +// elements of dst. +// +// dst[63:0] := MEM[mem_addr+63:mem_addr] +// dst[127:64] := MEM[mem_addr+63:mem_addr] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_load1_pd +FORCE_INLINE __m128d _mm_load1_pd(const double *p) +{ +#if defined(__aarch64__) + return vreinterpretq_m128d_f64(vld1q_dup_f64(p)); +#else + return vreinterpretq_m128d_s64(vdupq_n_s64(*(const int64_t *) p)); +#endif +} + +// Load a double-precision (64-bit) floating-point element from memory into both +// elements of dst. +// +// dst[63:0] := MEM[mem_addr+63:mem_addr] +// dst[127:64] := MEM[mem_addr+63:mem_addr] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_load_pd1 +#define _mm_load_pd1 _mm_load1_pd + +// Load a double-precision (64-bit) floating-point element from memory into the +// upper element of dst, and copy the lower element from a to dst. mem_addr does +// not need to be aligned on any particular boundary. +// +// dst[63:0] := a[63:0] +// dst[127:64] := MEM[mem_addr+63:mem_addr] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadh_pd +FORCE_INLINE __m128d _mm_loadh_pd(__m128d a, const double *p) +{ +#if defined(__aarch64__) + return vreinterpretq_m128d_f64( + vcombine_f64(vget_low_f64(vreinterpretq_f64_m128d(a)), vld1_f64(p))); +#else + return vreinterpretq_m128d_f32(vcombine_f32( + vget_low_f32(vreinterpretq_f32_m128d(a)), vld1_f32((const float *) p))); +#endif +} + +// Load a double-precision (64-bit) floating-point element from memory into both +// elements of dst. +// +// dst[63:0] := MEM[mem_addr+63:mem_addr] +// dst[127:64] := MEM[mem_addr+63:mem_addr] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_load_pd1 +#define _mm_load_pd1 _mm_load1_pd + +// Load a double-precision (64-bit) floating-point element from memory into both +// elements of dst. +// +// dst[63:0] := MEM[mem_addr+63:mem_addr] +// dst[127:64] := MEM[mem_addr+63:mem_addr] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loaddup_pd +#define _mm_loaddup_pd _mm_load1_pd + // Loads 128-bit value. : // https://msdn.microsoft.com/zh-cn/library/f4k12ae8(v=vs.90).aspx FORCE_INLINE __m128i _mm_loadu_si128(const __m128i *p) @@ -3296,7 +5691,338 @@ FORCE_INLINE __m128i _mm_loadu_si128(const __m128i *p) return vreinterpretq_m128i_s32(vld1q_s32((const int32_t *) p)); } -// _mm_lddqu_si128 functions the same as _mm_loadu_si128. +// Load unaligned 32-bit integer from memory into the first element of dst. +// +// dst[31:0] := MEM[mem_addr+31:mem_addr] +// dst[MAX:32] := 0 +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadu_si32 +FORCE_INLINE __m128i _mm_loadu_si32(const void *p) +{ + return vreinterpretq_m128i_s32( + vsetq_lane_s32(*(const int32_t *) p, vdupq_n_s32(0), 0)); +} + +// Convert packed double-precision (64-bit) floating-point elements in a to +// packed single-precision (32-bit) floating-point elements, and store the +// results in dst. +// +// FOR j := 0 to 1 +// i := 32*j +// k := 64*j +// dst[i+31:i] := Convert_FP64_To_FP32(a[k+64:k]) +// ENDFOR +// dst[127:64] := 0 +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtpd_ps +FORCE_INLINE __m128 _mm_cvtpd_ps(__m128d a) +{ +#if defined(__aarch64__) + float32x2_t tmp = vcvt_f32_f64(vreinterpretq_f64_m128d(a)); + return vreinterpretq_m128_f32(vcombine_f32(tmp, vdup_n_f32(0))); +#else + float a0 = (float) ((double *) &a)[0]; + float a1 = (float) ((double *) &a)[1]; + return _mm_set_ps(0, 0, a1, a0); +#endif +} + +// Copy the lower double-precision (64-bit) floating-point element of a to dst. +// +// dst[63:0] := a[63:0] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsd_f64 +FORCE_INLINE double _mm_cvtsd_f64(__m128d a) +{ +#if defined(__aarch64__) + return (double) vgetq_lane_f64(vreinterpretq_f64_m128d(a), 0); +#else + return ((double *) &a)[0]; +#endif +} + +// Convert packed single-precision (32-bit) floating-point elements in a to +// packed double-precision (64-bit) floating-point elements, and store the +// results in dst. +// +// FOR j := 0 to 1 +// i := 64*j +// k := 32*j +// dst[i+63:i] := Convert_FP32_To_FP64(a[k+31:k]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtps_pd +FORCE_INLINE __m128d _mm_cvtps_pd(__m128 a) +{ +#if defined(__aarch64__) + return vreinterpretq_m128d_f64( + vcvt_f64_f32(vget_low_f32(vreinterpretq_f32_m128(a)))); +#else + double a0 = (double) vgetq_lane_f32(vreinterpretq_f32_m128(a), 0); + double a1 = (double) vgetq_lane_f32(vreinterpretq_f32_m128(a), 1); + return _mm_set_pd(a1, a0); +#endif +} + +// Cast vector of type __m128d to type __m128i. This intrinsic is only used for +// compilation and does not generate any instructions, thus it has zero latency. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_castpd_si128 +FORCE_INLINE __m128i _mm_castpd_si128(__m128d a) +{ + return vreinterpretq_m128i_s64(vreinterpretq_s64_m128d(a)); +} + +// Cast vector of type __m128d to type __m128. This intrinsic is only used for +// compilation and does not generate any instructions, thus it has zero latency. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_castpd_ps +FORCE_INLINE __m128 _mm_castpd_ps(__m128d a) +{ + return vreinterpretq_m128_s64(vreinterpretq_s64_m128d(a)); +} + +// Blend packed single-precision (32-bit) floating-point elements from a and b +// using mask, and store the results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_blendv_ps +FORCE_INLINE __m128 _mm_blendv_ps(__m128 _a, __m128 _b, __m128 _mask) +{ + // Use a signed shift right to create a mask with the sign bit + uint32x4_t mask = + vreinterpretq_u32_s32(vshrq_n_s32(vreinterpretq_s32_m128(_mask), 31)); + float32x4_t a = vreinterpretq_f32_m128(_a); + float32x4_t b = vreinterpretq_f32_m128(_b); + return vreinterpretq_m128_f32(vbslq_f32(mask, b, a)); +} + +// Blend packed single-precision (32-bit) floating-point elements from a and b +// using mask, and store the results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_blend_ps +FORCE_INLINE __m128 _mm_blend_ps(__m128 _a, __m128 _b, const char imm8) +{ + const uint32_t ALIGN_STRUCT(16) + data[4] = {((imm8) & (1 << 0)) ? UINT32_MAX : 0, + ((imm8) & (1 << 1)) ? UINT32_MAX : 0, + ((imm8) & (1 << 2)) ? UINT32_MAX : 0, + ((imm8) & (1 << 3)) ? UINT32_MAX : 0}; + uint32x4_t mask = vld1q_u32(data); + float32x4_t a = vreinterpretq_f32_m128(_a); + float32x4_t b = vreinterpretq_f32_m128(_b); + return vreinterpretq_m128_f32(vbslq_f32(mask, b, a)); +} + +// Blend packed double-precision (64-bit) floating-point elements from a and b +// using mask, and store the results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_blendv_pd +FORCE_INLINE __m128d _mm_blendv_pd(__m128d _a, __m128d _b, __m128d _mask) +{ + uint64x2_t mask = + vreinterpretq_u64_s64(vshrq_n_s64(vreinterpretq_s64_m128d(_mask), 63)); +#if defined(__aarch64__) + float64x2_t a = vreinterpretq_f64_m128d(_a); + float64x2_t b = vreinterpretq_f64_m128d(_b); + return vreinterpretq_m128d_f64(vbslq_f64(mask, b, a)); +#else + uint64x2_t a = vreinterpretq_u64_m128d(_a); + uint64x2_t b = vreinterpretq_u64_m128d(_b); + return vreinterpretq_m128d_u64(vbslq_u64(mask, b, a)); +#endif +} + +typedef struct { + uint16_t res0; + uint8_t res1 : 6; + uint8_t bit22 : 1; + uint8_t bit23 : 1; + uint8_t res2; +#if defined(__aarch64__) + uint32_t res3; +#endif +} fpcr_bitfield; + +// Macro: Set the rounding mode bits of the MXCSR control and status register to +// the value in unsigned 32-bit integer a. The rounding mode may contain any of +// the following flags: _MM_ROUND_NEAREST, _MM_ROUND_DOWN, _MM_ROUND_UP, +// _MM_ROUND_TOWARD_ZERO +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_MM_SET_ROUNDING_MODE +FORCE_INLINE void _MM_SET_ROUNDING_MODE(int rounding) +{ + union { + fpcr_bitfield field; +#if defined(__aarch64__) + uint64_t value; +#else + uint32_t value; +#endif + } r; + +#if defined(__aarch64__) + asm volatile("mrs %0, FPCR" : "=r"(r.value)); /* read */ +#else + asm volatile("vmrs %0, FPSCR" : "=r"(r.value)); /* read */ +#endif + + switch (rounding) { + case _MM_ROUND_TOWARD_ZERO: + r.field.bit22 = 1; + r.field.bit23 = 1; + break; + case _MM_ROUND_DOWN: + r.field.bit22 = 0; + r.field.bit23 = 1; + break; + case _MM_ROUND_UP: + r.field.bit22 = 1; + r.field.bit23 = 0; + break; + default: //_MM_ROUND_NEAREST + r.field.bit22 = 0; + r.field.bit23 = 0; + } + +#if defined(__aarch64__) + asm volatile("msr FPCR, %0" ::"r"(r)); /* write */ +#else + asm volatile("vmsr FPSCR, %0" ::"r"(r)); /* write */ +#endif +} + +FORCE_INLINE void _mm_setcsr(unsigned int a) +{ + _MM_SET_ROUNDING_MODE(a); +} + +// Round the packed single-precision (32-bit) floating-point elements in a using +// the rounding parameter, and store the results as packed single-precision +// floating-point elements in dst. +// software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_round_ps +FORCE_INLINE __m128 _mm_round_ps(__m128 a, int rounding) +{ +#if defined(__aarch64__) + switch (rounding) { + case (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC): + return vreinterpretq_m128_f32(vrndnq_f32(vreinterpretq_f32_m128(a))); + case (_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC): + return vreinterpretq_m128_f32(vrndmq_f32(vreinterpretq_f32_m128(a))); + case (_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC): + return vreinterpretq_m128_f32(vrndpq_f32(vreinterpretq_f32_m128(a))); + case (_MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC): + return vreinterpretq_m128_f32(vrndq_f32(vreinterpretq_f32_m128(a))); + default: //_MM_FROUND_CUR_DIRECTION + return vreinterpretq_m128_f32(vrndiq_f32(vreinterpretq_f32_m128(a))); + } +#else + float *v_float = (float *) &a; + __m128 zero, neg_inf, pos_inf; + + switch (rounding) { + case (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC): + return _mm_cvtepi32_ps(_mm_cvtps_epi32(a)); + case (_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC): + return (__m128){floorf(v_float[0]), floorf(v_float[1]), + floorf(v_float[2]), floorf(v_float[3])}; + case (_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC): + return (__m128){ceilf(v_float[0]), ceilf(v_float[1]), ceilf(v_float[2]), + ceilf(v_float[3])}; + case (_MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC): + zero = _mm_set_ps(0.0f, 0.0f, 0.0f, 0.0f); + neg_inf = _mm_set_ps(floorf(v_float[0]), floorf(v_float[1]), + floorf(v_float[2]), floorf(v_float[3])); + pos_inf = _mm_set_ps(ceilf(v_float[0]), ceilf(v_float[1]), + ceilf(v_float[2]), ceilf(v_float[3])); + return _mm_blendv_ps(pos_inf, neg_inf, _mm_cmple_ps(a, zero)); + default: //_MM_FROUND_CUR_DIRECTION + return (__m128){roundf(v_float[0]), roundf(v_float[1]), + roundf(v_float[2]), roundf(v_float[3])}; + } +#endif +} + +// Convert packed single-precision (32-bit) floating-point elements in a to +// packed 32-bit integers, and store the results in dst. +// +// FOR j := 0 to 1 +// i := 32*j +// dst[i+31:i] := Convert_FP32_To_Int32(a[i+31:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_ps2pi +FORCE_INLINE __m64 _mm_cvt_ps2pi(__m128 a) +{ +#if defined(__aarch64__) + return vreinterpret_m64_s32( + vget_low_s32(vcvtnq_s32_f32(vreinterpretq_f32_m128(a)))); +#else + return vreinterpret_m64_s32( + vcvt_s32_f32(vget_low_f32(vreinterpretq_f32_m128( + _mm_round_ps(a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC))))); +#endif +} + +// Convert packed single-precision (32-bit) floating-point elements in a to +// packed 32-bit integers, and store the results in dst. +// +// FOR j := 0 to 1 +// i := 32*j +// dst[i+31:i] := Convert_FP32_To_Int32(a[i+31:i]) +// ENDFOR +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtps_pi32 +#define _mm_cvtps_pi32(a) _mm_cvt_ps2pi(a) + +// Round the packed single-precision (32-bit) floating-point elements in a up to +// an integer value, and store the results as packed single-precision +// floating-point elements in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_ceil_ps +FORCE_INLINE __m128 _mm_ceil_ps(__m128 a) +{ + return _mm_round_ps(a, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); +} + +// Round the lower single-precision (32-bit) floating-point element in b up to +// an integer value, store the result as a single-precision floating-point +// element in the lower element of dst, and copy the upper 3 packed elements +// from a to the upper elements of dst. +// +// dst[31:0] := CEIL(b[31:0]) +// dst[127:32] := a[127:32] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_ceil_ss +FORCE_INLINE __m128 _mm_ceil_ss(__m128 a, __m128 b) +{ + return _mm_move_ss( + a, _mm_round_ps(b, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC)); +} + +// Round the packed single-precision (32-bit) floating-point elements in a down +// to an integer value, and store the results as packed single-precision +// floating-point elements in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_floor_ps +FORCE_INLINE __m128 _mm_floor_ps(__m128 a) +{ + return _mm_round_ps(a, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); +} + +// Round the lower single-precision (32-bit) floating-point element in b down to +// an integer value, store the result as a single-precision floating-point +// element in the lower element of dst, and copy the upper 3 packed elements +// from a to the upper elements of dst. +// +// dst[31:0] := FLOOR(b[31:0]) +// dst[127:32] := a[127:32] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_floor_ss +FORCE_INLINE __m128 _mm_floor_ss(__m128 a, __m128 b) +{ + return _mm_move_ss( + a, _mm_round_ps(b, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC)); +} + +// Load 128-bits of integer data from unaligned memory into dst. This intrinsic +// may perform better than _mm_loadu_si128 when the data crosses a cache line +// boundary. +// +// dst[127:0] := MEM[mem_addr+127:mem_addr] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_lddqu_si128 #define _mm_lddqu_si128 _mm_loadu_si128 /* Miscellaneous Operations */ @@ -3313,7 +6039,7 @@ FORCE_INLINE __m128i _mm_loadu_si128(const __m128i *p) FORCE_INLINE __m128i _mm_sra_epi16(__m128i a, __m128i count) { int64_t c = (int64_t) vget_low_s64((int64x2_t) count); - if (c > 15) + if (unlikely(c > 15)) return _mm_cmplt_epi16(a, _mm_setzero_si128()); return vreinterpretq_m128i_s16(vshlq_s16((int16x8_t) a, vdupq_n_s16(-c))); } @@ -3330,7 +6056,7 @@ FORCE_INLINE __m128i _mm_sra_epi16(__m128i a, __m128i count) FORCE_INLINE __m128i _mm_sra_epi32(__m128i a, __m128i count) { int64_t c = (int64_t) vget_low_s64((int64x2_t) count); - if (c > 31) + if (unlikely(c > 31)) return _mm_cmplt_epi32(a, _mm_setzero_si128()); return vreinterpretq_m128i_s32(vshlq_s32((int32x4_t) a, vdupq_n_s32(-c))); } @@ -3399,8 +6125,8 @@ FORCE_INLINE __m128i _mm_packs_epi32(__m128i a, __m128i b) FORCE_INLINE __m128i _mm_packus_epi32(__m128i a, __m128i b) { return vreinterpretq_m128i_u16( - vcombine_u16(vqmovn_u32(vreinterpretq_u32_m128i(a)), - vqmovn_u32(vreinterpretq_u32_m128i(b)))); + vcombine_u16(vqmovun_s32(vreinterpretq_s32_m128i(a)), + vqmovun_s32(vreinterpretq_s32_m128i(b)))); } // Interleaves the lower 8 signed or unsigned 8-bit integers in a with the lower @@ -3505,6 +6231,52 @@ FORCE_INLINE __m128 _mm_unpacklo_ps(__m128 a, __m128 b) #endif } +// Unpack and interleave double-precision (64-bit) floating-point elements from +// the low half of a and b, and store the results in dst. +// +// DEFINE INTERLEAVE_QWORDS(src1[127:0], src2[127:0]) { +// dst[63:0] := src1[63:0] +// dst[127:64] := src2[63:0] +// RETURN dst[127:0] +// } +// dst[127:0] := INTERLEAVE_QWORDS(a[127:0], b[127:0]) +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_unpacklo_pd +FORCE_INLINE __m128d _mm_unpacklo_pd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) + return vreinterpretq_m128d_f64( + vzip1q_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#else + return vreinterpretq_m128d_s64( + vcombine_s64(vget_low_s64(vreinterpretq_s64_m128d(a)), + vget_low_s64(vreinterpretq_s64_m128d(b)))); +#endif +} + +// Unpack and interleave double-precision (64-bit) floating-point elements from +// the high half of a and b, and store the results in dst. +// +// DEFINE INTERLEAVE_HIGH_QWORDS(src1[127:0], src2[127:0]) { +// dst[63:0] := src1[127:64] +// dst[127:64] := src2[127:64] +// RETURN dst[127:0] +// } +// dst[127:0] := INTERLEAVE_HIGH_QWORDS(a[127:0], b[127:0]) +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_unpackhi_pd +FORCE_INLINE __m128d _mm_unpackhi_pd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) + return vreinterpretq_m128d_f64( + vzip2q_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#else + return vreinterpretq_m128d_s64( + vcombine_s64(vget_high_s64(vreinterpretq_s64_m128d(a)), + vget_high_s64(vreinterpretq_s64_m128d(b)))); +#endif +} + // Selects and interleaves the upper two single-precision, floating-point values // from a and b. // @@ -3624,7 +6396,7 @@ FORCE_INLINE __m128i _mm_unpackhi_epi64(__m128i a, __m128i b) // dst[18:16] := index[2:0] // dst[127:19] := 0 // -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_minpos_epu16&expand=3789 +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_minpos_epu16 FORCE_INLINE __m128i _mm_minpos_epu16(__m128i a) { __m128i dst; @@ -3633,15 +6405,15 @@ FORCE_INLINE __m128i _mm_minpos_epu16(__m128i a) #if defined(__aarch64__) min = vminvq_u16(vreinterpretq_u16_m128i(a)); #else - __m64i tmp; - tmp = vreinterpret_m64i_u16( + __m64 tmp; + tmp = vreinterpret_m64_u16( vmin_u16(vget_low_u16(vreinterpretq_u16_m128i(a)), vget_high_u16(vreinterpretq_u16_m128i(a)))); - tmp = vreinterpret_m64i_u16( - vpmin_u16(vreinterpret_u16_m64i(tmp), vreinterpret_u16_m64i(tmp))); - tmp = vreinterpret_m64i_u16( - vpmin_u16(vreinterpret_u16_m64i(tmp), vreinterpret_u16_m64i(tmp))); - min = vget_lane_u16(vreinterpret_u16_m64i(tmp), 0); + tmp = vreinterpret_m64_u16( + vpmin_u16(vreinterpret_u16_m64(tmp), vreinterpret_u16_m64(tmp))); + tmp = vreinterpret_m64_u16( + vpmin_u16(vreinterpret_u16_m64(tmp), vreinterpret_u16_m64(tmp))); + min = vget_lane_u16(vreinterpret_u16_m64(tmp), 0); #endif // Get the index of the minimum value int i; @@ -3661,13 +6433,30 @@ FORCE_INLINE __m128i _mm_minpos_epu16(__m128i a) return dst; } -// shift to right -// https://msdn.microsoft.com/en-us/library/bb514041(v=vs.120).aspx -// http://blog.csdn.net/hemmingway/article/details/44828303 -// Clang requires a macro here, as it is extremely picky about c being a -// literal. -#define _mm_alignr_epi8(a, b, c) \ - ((__m128i) vextq_s8((int8x16_t)(b), (int8x16_t)(a), (c))) +// Compute the bitwise AND of 128 bits (representing integer data) in a and b, +// and set ZF to 1 if the result is zero, otherwise set ZF to 0. Compute the +// bitwise NOT of a and then AND with b, and set CF to 1 if the result is zero, +// otherwise set CF to 0. Return the CF value. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_testc_si128 +FORCE_INLINE int _mm_testc_si128(__m128i a, __m128i b) +{ + int64x2_t s64 = + vandq_s64(vreinterpretq_s64_s32(vmvnq_s32(vreinterpretq_s32_m128i(a))), + vreinterpretq_s64_m128i(b)); + return !(vgetq_lane_s64(s64, 0) | vgetq_lane_s64(s64, 1)); +} + +// Compute the bitwise AND of 128 bits (representing integer data) in a and b, +// and set ZF to 1 if the result is zero, otherwise set ZF to 0. Compute the +// bitwise NOT of a and then AND with b, and set CF to 1 if the result is zero, +// otherwise set CF to 0. Return the ZF value. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_testz_si128 +FORCE_INLINE int _mm_testz_si128(__m128i a, __m128i b) +{ + int64x2_t s64 = + vandq_s64(vreinterpretq_s64_m128i(a), vreinterpretq_s64_m128i(b)); + return !(vgetq_lane_s64(s64, 0) | vgetq_lane_s64(s64, 1)); +} // Extracts the selected signed or unsigned 8-bit integer from a and zero // extends. @@ -3691,6 +6480,12 @@ FORCE_INLINE __m128i _mm_minpos_epu16(__m128i a) #define _mm_extract_epi16(a, imm) \ vgetq_lane_u16(vreinterpretq_u16_m128i(a), (imm)) +// Extract a 16-bit integer from a, selected with imm8, and store the result in +// the lower element of dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_extract_pi16 +#define _mm_extract_pi16(a, imm) \ + (int32_t) vget_lane_u16(vreinterpret_u16_m64(a), (imm)) + // Inserts the least significant 16 bits of b into the selected 16-bit integer // of a. // https://msdn.microsoft.com/en-us/library/kaze8hz1%28v=vs.100%29.aspx @@ -3702,6 +6497,15 @@ FORCE_INLINE __m128i _mm_minpos_epu16(__m128i a) vsetq_lane_s16((b), vreinterpretq_s16_m128i(a), (imm))); \ }) +// Copy a to dst, and insert the 16-bit integer i into dst at the location +// specified by imm8. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_insert_pi16 +#define _mm_insert_pi16(a, b, imm) \ + __extension__({ \ + vreinterpret_m64_s16( \ + vset_lane_s16((b), vreinterpret_s16_m64(a), (imm))); \ + }) + // Extracts the selected signed or unsigned 32-bit integer from a and zero // extends. // FORCE_INLINE int _mm_extract_epi32(__m128i a, __constrange(0,4) int imm) @@ -3796,18 +6600,19 @@ FORCE_INLINE int64_t _mm_popcnt_u64(uint64_t a) // Macro: Transpose the 4x4 matrix formed by the 4 rows of single-precision // (32-bit) floating-point elements in row0, row1, row2, and row3, and store the // transposed matrix in these vectors (row0 now contains column 0, etc.). -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=MM_TRANSPOSE4_PS&expand=5949 -#define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) \ - do { \ - __m128 tmp0, tmp1, tmp2, tmp3; \ - tmp0 = _mm_unpacklo_ps(row0, row1); \ - tmp2 = _mm_unpacklo_ps(row2, row3); \ - tmp1 = _mm_unpackhi_ps(row0, row1); \ - tmp3 = _mm_unpackhi_ps(row2, row3); \ - row0 = _mm_movelh_ps(tmp0, tmp2); \ - row1 = _mm_movehl_ps(tmp2, tmp0); \ - row2 = _mm_movelh_ps(tmp1, tmp3); \ - row3 = _mm_movehl_ps(tmp3, tmp1); \ +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=MM_TRANSPOSE4_PS +#define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) \ + do { \ + float32x4x2_t ROW01 = vtrnq_f32(row0, row1); \ + float32x4x2_t ROW23 = vtrnq_f32(row2, row3); \ + row0 = vcombine_f32(vget_low_f32(ROW01.val[0]), \ + vget_low_f32(ROW23.val[0])); \ + row1 = vcombine_f32(vget_low_f32(ROW01.val[1]), \ + vget_low_f32(ROW23.val[1])); \ + row2 = vcombine_f32(vget_high_f32(ROW01.val[0]), \ + vget_high_f32(ROW23.val[0])); \ + row3 = vcombine_f32(vget_high_f32(ROW01.val[1]), \ + vget_high_f32(ROW23.val[1])); \ } while (0) /* Crypto Extensions */ @@ -3927,6 +6732,9 @@ static uint64x2_t _sse2neon_vmull_p64(uint64x1_t _a, uint64x1_t _b) } #endif // ARMv7 polyfill +// Perform a carry-less multiplication of two 64-bit integers, selected from a +// and b according to imm8, and store the results in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_clmulepi64_si128 FORCE_INLINE __m128i _mm_clmulepi64_si128(__m128i _a, __m128i _b, const int imm) { uint64x2_t a = vreinterpretq_u64_m128i(_a); @@ -3949,7 +6757,55 @@ FORCE_INLINE __m128i _mm_clmulepi64_si128(__m128i _a, __m128i _b, const int imm) } } -#if !defined(__ARM_FEATURE_CRYPTO) && defined(__aarch64__) +#if !defined(__ARM_FEATURE_CRYPTO) +/* clang-format off */ +#define SSE2NEON_AES_DATA(w) \ + { \ + w(0x63), w(0x7c), w(0x77), w(0x7b), w(0xf2), w(0x6b), w(0x6f), \ + w(0xc5), w(0x30), w(0x01), w(0x67), w(0x2b), w(0xfe), w(0xd7), \ + w(0xab), w(0x76), w(0xca), w(0x82), w(0xc9), w(0x7d), w(0xfa), \ + w(0x59), w(0x47), w(0xf0), w(0xad), w(0xd4), w(0xa2), w(0xaf), \ + w(0x9c), w(0xa4), w(0x72), w(0xc0), w(0xb7), w(0xfd), w(0x93), \ + w(0x26), w(0x36), w(0x3f), w(0xf7), w(0xcc), w(0x34), w(0xa5), \ + w(0xe5), w(0xf1), w(0x71), w(0xd8), w(0x31), w(0x15), w(0x04), \ + w(0xc7), w(0x23), w(0xc3), w(0x18), w(0x96), w(0x05), w(0x9a), \ + w(0x07), w(0x12), w(0x80), w(0xe2), w(0xeb), w(0x27), w(0xb2), \ + w(0x75), w(0x09), w(0x83), w(0x2c), w(0x1a), w(0x1b), w(0x6e), \ + w(0x5a), w(0xa0), w(0x52), w(0x3b), w(0xd6), w(0xb3), w(0x29), \ + w(0xe3), w(0x2f), w(0x84), w(0x53), w(0xd1), w(0x00), w(0xed), \ + w(0x20), w(0xfc), w(0xb1), w(0x5b), w(0x6a), w(0xcb), w(0xbe), \ + w(0x39), w(0x4a), w(0x4c), w(0x58), w(0xcf), w(0xd0), w(0xef), \ + w(0xaa), w(0xfb), w(0x43), w(0x4d), w(0x33), w(0x85), w(0x45), \ + w(0xf9), w(0x02), w(0x7f), w(0x50), w(0x3c), w(0x9f), w(0xa8), \ + w(0x51), w(0xa3), w(0x40), w(0x8f), w(0x92), w(0x9d), w(0x38), \ + w(0xf5), w(0xbc), w(0xb6), w(0xda), w(0x21), w(0x10), w(0xff), \ + w(0xf3), w(0xd2), w(0xcd), w(0x0c), w(0x13), w(0xec), w(0x5f), \ + w(0x97), w(0x44), w(0x17), w(0xc4), w(0xa7), w(0x7e), w(0x3d), \ + w(0x64), w(0x5d), w(0x19), w(0x73), w(0x60), w(0x81), w(0x4f), \ + w(0xdc), w(0x22), w(0x2a), w(0x90), w(0x88), w(0x46), w(0xee), \ + w(0xb8), w(0x14), w(0xde), w(0x5e), w(0x0b), w(0xdb), w(0xe0), \ + w(0x32), w(0x3a), w(0x0a), w(0x49), w(0x06), w(0x24), w(0x5c), \ + w(0xc2), w(0xd3), w(0xac), w(0x62), w(0x91), w(0x95), w(0xe4), \ + w(0x79), w(0xe7), w(0xc8), w(0x37), w(0x6d), w(0x8d), w(0xd5), \ + w(0x4e), w(0xa9), w(0x6c), w(0x56), w(0xf4), w(0xea), w(0x65), \ + w(0x7a), w(0xae), w(0x08), w(0xba), w(0x78), w(0x25), w(0x2e), \ + w(0x1c), w(0xa6), w(0xb4), w(0xc6), w(0xe8), w(0xdd), w(0x74), \ + w(0x1f), w(0x4b), w(0xbd), w(0x8b), w(0x8a), w(0x70), w(0x3e), \ + w(0xb5), w(0x66), w(0x48), w(0x03), w(0xf6), w(0x0e), w(0x61), \ + w(0x35), w(0x57), w(0xb9), w(0x86), w(0xc1), w(0x1d), w(0x9e), \ + w(0xe1), w(0xf8), w(0x98), w(0x11), w(0x69), w(0xd9), w(0x8e), \ + w(0x94), w(0x9b), w(0x1e), w(0x87), w(0xe9), w(0xce), w(0x55), \ + w(0x28), w(0xdf), w(0x8c), w(0xa1), w(0x89), w(0x0d), w(0xbf), \ + w(0xe6), w(0x42), w(0x68), w(0x41), w(0x99), w(0x2d), w(0x0f), \ + w(0xb0), w(0x54), w(0xbb), w(0x16) \ + } +/* clang-format on */ + +/* X Macro trick. See https://en.wikipedia.org/wiki/X_Macro */ +#define SSE2NEON_AES_H0(x) (x) +static const uint8_t SSE2NEON_sbox[256] = SSE2NEON_AES_DATA(SSE2NEON_AES_H0); +#undef SSE2NEON_AES_H0 + // In the absence of crypto extensions, implement aesenc using regular neon // intrinsics instead. See: // https://www.workofard.com/2017/01/accelerated-aes-for-the-arm64-linux-kernel/ @@ -3958,29 +6814,7 @@ FORCE_INLINE __m128i _mm_clmulepi64_si128(__m128i _a, __m128i _b, const int imm) // for more information Reproduced with permission of the author. FORCE_INLINE __m128i _mm_aesenc_si128(__m128i EncBlock, __m128i RoundKey) { - static const uint8_t crypto_aes_sbox[256] = { - 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, - 0xfe, 0xd7, 0xab, 0x76, 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, - 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 0xb7, 0xfd, 0x93, 0x26, - 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, - 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, - 0xeb, 0x27, 0xb2, 0x75, 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, - 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 0x53, 0xd1, 0x00, 0xed, - 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, - 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, - 0x50, 0x3c, 0x9f, 0xa8, 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, - 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 0xcd, 0x0c, 0x13, 0xec, - 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, - 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, - 0xde, 0x5e, 0x0b, 0xdb, 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, - 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 0xe7, 0xc8, 0x37, 0x6d, - 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, - 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, - 0x4b, 0xbd, 0x8b, 0x8a, 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, - 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 0xe1, 0xf8, 0x98, 0x11, - 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, - 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, - 0xb0, 0x54, 0xbb, 0x16}; +#if defined(__aarch64__) static const uint8_t shift_rows[] = {0x0, 0x5, 0xa, 0xf, 0x4, 0x9, 0xe, 0x3, 0x8, 0xd, 0x2, 0x7, 0xc, 0x1, 0x6, 0xb}; @@ -3994,10 +6828,10 @@ FORCE_INLINE __m128i _mm_aesenc_si128(__m128i EncBlock, __m128i RoundKey) w = vqtbl1q_u8(w, vld1q_u8(shift_rows)); // sub bytes - v = vqtbl4q_u8(vld1q_u8_x4(crypto_aes_sbox), w); - v = vqtbx4q_u8(v, vld1q_u8_x4(crypto_aes_sbox + 0x40), w - 0x40); - v = vqtbx4q_u8(v, vld1q_u8_x4(crypto_aes_sbox + 0x80), w - 0x80); - v = vqtbx4q_u8(v, vld1q_u8_x4(crypto_aes_sbox + 0xc0), w - 0xc0); + v = vqtbl4q_u8(_sse2neon_vld1q_u8_x4(SSE2NEON_sbox), w); + v = vqtbx4q_u8(v, _sse2neon_vld1q_u8_x4(SSE2NEON_sbox + 0x40), w - 0x40); + v = vqtbx4q_u8(v, _sse2neon_vld1q_u8_x4(SSE2NEON_sbox + 0x80), w - 0x80); + v = vqtbx4q_u8(v, _sse2neon_vld1q_u8_x4(SSE2NEON_sbox + 0xc0), w - 0xc0); // mix columns w = (v << 1) ^ (uint8x16_t)(((int8x16_t) v >> 7) & 0x1b); @@ -4006,20 +6840,140 @@ FORCE_INLINE __m128i _mm_aesenc_si128(__m128i EncBlock, __m128i RoundKey) // add round key return vreinterpretq_m128i_u8(w) ^ RoundKey; + +#else /* ARMv7-A NEON implementation */ +#define SSE2NEON_AES_B2W(b0, b1, b2, b3) \ + (((uint32_t)(b3) << 24) | ((uint32_t)(b2) << 16) | ((uint32_t)(b1) << 8) | \ + (b0)) +#define SSE2NEON_AES_F2(x) ((x << 1) ^ (((x >> 7) & 1) * 0x011b /* WPOLY */)) +#define SSE2NEON_AES_F3(x) (SSE2NEON_AES_F2(x) ^ x) +#define SSE2NEON_AES_U0(p) \ + SSE2NEON_AES_B2W(SSE2NEON_AES_F2(p), p, p, SSE2NEON_AES_F3(p)) +#define SSE2NEON_AES_U1(p) \ + SSE2NEON_AES_B2W(SSE2NEON_AES_F3(p), SSE2NEON_AES_F2(p), p, p) +#define SSE2NEON_AES_U2(p) \ + SSE2NEON_AES_B2W(p, SSE2NEON_AES_F3(p), SSE2NEON_AES_F2(p), p) +#define SSE2NEON_AES_U3(p) \ + SSE2NEON_AES_B2W(p, p, SSE2NEON_AES_F3(p), SSE2NEON_AES_F2(p)) + static const uint32_t ALIGN_STRUCT(16) aes_table[4][256] = { + SSE2NEON_AES_DATA(SSE2NEON_AES_U0), + SSE2NEON_AES_DATA(SSE2NEON_AES_U1), + SSE2NEON_AES_DATA(SSE2NEON_AES_U2), + SSE2NEON_AES_DATA(SSE2NEON_AES_U3), + }; +#undef SSE2NEON_AES_B2W +#undef SSE2NEON_AES_F2 +#undef SSE2NEON_AES_F3 +#undef SSE2NEON_AES_U0 +#undef SSE2NEON_AES_U1 +#undef SSE2NEON_AES_U2 +#undef SSE2NEON_AES_U3 + + uint32_t x0 = _mm_cvtsi128_si32(EncBlock); + uint32_t x1 = _mm_cvtsi128_si32(_mm_shuffle_epi32(EncBlock, 0x55)); + uint32_t x2 = _mm_cvtsi128_si32(_mm_shuffle_epi32(EncBlock, 0xAA)); + uint32_t x3 = _mm_cvtsi128_si32(_mm_shuffle_epi32(EncBlock, 0xFF)); + + __m128i out = _mm_set_epi32( + (aes_table[0][x3 & 0xff] ^ aes_table[1][(x0 >> 8) & 0xff] ^ + aes_table[2][(x1 >> 16) & 0xff] ^ aes_table[3][x2 >> 24]), + (aes_table[0][x2 & 0xff] ^ aes_table[1][(x3 >> 8) & 0xff] ^ + aes_table[2][(x0 >> 16) & 0xff] ^ aes_table[3][x1 >> 24]), + (aes_table[0][x1 & 0xff] ^ aes_table[1][(x2 >> 8) & 0xff] ^ + aes_table[2][(x3 >> 16) & 0xff] ^ aes_table[3][x0 >> 24]), + (aes_table[0][x0 & 0xff] ^ aes_table[1][(x1 >> 8) & 0xff] ^ + aes_table[2][(x2 >> 16) & 0xff] ^ aes_table[3][x3 >> 24])); + + return _mm_xor_si128(out, RoundKey); +#endif } -#elif defined(__ARM_FEATURE_CRYPTO) + +// Perform the last round of an AES encryption flow on data (state) in a using +// the round key in RoundKey, and store the result in dst. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_aesenclast_si128 +FORCE_INLINE __m128i _mm_aesenclast_si128(__m128i a, __m128i RoundKey) +{ + /* FIXME: optimized for NEON */ + uint8_t v[4][4] = { + [0] = {SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 0)], + SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 5)], + SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 10)], + SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 15)]}, + [1] = {SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 4)], + SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 9)], + SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 14)], + SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 3)]}, + [2] = {SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 8)], + SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 13)], + SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 2)], + SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 7)]}, + [3] = {SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 12)], + SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 1)], + SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 6)], + SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 11)]}, + }; + for (int i = 0; i < 16; i++) + vreinterpretq_nth_u8_m128i(a, i) = + v[i / 4][i % 4] ^ vreinterpretq_nth_u8_m128i(RoundKey, i); + return a; +} + +// Emits the Advanced Encryption Standard (AES) instruction aeskeygenassist. +// This instruction generates a round key for AES encryption. See +// https://kazakov.life/2017/11/01/cryptocurrency-mining-on-ios-devices/ +// for details. +// +// https://msdn.microsoft.com/en-us/library/cc714138(v=vs.120).aspx +FORCE_INLINE __m128i _mm_aeskeygenassist_si128(__m128i key, const int rcon) +{ + uint32_t X1 = _mm_cvtsi128_si32(_mm_shuffle_epi32(key, 0x55)); + uint32_t X3 = _mm_cvtsi128_si32(_mm_shuffle_epi32(key, 0xFF)); + for (int i = 0; i < 4; ++i) { + ((uint8_t *) &X1)[i] = SSE2NEON_sbox[((uint8_t *) &X1)[i]]; + ((uint8_t *) &X3)[i] = SSE2NEON_sbox[((uint8_t *) &X3)[i]]; + } + return _mm_set_epi32(((X3 >> 8) | (X3 << 24)) ^ rcon, X3, + ((X1 >> 8) | (X1 << 24)) ^ rcon, X1); +} +#undef SSE2NEON_AES_DATA + +#else /* __ARM_FEATURE_CRYPTO */ // Implements equivalent of 'aesenc' by combining AESE (with an empty key) and -// AESMC and then manually applying the real key as an xor operation This +// AESMC and then manually applying the real key as an xor operation. This // unfortunately means an additional xor op; the compiler should be able to -// optimise this away for repeated calls however See +// optimize this away for repeated calls however. See // https://blog.michaelbrase.com/2018/05/08/emulating-x86-aes-intrinsics-on-armv8-a // for more details. -inline __m128i _mm_aesenc_si128(__m128i a, __m128i b) +FORCE_INLINE __m128i _mm_aesenc_si128(__m128i a, __m128i b) { return vreinterpretq_m128i_u8( vaesmcq_u8(vaeseq_u8(vreinterpretq_u8_m128i(a), vdupq_n_u8(0))) ^ vreinterpretq_u8_m128i(b)); } + +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_aesenclast_si128 +FORCE_INLINE __m128i _mm_aesenclast_si128(__m128i a, __m128i RoundKey) +{ + return _mm_xor_si128(vreinterpretq_m128i_u8(vaeseq_u8( + vreinterpretq_u8_m128i(a), vdupq_n_u8(0))), + RoundKey); +} + +FORCE_INLINE __m128i _mm_aeskeygenassist_si128(__m128i a, const int rcon) +{ + // AESE does ShiftRows and SubBytes on A + uint8x16_t u8 = vaeseq_u8(vreinterpretq_u8_m128i(a), vdupq_n_u8(0)); + + uint8x16_t dest = { + // Undo ShiftRows step from AESE and extract X1 and X3 + u8[0x4], u8[0x1], u8[0xE], u8[0xB], // SubBytes(X1) + u8[0x1], u8[0xE], u8[0xB], u8[0x4], // ROT(SubBytes(X1)) + u8[0xC], u8[0x9], u8[0x6], u8[0x3], // SubBytes(X3) + u8[0x9], u8[0x6], u8[0x3], u8[0xC], // ROT(SubBytes(X3)) + }; + uint32x4_t r = {0, (unsigned) rcon, 0, (unsigned) rcon}; + return vreinterpretq_m128i_u8(dest) ^ vreinterpretq_m128i_u32(r); +} #endif /* Streaming Extensions */ @@ -4032,9 +6986,21 @@ FORCE_INLINE void _mm_sfence(void) __sync_synchronize(); } +// Store 128-bits (composed of 4 packed single-precision (32-bit) floating- +// point elements) from a into memory using a non-temporal memory hint. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_stream_ps +FORCE_INLINE void _mm_stream_ps(float *p, __m128 a) +{ +#if __has_builtin(__builtin_nontemporal_store) + __builtin_nontemporal_store(a, (float32x4_t *) p); +#else + vst1q_f32(p, vreinterpretq_f32_m128(a)); +#endif +} + // Stores the data in a to the address p without polluting the caches. If the // cache line containing address p is already in the cache, the cache will be -// updated.Address p must be 16 - byte aligned. +// updated. // https://msdn.microsoft.com/en-us/library/ba08y07y%28v=vs.90%29.aspx FORCE_INLINE void _mm_stream_si128(__m128i *p, __m128i a) { @@ -4045,6 +7011,22 @@ FORCE_INLINE void _mm_stream_si128(__m128i *p, __m128i a) #endif } +// Load 128-bits of integer data from memory into dst using a non-temporal +// memory hint. mem_addr must be aligned on a 16-byte boundary or a +// general-protection exception may be generated. +// +// dst[127:0] := MEM[mem_addr+127:mem_addr] +// +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_stream_load_si128 +FORCE_INLINE __m128i _mm_stream_load_si128(__m128i *p) +{ +#if __has_builtin(__builtin_nontemporal_store) + return __builtin_nontemporal_load(p); +#else + return vreinterpretq_m128i_s64(vld1q_s64((int64_t *) p)); +#endif +} + // Cache line containing p is flushed and invalidated from all caches in the // coherency domain. : // https://msdn.microsoft.com/en-us/library/ba08y07y(v=vs.100).aspx @@ -4069,6 +7051,22 @@ FORCE_INLINE void *_mm_malloc(size_t size, size_t align) return NULL; } +// Conditionally store 8-bit integer elements from a into memory using mask +// (elements are not stored when the highest bit is not set in the corresponding +// element) and a non-temporal memory hint. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskmove_si64 +FORCE_INLINE void _mm_maskmove_si64(__m64 a, __m64 mask, char *mem_addr) +{ + int8x8_t shr_mask = vshr_n_s8(vreinterpret_s8_m64(mask), 7); + __m128 b = _mm_load_ps((const float *) mem_addr); + int8x8_t masked = + vbsl_s8(vreinterpret_u8_s8(shr_mask), vreinterpret_s8_m64(a), + vreinterpret_s8_u64(vget_low_u64(vreinterpretq_u64_m128(b)))); + vst1_s8((int8_t *) mem_addr, masked); +} + +// Free aligned memory that was allocated with _mm_malloc. +// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_free FORCE_INLINE void _mm_free(void *addr) { free(addr); @@ -4087,7 +7085,7 @@ FORCE_INLINE uint32_t _mm_crc32_u8(uint32_t crc, uint8_t v) crc ^= v; for (int bit = 0; bit < 8; bit++) { if (crc & 1) - crc = (crc >> 1) ^ uint32_t(0x82f63b78); + crc = (crc >> 1) ^ UINT32_C(0x82f63b78); else crc = (crc >> 1); } @@ -4148,4 +7146,8 @@ FORCE_INLINE uint64_t _mm_crc32_u64(uint64_t crc, uint64_t v) #pragma pop_macro("FORCE_INLINE") #endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC pop_options +#endif + #endif From a5a7ee716d25f4df1736378412f3c64b3403209b Mon Sep 17 00:00:00 2001 From: XMRig Date: Wed, 3 Mar 2021 19:38:54 +0700 Subject: [PATCH 08/16] Update build scripts. --- scripts/build.hwloc.sh | 2 +- scripts/build.openssl.sh | 2 +- scripts/build.uv.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build.hwloc.sh b/scripts/build.hwloc.sh index 1525bd4e..66809075 100755 --- a/scripts/build.hwloc.sh +++ b/scripts/build.hwloc.sh @@ -1,6 +1,6 @@ #!/bin/bash -e -HWLOC_VERSION="2.4.0" +HWLOC_VERSION="2.4.1" mkdir -p deps mkdir -p deps/include diff --git a/scripts/build.openssl.sh b/scripts/build.openssl.sh index faeca1b2..979fdfec 100755 --- a/scripts/build.openssl.sh +++ b/scripts/build.openssl.sh @@ -1,6 +1,6 @@ #!/bin/bash -e -OPENSSL_VERSION="1.1.1i" +OPENSSL_VERSION="1.1.1j" mkdir -p deps mkdir -p deps/include diff --git a/scripts/build.uv.sh b/scripts/build.uv.sh index c240bce1..6fb5d9bf 100755 --- a/scripts/build.uv.sh +++ b/scripts/build.uv.sh @@ -1,6 +1,6 @@ #!/bin/bash -e -UV_VERSION="1.40.0" +UV_VERSION="1.41.0" mkdir -p deps mkdir -p deps/include From 1f9cdc0564f264c5c4f0d50bb4e24d6a82ab9943 Mon Sep 17 00:00:00 2001 From: XMRig Date: Thu, 4 Mar 2021 03:23:26 +0700 Subject: [PATCH 09/16] Update hwloc for MSVC. --- src/3rdparty/hwloc/include/hwloc.h | 53 +++-- .../hwloc/include/hwloc/autogen/config.h | 4 +- src/3rdparty/hwloc/include/hwloc/opencl.h | 9 +- src/3rdparty/hwloc/include/hwloc/rsmi.h | 201 ++++++++++++++++++ src/3rdparty/hwloc/src/cpukinds.c | 4 +- src/3rdparty/hwloc/src/static-components.h | 7 +- src/3rdparty/hwloc/src/topology-x86.c | 16 +- src/3rdparty/hwloc/src/topology-xml.c | 3 + src/3rdparty/hwloc/src/topology.c | 55 ++++- 9 files changed, 301 insertions(+), 51 deletions(-) create mode 100644 src/3rdparty/hwloc/include/hwloc/rsmi.h diff --git a/src/3rdparty/hwloc/include/hwloc.h b/src/3rdparty/hwloc/include/hwloc.h index 9c8c86cc..261626f4 100644 --- a/src/3rdparty/hwloc/include/hwloc.h +++ b/src/3rdparty/hwloc/include/hwloc.h @@ -1,6 +1,6 @@ /* * Copyright © 2009 CNRS - * Copyright © 2009-2020 Inria. All rights reserved. + * Copyright © 2009-2021 Inria. All rights reserved. * Copyright © 2009-2012 Université Bordeaux * Copyright © 2009-2020 Cisco Systems, Inc. All rights reserved. * See COPYING in top-level directory. @@ -2362,22 +2362,9 @@ HWLOC_DECLSPEC hwloc_obj_t hwloc_topology_insert_misc_object(hwloc_topology_t to /** \brief Allocate a Group object to insert later with hwloc_topology_insert_group_object(). * * This function returns a new Group object. - * The caller should (at least) initialize its sets before inserting the object. - * See hwloc_topology_insert_group_object(). * - * The \p subtype object attribute may be set to display something else - * than "Group" as the type name for this object in lstopo. - * Custom name/value info pairs may be added with hwloc_obj_add_info() after - * insertion. - * - * The \p kind group attribute should be 0. The \p subkind group attribute may - * be set to identify multiple Groups of the same level. - * - * It is recommended not to set any other object attribute before insertion, - * since the Group may get discarded during insertion. - * - * The object will be destroyed if passed to hwloc_topology_insert_group_object() - * without any set defined. + * The caller should (at least) initialize its sets before inserting + * the object in the topology. See hwloc_topology_insert_group_object(). */ HWLOC_DECLSPEC hwloc_obj_t hwloc_topology_alloc_group_object(hwloc_topology_t topology); @@ -2388,34 +2375,44 @@ HWLOC_DECLSPEC hwloc_obj_t hwloc_topology_alloc_group_object(hwloc_topology_t to * the final location of the Group in the topology. * Then the object can be passed to this function for actual insertion in the topology. * - * The group \p dont_merge attribute may be set to prevent the core from - * ever merging this object with another object hierarchically-identical. - * * Either the cpuset or nodeset field (or both, if compatible) must be set * to a non-empty bitmap. The complete_cpuset or complete_nodeset may be set * instead if inserting with respect to the complete topology * (including disallowed, offline or unknown objects). - * - * It grouping several objects, hwloc_obj_add_other_obj_sets() is an easy way + * If grouping several objects, hwloc_obj_add_other_obj_sets() is an easy way * to build the Group sets iteratively. - * * These sets cannot be larger than the current topology, or they would get * restricted silently. - * * The core will setup the other sets after actual insertion. * + * The \p subtype object attribute may be defined (to a dynamically + * allocated string) to display something else than "Group" as the + * type name for this object in lstopo. + * Custom name/value info pairs may be added with hwloc_obj_add_info() after + * insertion. + * + * The group \p dont_merge attribute may be set to \c 1 to prevent + * the hwloc core from ever merging this object with another + * hierarchically-identical object. + * This is useful when the Group itself describes an important feature + * that cannot be exposed anywhere else in the hierarchy. + * + * The group \p kind attribute may be set to a high value such + * as \c 0xffffffff to tell hwloc that this new Group should always + * be discarded in favor of any existing Group with the same locality. + * * \return The inserted object if it was properly inserted. * - * \return An existing object if the Group was discarded because the topology already - * contained an object at the same location (the Group did not add any locality information). - * Any name/info key pair set before inserting is appended to the existing object. + * \return An existing object if the Group was merged or discarded + * because the topology already contained an object at the same + * location (the Group did not add any hierarchy information). * * \return \c NULL if the insertion failed because of conflicting sets in topology tree. * * \return \c NULL if Group objects are filtered-out of the topology (::HWLOC_TYPE_FILTER_KEEP_NONE). * - * \return \c NULL if the object was discarded because no set was initialized in the Group - * before insert, or all of them were empty. + * \return \c NULL if the object was discarded because no set was + * initialized in the Group before insert, or all of them were empty. */ HWLOC_DECLSPEC hwloc_obj_t hwloc_topology_insert_group_object(hwloc_topology_t topology, hwloc_obj_t group); diff --git a/src/3rdparty/hwloc/include/hwloc/autogen/config.h b/src/3rdparty/hwloc/include/hwloc/autogen/config.h index 8b69185f..e490466b 100644 --- a/src/3rdparty/hwloc/include/hwloc/autogen/config.h +++ b/src/3rdparty/hwloc/include/hwloc/autogen/config.h @@ -11,10 +11,10 @@ #ifndef HWLOC_CONFIG_H #define HWLOC_CONFIG_H -#define HWLOC_VERSION "2.4.0" +#define HWLOC_VERSION "2.4.1" #define HWLOC_VERSION_MAJOR 2 #define HWLOC_VERSION_MINOR 4 -#define HWLOC_VERSION_RELEASE 0 +#define HWLOC_VERSION_RELEASE 1 #define HWLOC_VERSION_GREEK "" #define __hwloc_restrict diff --git a/src/3rdparty/hwloc/include/hwloc/opencl.h b/src/3rdparty/hwloc/include/hwloc/opencl.h index d498d606..9a2fdacb 100644 --- a/src/3rdparty/hwloc/include/hwloc/opencl.h +++ b/src/3rdparty/hwloc/include/hwloc/opencl.h @@ -1,5 +1,5 @@ /* - * Copyright © 2012-2020 Inria. All rights reserved. + * Copyright © 2012-2021 Inria. All rights reserved. * Copyright © 2013, 2018 Université Bordeaux. All right reserved. * See COPYING in top-level directory. */ @@ -82,9 +82,10 @@ hwloc_opencl_get_device_pci_busid(cl_device_id device, if (CL_SUCCESS == clret && HWLOC_CL_DEVICE_TOPOLOGY_TYPE_PCIE_AMD == amdtopo.raw.type) { *domain = 0; /* can't do anything better */ - *bus = (unsigned) amdtopo.pcie.bus; - *dev = (unsigned) amdtopo.pcie.device; - *func = (unsigned) amdtopo.pcie.function; + /* cl_device_topology_amd stores bus ID in cl_char, dont convert those signed char directly to unsigned int */ + *bus = (unsigned) (unsigned char) amdtopo.pcie.bus; + *dev = (unsigned) (unsigned char) amdtopo.pcie.device; + *func = (unsigned) (unsigned char) amdtopo.pcie.function; return 0; } diff --git a/src/3rdparty/hwloc/include/hwloc/rsmi.h b/src/3rdparty/hwloc/include/hwloc/rsmi.h new file mode 100644 index 00000000..a6d55b3c --- /dev/null +++ b/src/3rdparty/hwloc/include/hwloc/rsmi.h @@ -0,0 +1,201 @@ +/* + * Copyright © 2012-2020 Inria. All rights reserved. + * Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved. + * Written by Advanced Micro Devices, + * See COPYING in top-level directory. + */ + +/** \file + * \brief Macros to help interaction between hwloc and the ROCm SMI Management Library. + * + * Applications that use both hwloc and the ROCm SMI Management Library may want to + * include this file so as to get topology information for AMD GPU devices. + */ + +#ifndef HWLOC_RSMI_H +#define HWLOC_RSMI_H + +#include "hwloc.h" +#include "hwloc/autogen/config.h" +#include "hwloc/helper.h" +#ifdef HWLOC_LINUX_SYS +#include "hwloc/linux.h" +#endif + +#include + + +#ifdef __cplusplus +extern "C" { +#endif + + +/** \defgroup hwlocality_rsmi Interoperability with the ROCm SMI Management Library + * + * This interface offers ways to retrieve topology information about + * devices managed by the ROCm SMI Management Library. + * + * @{ + */ + +/** \brief Get the CPU set of logical processors that are physically + * close to AMD GPU device whose index is \p dv_ind. + * + * Return the CPU set describing the locality of the AMD GPU device + * whose index is \p dv_ind. + * + * Topology \p topology and device \p dv_ind must match the local machine. + * I/O devices detection and the ROCm SMI component are not needed in the + * topology. + * + * The function only returns the locality of the device. + * If more information about the device is needed, OS objects should + * be used instead, see hwloc_rsmi_get_device_osdev() + * and hwloc_rsmi_get_device_osdev_by_index(). + * + * This function is currently only implemented in a meaningful way for + * Linux; other systems will simply get a full cpuset. + */ +static __hwloc_inline int +hwloc_rsmi_get_device_cpuset(hwloc_topology_t topology __hwloc_attribute_unused, + uint32_t dv_ind, hwloc_cpuset_t set) +{ +#ifdef HWLOC_LINUX_SYS + /* If we're on Linux, use the sysfs mechanism to get the local cpus */ +#define HWLOC_RSMI_DEVICE_SYSFS_PATH_MAX 128 + char path[HWLOC_RSMI_DEVICE_SYSFS_PATH_MAX]; + rsmi_status_t ret; + uint64_t bdfid = 0; + unsigned domain, device, bus; + + if (!hwloc_topology_is_thissystem(topology)) { + errno = EINVAL; + return -1; + } + + ret = rsmi_dev_pci_id_get(dv_ind, &bdfid); + if (RSMI_STATUS_SUCCESS != ret) { + errno = EINVAL; + return -1; + } + domain = (bdfid>>32) & 0xffffffff; + bus = ((bdfid & 0xffff)>>8) & 0xff; + device = ((bdfid & 0xff)>>3) & 0x1f; + + sprintf(path, "/sys/bus/pci/devices/%04x:%02x:%02x.0/local_cpus", domain, bus, device); + if (hwloc_linux_read_path_as_cpumask(path, set) < 0 + || hwloc_bitmap_iszero(set)) + hwloc_bitmap_copy(set, hwloc_topology_get_complete_cpuset(topology)); +#else + /* Non-Linux systems simply get a full cpuset */ + hwloc_bitmap_copy(set, hwloc_topology_get_complete_cpuset(topology)); +#endif + return 0; +} + +/** \brief Get the hwloc OS device object corresponding to the + * AMD GPU device whose index is \p dv_ind. + * + * Return the OS device object describing the AMD GPU device whose + * index is \p dv_ind. Returns NULL if there is none. + * + * The topology \p topology does not necessarily have to match the current + * machine. For instance the topology may be an XML import of a remote host. + * I/O devices detection and the ROCm SMI component must be enabled in the + * topology. + * + * \note The corresponding PCI device object can be obtained by looking + * at the OS device parent object (unless PCI devices are filtered out). + */ +static __hwloc_inline hwloc_obj_t +hwloc_rsmi_get_device_osdev_by_index(hwloc_topology_t topology, uint32_t dv_ind) +{ + hwloc_obj_t osdev = NULL; + while ((osdev = hwloc_get_next_osdev(topology, osdev)) != NULL) { + if (HWLOC_OBJ_OSDEV_GPU == osdev->attr->osdev.type + && osdev->name + && !strncmp("rsmi", osdev->name, 4) + && atoi(osdev->name + 4) == (int) dv_ind) + return osdev; + } + return NULL; +} + +/** \brief Get the hwloc OS device object corresponding to AMD GPU device, + * whose index is \p dv_ind. + * + * Return the hwloc OS device object that describes the given + * AMD GPU, whose index is \p dv_ind Return NULL if there is none. + * + * Topology \p topology and device \p dv_ind must match the local machine. + * I/O devices detection and the ROCm SMI component must be enabled in the + * topology. If not, the locality of the object may still be found using + * hwloc_rsmi_get_device_cpuset(). + * + * \note The corresponding hwloc PCI device may be found by looking + * at the result parent pointer (unless PCI devices are filtered out). + */ +static __hwloc_inline hwloc_obj_t +hwloc_rsmi_get_device_osdev(hwloc_topology_t topology, uint32_t dv_ind) +{ + hwloc_obj_t osdev; + rsmi_status_t ret; + uint64_t bdfid = 0; + unsigned domain, device, bus, func; + uint64_t id; + char uuid[64]; + + if (!hwloc_topology_is_thissystem(topology)) { + errno = EINVAL; + return NULL; + } + + ret = rsmi_dev_pci_id_get(dv_ind, &bdfid); + if (RSMI_STATUS_SUCCESS != ret) { + errno = EINVAL; + return NULL; + } + domain = (bdfid>>32) & 0xffffffff; + bus = ((bdfid & 0xffff)>>8) & 0xff; + device = ((bdfid & 0xff)>>3) & 0x1f; + func = bdfid & 0x7; + + ret = rsmi_dev_unique_id_get(dv_ind, &id); + if (RSMI_STATUS_SUCCESS != ret) + uuid[0] = '\0'; + else + sprintf(uuid, "%lx", id); + + osdev = NULL; + while ((osdev = hwloc_get_next_osdev(topology, osdev)) != NULL) { + hwloc_obj_t pcidev = osdev->parent; + const char *info; + + if (strncmp(osdev->name, "rsmi", 4)) + continue; + + if (pcidev + && pcidev->type == HWLOC_OBJ_PCI_DEVICE + && pcidev->attr->pcidev.domain == domain + && pcidev->attr->pcidev.bus == bus + && pcidev->attr->pcidev.dev == device + && pcidev->attr->pcidev.func == func) + return osdev; + + info = hwloc_obj_get_info_by_name(osdev, "AMDUUID"); + if (info && !strcmp(info, uuid)) + return osdev; + } + + return NULL; +} + +/** @} */ + + +#ifdef __cplusplus +} /* extern "C" */ +#endif + + +#endif /* HWLOC_RSMI_H */ diff --git a/src/3rdparty/hwloc/src/cpukinds.c b/src/3rdparty/hwloc/src/cpukinds.c index 5f2dd1aa..ef6297d7 100644 --- a/src/3rdparty/hwloc/src/cpukinds.c +++ b/src/3rdparty/hwloc/src/cpukinds.c @@ -1,5 +1,5 @@ /* - * Copyright © 2020 Inria. All rights reserved. + * Copyright © 2020-2021 Inria. All rights reserved. * See COPYING in top-level directory. */ @@ -270,7 +270,7 @@ hwloc__cpukinds_check_duplicate_rankings(struct hwloc_topology *topology) unsigned i,j; for(i=0; inr_cpukinds; i++) for(j=i+1; jnr_cpukinds; j++) - if (topology->cpukinds[i].forced_efficiency == topology->cpukinds[j].forced_efficiency) + if (topology->cpukinds[i].ranking_value == topology->cpukinds[j].ranking_value) /* if any duplicate, fail */ return -1; return 0; diff --git a/src/3rdparty/hwloc/src/static-components.h b/src/3rdparty/hwloc/src/static-components.h index f2cb254a..dac227a6 100644 --- a/src/3rdparty/hwloc/src/static-components.h +++ b/src/3rdparty/hwloc/src/static-components.h @@ -1,4 +1,9 @@ -#include +HWLOC_DECLSPEC extern const struct hwloc_component hwloc_noos_component; +HWLOC_DECLSPEC extern const struct hwloc_component hwloc_xml_component; +HWLOC_DECLSPEC extern const struct hwloc_component hwloc_synthetic_component; +HWLOC_DECLSPEC extern const struct hwloc_component hwloc_xml_nolibxml_component; +HWLOC_DECLSPEC extern const struct hwloc_component hwloc_windows_component; +HWLOC_DECLSPEC extern const struct hwloc_component hwloc_x86_component; static const struct hwloc_component * hwloc_static_components[] = { &hwloc_noos_component, &hwloc_xml_component, diff --git a/src/3rdparty/hwloc/src/topology-x86.c b/src/3rdparty/hwloc/src/topology-x86.c index 267384ee..71e396e2 100644 --- a/src/3rdparty/hwloc/src/topology-x86.c +++ b/src/3rdparty/hwloc/src/topology-x86.c @@ -1,5 +1,5 @@ /* - * Copyright © 2010-2020 Inria. All rights reserved. + * Copyright © 2010-2021 Inria. All rights reserved. * Copyright © 2010-2013 Université Bordeaux * Copyright © 2010-2011 Cisco Systems, Inc. All rights reserved. * See COPYING in top-level directory. @@ -908,6 +908,16 @@ static void summarize(struct hwloc_backend *backend, struct procinfo *infos, uns int gotnuma = 0; int fulldiscovery = (flags & HWLOC_X86_DISC_FLAG_FULL); +#ifdef HWLOC_DEBUG + hwloc_debug("\nSummary of x86 CPUID topology:\n"); + for(i=0; ilevels[0][0]->cpuset) { - /* somebody else discovered things */ + /* somebody else discovered things, reconnect levels so that we can look at them */ + hwloc_topology_reconnect(topology, 0); if (topology->nb_levels == 2 && topology->level_nbobjects[1] == data->nbprocs) { /* only PUs were discovered, as much as we would, complete the topology with everything else */ alreadypus = 1; @@ -1595,7 +1606,6 @@ hwloc_x86_discover(struct hwloc_backend *backend, struct hwloc_disc_status *dsta } /* several object types were added, we can't easily complete, just do partial discovery */ - hwloc_topology_reconnect(topology, 0); ret = hwloc_look_x86(backend, flags); if (ret) hwloc_obj_add_info(topology->levels[0][0], "Backend", "x86"); diff --git a/src/3rdparty/hwloc/src/topology-xml.c b/src/3rdparty/hwloc/src/topology-xml.c index fe04dd94..6aacc052 100644 --- a/src/3rdparty/hwloc/src/topology-xml.c +++ b/src/3rdparty/hwloc/src/topology-xml.c @@ -1462,6 +1462,9 @@ hwloc__xml_v2import_distances(hwloc_topology_t topology, unsigned long long u; if (heterotypes) { hwloc_obj_type_t t = HWLOC_OBJ_TYPE_NONE; + if (!*tmp) + /* reached the end of this indexes attribute */ + break; if (hwloc_type_sscanf(tmp, &t, NULL, 0) < 0) { if (hwloc__xml_verbose()) fprintf(stderr, "%s: %s with unrecognized heterogeneous type %s\n", diff --git a/src/3rdparty/hwloc/src/topology.c b/src/3rdparty/hwloc/src/topology.c index 94387ece..3944f3c1 100644 --- a/src/3rdparty/hwloc/src/topology.c +++ b/src/3rdparty/hwloc/src/topology.c @@ -1,6 +1,6 @@ /* * Copyright © 2009 CNRS - * Copyright © 2009-2020 Inria. All rights reserved. + * Copyright © 2009-2021 Inria. All rights reserved. * Copyright © 2009-2012, 2020 Université Bordeaux * Copyright © 2009-2011 Cisco Systems, Inc. All rights reserved. * See COPYING in top-level directory. @@ -567,8 +567,9 @@ hwloc_free_unlinked_object(hwloc_obj_t obj) } /* Replace old with contents of new object, and make new freeable by the caller. - * Only updates next_sibling/first_child pointers, - * so may only be used during early discovery. + * Requires reconnect (for siblings pointers and group depth), + * fixup of sets (only the main cpuset was likely compared before merging), + * and update of total_memory and group depth. */ static void hwloc_replace_linked_object(hwloc_obj_t old, hwloc_obj_t new) @@ -1348,7 +1349,7 @@ merge_insert_equal(hwloc_obj_t new, hwloc_obj_t old) /* returns the result of merge, or NULL if not merged */ static __hwloc_inline hwloc_obj_t -hwloc__insert_try_merge_group(hwloc_obj_t old, hwloc_obj_t new) +hwloc__insert_try_merge_group(hwloc_topology_t topology, hwloc_obj_t old, hwloc_obj_t new) { if (new->type == HWLOC_OBJ_GROUP && old->type == HWLOC_OBJ_GROUP) { /* which group do we keep? */ @@ -1359,6 +1360,7 @@ hwloc__insert_try_merge_group(hwloc_obj_t old, hwloc_obj_t new) /* keep the new one, it doesn't want to be merged */ hwloc_replace_linked_object(old, new); + topology->modified = 1; return new; } else { @@ -1366,9 +1368,12 @@ hwloc__insert_try_merge_group(hwloc_obj_t old, hwloc_obj_t new) /* keep the old one, it doesn't want to be merged */ return old; - /* compare subkinds to decice who to keep */ - if (new->attr->group.kind < old->attr->group.kind) + /* compare subkinds to decide which group to keep */ + if (new->attr->group.kind < old->attr->group.kind) { + /* keep smaller kind */ hwloc_replace_linked_object(old, new); + topology->modified = 1; + } return old; } } @@ -1394,6 +1399,7 @@ hwloc__insert_try_merge_group(hwloc_obj_t old, hwloc_obj_t new) * and let the caller free the new object */ hwloc_replace_linked_object(old, new); + topology->modified = 1; return old; } else { @@ -1435,7 +1441,7 @@ hwloc___insert_object_by_cpuset(struct hwloc_topology *topology, hwloc_obj_t cur int setres = res; if (res == HWLOC_OBJ_EQUAL) { - hwloc_obj_t merged = hwloc__insert_try_merge_group(child, obj); + hwloc_obj_t merged = hwloc__insert_try_merge_group(topology, child, obj); if (merged) return merged; /* otherwise compare actual types to decide of the inclusion */ @@ -1931,12 +1937,24 @@ hwloc_topology_insert_group_object(struct hwloc_topology *topology, hwloc_obj_t if (!res) return NULL; - if (res != obj) - /* merged */ + + if (res != obj && res->type != HWLOC_OBJ_GROUP) + /* merged, not into a Group, nothing to update */ return res; + /* res == obj means that the object was inserted. + * We need to reconnect levels, fill all its cpu/node sets, + * compute its total memory, group depth, etc. + * + * res != obj usually means that our new group was merged into an + * existing object, no need to recompute anything. + * However, if merging with an existing group, depending on their kinds, + * the contents of obj may overwrite the contents of the old group. + * This requires reconnecting levels, filling sets, recomputing total memory, etc. + */ + /* properly inserted */ - hwloc_obj_add_children_sets(obj); + hwloc_obj_add_children_sets(res); if (hwloc_topology_reconnect(topology, 0) < 0) return NULL; @@ -1948,7 +1966,7 @@ hwloc_topology_insert_group_object(struct hwloc_topology *topology, hwloc_obj_t #endif hwloc_topology_check(topology); - return obj; + return res; } hwloc_obj_t @@ -4658,6 +4676,9 @@ hwloc__check_misc_children(hwloc_topology_t topology, hwloc_bitmap_t gp_indexes, static void hwloc__check_object(hwloc_topology_t topology, hwloc_bitmap_t gp_indexes, hwloc_obj_t obj) { + hwloc_uint64_t total_memory; + hwloc_obj_t child; + assert(!hwloc_bitmap_isset(gp_indexes, obj->gp_index)); hwloc_bitmap_set(gp_indexes, obj->gp_index); @@ -4715,6 +4736,18 @@ hwloc__check_object(hwloc_topology_t topology, hwloc_bitmap_t gp_indexes, hwloc_ assert(hwloc_cache_type_by_depth_type(obj->attr->cache.depth, obj->attr->cache.type) == obj->type); } + /* check total memory */ + total_memory = 0; + if (obj->type == HWLOC_OBJ_NUMANODE) + total_memory += obj->attr->numanode.local_memory; + for_each_child(child, obj) { + total_memory += child->total_memory; + } + for_each_memory_child(child, obj) { + total_memory += child->total_memory; + } + assert(total_memory == obj->total_memory); + /* check children */ hwloc__check_normal_children(topology, gp_indexes, obj); hwloc__check_memory_children(topology, gp_indexes, obj); From 70cddc06ba9b690f8ef3c27f378504cd7705fcdf Mon Sep 17 00:00:00 2001 From: SChernykh Date: Thu, 4 Mar 2021 10:02:35 +0100 Subject: [PATCH 10/16] Fix crash in cn-heavy on Zen3 with manual thread count --- src/backend/cpu/CpuWorker.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backend/cpu/CpuWorker.cpp b/src/backend/cpu/CpuWorker.cpp index aed79190..6263f89a 100644 --- a/src/backend/cpu/CpuWorker.cpp +++ b/src/backend/cpu/CpuWorker.cpp @@ -86,7 +86,9 @@ xmrig::CpuWorker::CpuWorker(size_t id, const CpuLaunchData &data) : if ((N == 1) && (m_av == CnHash::AV_SINGLE) && (m_algorithm.family() == Algorithm::CN_HEAVY) && (m_assembly != Assembly::NONE) && (Cpu::info()->arch() == ICpuInfo::ARCH_ZEN3)) { std::lock_guard lock(cn_heavyZen3MemoryMutex); if (!cn_heavyZen3Memory) { - cn_heavyZen3Memory = new VirtualMemory(m_algorithm.l3() * m_threads, data.hugePages, false, false, node()); + // Round up number of threads to the multiple of 8 + const size_t N = ((m_threads + 7) / 8) * 8; + cn_heavyZen3Memory = new VirtualMemory(m_algorithm.l3() * N, data.hugePages, false, false, node()); } m_memory = cn_heavyZen3Memory; } From 548a7d46e1a16592ed26472bbc37b9ca4e37d395 Mon Sep 17 00:00:00 2001 From: XMRig Date: Thu, 4 Mar 2021 16:19:06 +0700 Subject: [PATCH 11/16] Add note about CPU affinity. --- src/core/config/usage.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/config/usage.h b/src/core/config/usage.h index fb887206..57374620 100644 --- a/src/core/config/usage.h +++ b/src/core/config/usage.h @@ -75,9 +75,9 @@ static inline const std::string &usage() u += "\nCPU backend:\n"; u += " --no-cpu disable CPU mining backend\n"; - u += " -t, --threads=N number of CPU threads\n"; - u += " -v, --av=N algorithm variation, 0 auto select\n"; + u += " -t, --threads=N number of CPU threads, proper CPU affinity required for some optimizations.\n"; u += " --cpu-affinity set process affinity to CPU core(s), mask 0x3 for cores 0 and 1\n"; + u += " -v, --av=N algorithm variation, 0 auto select\n"; u += " --cpu-priority set process priority (0 idle, 2 normal to 5 highest)\n"; u += " --cpu-max-threads-hint=N maximum CPU threads count (in percentage) hint for autoconfig\n"; u += " --cpu-memory-pool=N number of 2 MB pages for persistent memory pool, -1 (auto), 0 (disable)\n"; From 94d2cac7752c9883eae35be0ebb50508b171d32c Mon Sep 17 00:00:00 2001 From: SChernykh Date: Thu, 4 Mar 2021 10:45:39 +0100 Subject: [PATCH 12/16] Fix GCC compilation --- src/backend/cpu/CpuWorker.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/cpu/CpuWorker.cpp b/src/backend/cpu/CpuWorker.cpp index 6263f89a..818ea0c8 100644 --- a/src/backend/cpu/CpuWorker.cpp +++ b/src/backend/cpu/CpuWorker.cpp @@ -87,8 +87,8 @@ xmrig::CpuWorker::CpuWorker(size_t id, const CpuLaunchData &data) : std::lock_guard lock(cn_heavyZen3MemoryMutex); if (!cn_heavyZen3Memory) { // Round up number of threads to the multiple of 8 - const size_t N = ((m_threads + 7) / 8) * 8; - cn_heavyZen3Memory = new VirtualMemory(m_algorithm.l3() * N, data.hugePages, false, false, node()); + const size_t num_threads = ((m_threads + 7) / 8) * 8; + cn_heavyZen3Memory = new VirtualMemory(m_algorithm.l3() * num_threads, data.hugePages, false, false, node()); } m_memory = cn_heavyZen3Memory; } From b3dbf6e23fa8e3f2f5de094eba2c04b3aad1ba92 Mon Sep 17 00:00:00 2001 From: XMRig Date: Sat, 6 Mar 2021 01:46:49 +0700 Subject: [PATCH 13/16] http-parser replaced to llhttp. --- src/3rdparty/http-parser/AUTHORS | 68 - src/3rdparty/http-parser/LICENSE-MIT | 19 - src/3rdparty/http-parser/README.md | 246 - src/3rdparty/http-parser/http_parser.c | 2565 ---- src/3rdparty/http-parser/http_parser.h | 442 - src/3rdparty/llhttp/LICENSE-MIT | 22 + src/3rdparty/llhttp/README.md | 135 + src/3rdparty/llhttp/api.c | 275 + src/3rdparty/llhttp/api.h | 187 + src/3rdparty/llhttp/http.c | 149 + src/3rdparty/llhttp/llhttp.c | 14871 ++++++++++++++++++++ src/3rdparty/llhttp/llhttp.h | 439 + src/base/api/Api.cpp | 7 +- src/base/api/Api.h | 4 +- src/base/api/Httpd.cpp | 31 +- src/base/api/Httpd.h | 10 +- src/base/api/requests/HttpApiRequest.cpp | 24 +- src/base/api/requests/HttpApiRequest.h | 10 +- src/base/base.cmake | 6 +- src/base/net/http/Fetch.cpp | 10 +- src/base/net/http/Fetch.h | 20 +- src/base/net/http/HttpApiResponse.cpp | 15 +- src/base/net/http/HttpApiResponse.h | 13 +- src/base/net/http/HttpClient.cpp | 8 +- src/base/net/http/HttpClient.h | 4 +- src/base/net/http/HttpContext.cpp | 24 +- src/base/net/http/HttpContext.h | 12 +- src/base/net/http/HttpData.cpp | 109 +- src/base/net/http/HttpData.h | 10 +- src/base/net/http/HttpListener.cpp | 8 +- src/base/net/http/HttpListener.h | 4 +- src/base/net/http/HttpResponse.cpp | 18 +- src/base/net/http/HttpResponse.h | 13 +- src/base/net/https/HttpsContext.cpp | 6 +- src/base/net/https/HttpsContext.h | 4 +- src/base/net/stratum/DaemonClient.cpp | 3 +- src/base/net/stratum/SelfSelectClient.cpp | 3 +- 37 files changed, 16281 insertions(+), 3513 deletions(-) delete mode 100644 src/3rdparty/http-parser/AUTHORS delete mode 100644 src/3rdparty/http-parser/LICENSE-MIT delete mode 100644 src/3rdparty/http-parser/README.md delete mode 100644 src/3rdparty/http-parser/http_parser.c delete mode 100644 src/3rdparty/http-parser/http_parser.h create mode 100644 src/3rdparty/llhttp/LICENSE-MIT create mode 100644 src/3rdparty/llhttp/README.md create mode 100644 src/3rdparty/llhttp/api.c create mode 100644 src/3rdparty/llhttp/api.h create mode 100644 src/3rdparty/llhttp/http.c create mode 100644 src/3rdparty/llhttp/llhttp.c create mode 100644 src/3rdparty/llhttp/llhttp.h diff --git a/src/3rdparty/http-parser/AUTHORS b/src/3rdparty/http-parser/AUTHORS deleted file mode 100644 index 5323b685..00000000 --- a/src/3rdparty/http-parser/AUTHORS +++ /dev/null @@ -1,68 +0,0 @@ -# Authors ordered by first contribution. -Ryan Dahl -Jeremy Hinegardner -Sergey Shepelev -Joe Damato -tomika -Phoenix Sol -Cliff Frey -Ewen Cheslack-Postava -Santiago Gala -Tim Becker -Jeff Terrace -Ben Noordhuis -Nathan Rajlich -Mark Nottingham -Aman Gupta -Tim Becker -Sean Cunningham -Peter Griess -Salman Haq -Cliff Frey -Jon Kolb -Fouad Mardini -Paul Querna -Felix Geisendörfer -koichik -Andre Caron -Ivo Raisr -James McLaughlin -David Gwynne -Thomas LE ROUX -Randy Rizun -Andre Louis Caron -Simon Zimmermann -Erik Dubbelboer -Martell Malone -Bertrand Paquet -BogDan Vatra -Peter Faiman -Corey Richardson -Tóth Tamás -Cam Swords -Chris Dickinson -Uli Köhler -Charlie Somerville -Patrik Stutz -Fedor Indutny -runner -Alexis Campailla -David Wragg -Vinnie Falco -Alex Butum -Rex Feng -Alex Kocharin -Mark Koopman -Helge Heß -Alexis La Goutte -George Miroshnykov -Maciej Małecki -Marc O'Morain -Jeff Pinner -Timothy J Fontaine -Akagi201 -Romain Giraud -Jay Satiro -Arne Steen -Kjell Schubert -Olivier Mengué diff --git a/src/3rdparty/http-parser/LICENSE-MIT b/src/3rdparty/http-parser/LICENSE-MIT deleted file mode 100644 index 1ec0ab4e..00000000 --- a/src/3rdparty/http-parser/LICENSE-MIT +++ /dev/null @@ -1,19 +0,0 @@ -Copyright Joyent, Inc. and other Node contributors. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to -deal in the Software without restriction, including without limitation the -rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -sell copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -IN THE SOFTWARE. diff --git a/src/3rdparty/http-parser/README.md b/src/3rdparty/http-parser/README.md deleted file mode 100644 index b265d717..00000000 --- a/src/3rdparty/http-parser/README.md +++ /dev/null @@ -1,246 +0,0 @@ -HTTP Parser -=========== - -[![Build Status](https://api.travis-ci.org/nodejs/http-parser.svg?branch=master)](https://travis-ci.org/nodejs/http-parser) - -This is a parser for HTTP messages written in C. It parses both requests and -responses. The parser is designed to be used in performance HTTP -applications. It does not make any syscalls nor allocations, it does not -buffer data, it can be interrupted at anytime. Depending on your -architecture, it only requires about 40 bytes of data per message -stream (in a web server that is per connection). - -Features: - - * No dependencies - * Handles persistent streams (keep-alive). - * Decodes chunked encoding. - * Upgrade support - * Defends against buffer overflow attacks. - -The parser extracts the following information from HTTP messages: - - * Header fields and values - * Content-Length - * Request method - * Response status code - * Transfer-Encoding - * HTTP version - * Request URL - * Message body - - -Usage ------ - -One `http_parser` object is used per TCP connection. Initialize the struct -using `http_parser_init()` and set the callbacks. That might look something -like this for a request parser: -```c -http_parser_settings settings; -settings.on_url = my_url_callback; -settings.on_header_field = my_header_field_callback; -/* ... */ - -http_parser *parser = malloc(sizeof(http_parser)); -http_parser_init(parser, HTTP_REQUEST); -parser->data = my_socket; -``` - -When data is received on the socket execute the parser and check for errors. - -```c -size_t len = 80*1024, nparsed; -char buf[len]; -ssize_t recved; - -recved = recv(fd, buf, len, 0); - -if (recved < 0) { - /* Handle error. */ -} - -/* Start up / continue the parser. - * Note we pass recved==0 to signal that EOF has been received. - */ -nparsed = http_parser_execute(parser, &settings, buf, recved); - -if (parser->upgrade) { - /* handle new protocol */ -} else if (nparsed != recved) { - /* Handle error. Usually just close the connection. */ -} -``` - -`http_parser` needs to know where the end of the stream is. For example, sometimes -servers send responses without Content-Length and expect the client to -consume input (for the body) until EOF. To tell `http_parser` about EOF, give -`0` as the fourth parameter to `http_parser_execute()`. Callbacks and errors -can still be encountered during an EOF, so one must still be prepared -to receive them. - -Scalar valued message information such as `status_code`, `method`, and the -HTTP version are stored in the parser structure. This data is only -temporally stored in `http_parser` and gets reset on each new message. If -this information is needed later, copy it out of the structure during the -`headers_complete` callback. - -The parser decodes the transfer-encoding for both requests and responses -transparently. That is, a chunked encoding is decoded before being sent to -the on_body callback. - - -The Special Problem of Upgrade ------------------------------- - -`http_parser` supports upgrading the connection to a different protocol. An -increasingly common example of this is the WebSocket protocol which sends -a request like - - GET /demo HTTP/1.1 - Upgrade: WebSocket - Connection: Upgrade - Host: example.com - Origin: http://example.com - WebSocket-Protocol: sample - -followed by non-HTTP data. - -(See [RFC6455](https://tools.ietf.org/html/rfc6455) for more information the -WebSocket protocol.) - -To support this, the parser will treat this as a normal HTTP message without a -body, issuing both on_headers_complete and on_message_complete callbacks. However -http_parser_execute() will stop parsing at the end of the headers and return. - -The user is expected to check if `parser->upgrade` has been set to 1 after -`http_parser_execute()` returns. Non-HTTP data begins at the buffer supplied -offset by the return value of `http_parser_execute()`. - - -Callbacks ---------- - -During the `http_parser_execute()` call, the callbacks set in -`http_parser_settings` will be executed. The parser maintains state and -never looks behind, so buffering the data is not necessary. If you need to -save certain data for later usage, you can do that from the callbacks. - -There are two types of callbacks: - -* notification `typedef int (*http_cb) (http_parser*);` - Callbacks: on_message_begin, on_headers_complete, on_message_complete. -* data `typedef int (*http_data_cb) (http_parser*, const char *at, size_t length);` - Callbacks: (requests only) on_url, - (common) on_header_field, on_header_value, on_body; - -Callbacks must return 0 on success. Returning a non-zero value indicates -error to the parser, making it exit immediately. - -For cases where it is necessary to pass local information to/from a callback, -the `http_parser` object's `data` field can be used. -An example of such a case is when using threads to handle a socket connection, -parse a request, and then give a response over that socket. By instantiation -of a thread-local struct containing relevant data (e.g. accepted socket, -allocated memory for callbacks to write into, etc), a parser's callbacks are -able to communicate data between the scope of the thread and the scope of the -callback in a threadsafe manner. This allows `http_parser` to be used in -multi-threaded contexts. - -Example: -```c - typedef struct { - socket_t sock; - void* buffer; - int buf_len; - } custom_data_t; - - -int my_url_callback(http_parser* parser, const char *at, size_t length) { - /* access to thread local custom_data_t struct. - Use this access save parsed data for later use into thread local - buffer, or communicate over socket - */ - parser->data; - ... - return 0; -} - -... - -void http_parser_thread(socket_t sock) { - int nparsed = 0; - /* allocate memory for user data */ - custom_data_t *my_data = malloc(sizeof(custom_data_t)); - - /* some information for use by callbacks. - * achieves thread -> callback information flow */ - my_data->sock = sock; - - /* instantiate a thread-local parser */ - http_parser *parser = malloc(sizeof(http_parser)); - http_parser_init(parser, HTTP_REQUEST); /* initialise parser */ - /* this custom data reference is accessible through the reference to the - parser supplied to callback functions */ - parser->data = my_data; - - http_parser_settings settings; /* set up callbacks */ - settings.on_url = my_url_callback; - - /* execute parser */ - nparsed = http_parser_execute(parser, &settings, buf, recved); - - ... - /* parsed information copied from callback. - can now perform action on data copied into thread-local memory from callbacks. - achieves callback -> thread information flow */ - my_data->buffer; - ... -} - -``` - -In case you parse HTTP message in chunks (i.e. `read()` request line -from socket, parse, read half headers, parse, etc) your data callbacks -may be called more than once. `http_parser` guarantees that data pointer is only -valid for the lifetime of callback. You can also `read()` into a heap allocated -buffer to avoid copying memory around if this fits your application. - -Reading headers may be a tricky task if you read/parse headers partially. -Basically, you need to remember whether last header callback was field or value -and apply the following logic: - - (on_header_field and on_header_value shortened to on_h_*) - ------------------------ ------------ -------------------------------------------- - | State (prev. callback) | Callback | Description/action | - ------------------------ ------------ -------------------------------------------- - | nothing (first call) | on_h_field | Allocate new buffer and copy callback data | - | | | into it | - ------------------------ ------------ -------------------------------------------- - | value | on_h_field | New header started. | - | | | Copy current name,value buffers to headers | - | | | list and allocate new buffer for new name | - ------------------------ ------------ -------------------------------------------- - | field | on_h_field | Previous name continues. Reallocate name | - | | | buffer and append callback data to it | - ------------------------ ------------ -------------------------------------------- - | field | on_h_value | Value for current header started. Allocate | - | | | new buffer and copy callback data to it | - ------------------------ ------------ -------------------------------------------- - | value | on_h_value | Value continues. Reallocate value buffer | - | | | and append callback data to it | - ------------------------ ------------ -------------------------------------------- - - -Parsing URLs ------------- - -A simplistic zero-copy URL parser is provided as `http_parser_parse_url()`. -Users of this library may wish to use it to parse URLs constructed from -consecutive `on_url` callbacks. - -See examples of reading in headers: - -* [partial example](http://gist.github.com/155877) in C -* [from http-parser tests](http://github.com/joyent/http-parser/blob/37a0ff8/test.c#L403) in C -* [from Node library](http://github.com/joyent/node/blob/842eaf4/src/http.js#L284) in Javascript diff --git a/src/3rdparty/http-parser/http_parser.c b/src/3rdparty/http-parser/http_parser.c deleted file mode 100644 index 0f76b6ac..00000000 --- a/src/3rdparty/http-parser/http_parser.c +++ /dev/null @@ -1,2565 +0,0 @@ -/* Copyright Joyent, Inc. and other Node contributors. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ -#include "http_parser.h" -#include -#include -#include -#include -#include - -static uint32_t max_header_size = HTTP_MAX_HEADER_SIZE; - -#ifndef ULLONG_MAX -# define ULLONG_MAX ((uint64_t) -1) /* 2^64-1 */ -#endif - -#ifndef MIN -# define MIN(a,b) ((a) < (b) ? (a) : (b)) -#endif - -#ifndef ARRAY_SIZE -# define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) -#endif - -#ifndef BIT_AT -# define BIT_AT(a, i) \ - (!!((unsigned int) (a)[(unsigned int) (i) >> 3] & \ - (1 << ((unsigned int) (i) & 7)))) -#endif - -#ifndef ELEM_AT -# define ELEM_AT(a, i, v) ((unsigned int) (i) < ARRAY_SIZE(a) ? (a)[(i)] : (v)) -#endif - -#define SET_ERRNO(e) \ -do { \ - parser->nread = nread; \ - parser->http_errno = (e); \ -} while(0) - -#define CURRENT_STATE() p_state -#define UPDATE_STATE(V) p_state = (enum state) (V); -#define RETURN(V) \ -do { \ - parser->nread = nread; \ - parser->state = CURRENT_STATE(); \ - return (V); \ -} while (0); -#define REEXECUTE() \ - goto reexecute; \ - - -#ifdef __GNUC__ -# define LIKELY(X) __builtin_expect(!!(X), 1) -# define UNLIKELY(X) __builtin_expect(!!(X), 0) -#else -# define LIKELY(X) (X) -# define UNLIKELY(X) (X) -#endif - - -/* Run the notify callback FOR, returning ER if it fails */ -#define CALLBACK_NOTIFY_(FOR, ER) \ -do { \ - assert(HTTP_PARSER_ERRNO(parser) == HPE_OK); \ - \ - if (LIKELY(settings->on_##FOR)) { \ - parser->state = CURRENT_STATE(); \ - if (UNLIKELY(0 != settings->on_##FOR(parser))) { \ - SET_ERRNO(HPE_CB_##FOR); \ - } \ - UPDATE_STATE(parser->state); \ - \ - /* We either errored above or got paused; get out */ \ - if (UNLIKELY(HTTP_PARSER_ERRNO(parser) != HPE_OK)) { \ - return (ER); \ - } \ - } \ -} while (0) - -/* Run the notify callback FOR and consume the current byte */ -#define CALLBACK_NOTIFY(FOR) CALLBACK_NOTIFY_(FOR, p - data + 1) - -/* Run the notify callback FOR and don't consume the current byte */ -#define CALLBACK_NOTIFY_NOADVANCE(FOR) CALLBACK_NOTIFY_(FOR, p - data) - -/* Run data callback FOR with LEN bytes, returning ER if it fails */ -#define CALLBACK_DATA_(FOR, LEN, ER) \ -do { \ - assert(HTTP_PARSER_ERRNO(parser) == HPE_OK); \ - \ - if (FOR##_mark) { \ - if (LIKELY(settings->on_##FOR)) { \ - parser->state = CURRENT_STATE(); \ - if (UNLIKELY(0 != \ - settings->on_##FOR(parser, FOR##_mark, (LEN)))) { \ - SET_ERRNO(HPE_CB_##FOR); \ - } \ - UPDATE_STATE(parser->state); \ - \ - /* We either errored above or got paused; get out */ \ - if (UNLIKELY(HTTP_PARSER_ERRNO(parser) != HPE_OK)) { \ - return (ER); \ - } \ - } \ - FOR##_mark = NULL; \ - } \ -} while (0) - -/* Run the data callback FOR and consume the current byte */ -#define CALLBACK_DATA(FOR) \ - CALLBACK_DATA_(FOR, p - FOR##_mark, p - data + 1) - -/* Run the data callback FOR and don't consume the current byte */ -#define CALLBACK_DATA_NOADVANCE(FOR) \ - CALLBACK_DATA_(FOR, p - FOR##_mark, p - data) - -/* Set the mark FOR; non-destructive if mark is already set */ -#define MARK(FOR) \ -do { \ - if (!FOR##_mark) { \ - FOR##_mark = p; \ - } \ -} while (0) - -/* Don't allow the total size of the HTTP headers (including the status - * line) to exceed max_header_size. This check is here to protect - * embedders against denial-of-service attacks where the attacker feeds - * us a never-ending header that the embedder keeps buffering. - * - * This check is arguably the responsibility of embedders but we're doing - * it on the embedder's behalf because most won't bother and this way we - * make the web a little safer. max_header_size is still far bigger - * than any reasonable request or response so this should never affect - * day-to-day operation. - */ -#define COUNT_HEADER_SIZE(V) \ -do { \ - nread += (uint32_t)(V); \ - if (UNLIKELY(nread > max_header_size)) { \ - SET_ERRNO(HPE_HEADER_OVERFLOW); \ - goto error; \ - } \ -} while (0) - - -#define PROXY_CONNECTION "proxy-connection" -#define CONNECTION "connection" -#define CONTENT_LENGTH "content-length" -#define TRANSFER_ENCODING "transfer-encoding" -#define UPGRADE "upgrade" -#define CHUNKED "chunked" -#define KEEP_ALIVE "keep-alive" -#define CLOSE "close" - - -static const char *method_strings[] = - { -#define XX(num, name, string) #string, - HTTP_METHOD_MAP(XX) -#undef XX - }; - - -/* Tokens as defined by rfc 2616. Also lowercases them. - * token = 1* - * separators = "(" | ")" | "<" | ">" | "@" - * | "," | ";" | ":" | "\" | <"> - * | "/" | "[" | "]" | "?" | "=" - * | "{" | "}" | SP | HT - */ -static const char tokens[256] = { -/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */ - 0, 0, 0, 0, 0, 0, 0, 0, -/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */ - 0, 0, 0, 0, 0, 0, 0, 0, -/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */ - 0, 0, 0, 0, 0, 0, 0, 0, -/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */ - 0, 0, 0, 0, 0, 0, 0, 0, -/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */ - ' ', '!', 0, '#', '$', '%', '&', '\'', -/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */ - 0, 0, '*', '+', 0, '-', '.', 0, -/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */ - '0', '1', '2', '3', '4', '5', '6', '7', -/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */ - '8', '9', 0, 0, 0, 0, 0, 0, -/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */ - 0, 'a', 'b', 'c', 'd', 'e', 'f', 'g', -/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */ - 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', -/* 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W */ - 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', -/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */ - 'x', 'y', 'z', 0, 0, 0, '^', '_', -/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */ - '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', -/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */ - 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', -/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */ - 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', -/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */ - 'x', 'y', 'z', 0, '|', 0, '~', 0 }; - - -static const int8_t unhex[256] = - {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 - ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 - ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 - , 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1 - ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1 - ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 - ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1 - ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 - }; - - -#if HTTP_PARSER_STRICT -# define T(v) 0 -#else -# define T(v) v -#endif - - -static const uint8_t normal_url_char[32] = { -/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */ - 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0, -/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */ - 0 | T(2) | 0 | 0 | T(16) | 0 | 0 | 0, -/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */ - 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0, -/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */ - 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0, -/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */ - 0 | 2 | 4 | 0 | 16 | 32 | 64 | 128, -/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */ - 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, -/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */ - 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, -/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */ - 1 | 2 | 4 | 8 | 16 | 32 | 64 | 0, -/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */ - 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, -/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */ - 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, -/* 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W */ - 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, -/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */ - 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, -/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */ - 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, -/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */ - 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, -/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */ - 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, -/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */ - 1 | 2 | 4 | 8 | 16 | 32 | 64 | 0, }; - -#undef T - -enum state - { s_dead = 1 /* important that this is > 0 */ - - , s_start_req_or_res - , s_res_or_resp_H - , s_start_res - , s_res_H - , s_res_HT - , s_res_HTT - , s_res_HTTP - , s_res_http_major - , s_res_http_dot - , s_res_http_minor - , s_res_http_end - , s_res_first_status_code - , s_res_status_code - , s_res_status_start - , s_res_status - , s_res_line_almost_done - - , s_start_req - - , s_req_method - , s_req_spaces_before_url - , s_req_schema - , s_req_schema_slash - , s_req_schema_slash_slash - , s_req_server_start - , s_req_server - , s_req_server_with_at - , s_req_path - , s_req_query_string_start - , s_req_query_string - , s_req_fragment_start - , s_req_fragment - , s_req_http_start - , s_req_http_H - , s_req_http_HT - , s_req_http_HTT - , s_req_http_HTTP - , s_req_http_I - , s_req_http_IC - , s_req_http_major - , s_req_http_dot - , s_req_http_minor - , s_req_http_end - , s_req_line_almost_done - - , s_header_field_start - , s_header_field - , s_header_value_discard_ws - , s_header_value_discard_ws_almost_done - , s_header_value_discard_lws - , s_header_value_start - , s_header_value - , s_header_value_lws - - , s_header_almost_done - - , s_chunk_size_start - , s_chunk_size - , s_chunk_parameters - , s_chunk_size_almost_done - - , s_headers_almost_done - , s_headers_done - - /* Important: 's_headers_done' must be the last 'header' state. All - * states beyond this must be 'body' states. It is used for overflow - * checking. See the PARSING_HEADER() macro. - */ - - , s_chunk_data - , s_chunk_data_almost_done - , s_chunk_data_done - - , s_body_identity - , s_body_identity_eof - - , s_message_done - }; - - -#define PARSING_HEADER(state) (state <= s_headers_done) - - -enum header_states - { h_general = 0 - , h_C - , h_CO - , h_CON - - , h_matching_connection - , h_matching_proxy_connection - , h_matching_content_length - , h_matching_transfer_encoding - , h_matching_upgrade - - , h_connection - , h_content_length - , h_content_length_num - , h_content_length_ws - , h_transfer_encoding - , h_upgrade - - , h_matching_transfer_encoding_token_start - , h_matching_transfer_encoding_chunked - , h_matching_transfer_encoding_token - - , h_matching_connection_token_start - , h_matching_connection_keep_alive - , h_matching_connection_close - , h_matching_connection_upgrade - , h_matching_connection_token - - , h_transfer_encoding_chunked - , h_connection_keep_alive - , h_connection_close - , h_connection_upgrade - }; - -enum http_host_state - { - s_http_host_dead = 1 - , s_http_userinfo_start - , s_http_userinfo - , s_http_host_start - , s_http_host_v6_start - , s_http_host - , s_http_host_v6 - , s_http_host_v6_end - , s_http_host_v6_zone_start - , s_http_host_v6_zone - , s_http_host_port_start - , s_http_host_port -}; - -/* Macros for character classes; depends on strict-mode */ -#define CR '\r' -#define LF '\n' -#define LOWER(c) (unsigned char)(c | 0x20) -#define IS_ALPHA(c) (LOWER(c) >= 'a' && LOWER(c) <= 'z') -#define IS_NUM(c) ((c) >= '0' && (c) <= '9') -#define IS_ALPHANUM(c) (IS_ALPHA(c) || IS_NUM(c)) -#define IS_HEX(c) (IS_NUM(c) || (LOWER(c) >= 'a' && LOWER(c) <= 'f')) -#define IS_MARK(c) ((c) == '-' || (c) == '_' || (c) == '.' || \ - (c) == '!' || (c) == '~' || (c) == '*' || (c) == '\'' || (c) == '(' || \ - (c) == ')') -#define IS_USERINFO_CHAR(c) (IS_ALPHANUM(c) || IS_MARK(c) || (c) == '%' || \ - (c) == ';' || (c) == ':' || (c) == '&' || (c) == '=' || (c) == '+' || \ - (c) == '$' || (c) == ',') - -#define STRICT_TOKEN(c) ((c == ' ') ? 0 : tokens[(unsigned char)c]) - -#if HTTP_PARSER_STRICT -#define TOKEN(c) STRICT_TOKEN(c) -#define IS_URL_CHAR(c) (BIT_AT(normal_url_char, (unsigned char)c)) -#define IS_HOST_CHAR(c) (IS_ALPHANUM(c) || (c) == '.' || (c) == '-') -#else -#define TOKEN(c) tokens[(unsigned char)c] -#define IS_URL_CHAR(c) \ - (BIT_AT(normal_url_char, (unsigned char)c) || ((c) & 0x80)) -#define IS_HOST_CHAR(c) \ - (IS_ALPHANUM(c) || (c) == '.' || (c) == '-' || (c) == '_') -#endif - -/** - * Verify that a char is a valid visible (printable) US-ASCII - * character or %x80-FF - **/ -#define IS_HEADER_CHAR(ch) \ - (ch == CR || ch == LF || ch == 9 || ((unsigned char)ch > 31 && ch != 127)) - -#define start_state (parser->type == HTTP_REQUEST ? s_start_req : s_start_res) - - -#if HTTP_PARSER_STRICT -# define STRICT_CHECK(cond) \ -do { \ - if (cond) { \ - SET_ERRNO(HPE_STRICT); \ - goto error; \ - } \ -} while (0) -# define NEW_MESSAGE() (http_should_keep_alive(parser) ? start_state : s_dead) -#else -# define STRICT_CHECK(cond) -# define NEW_MESSAGE() start_state -#endif - - -/* Map errno values to strings for human-readable output */ -#define HTTP_STRERROR_GEN(n, s) { "HPE_" #n, s }, -static struct { - const char *name; - const char *description; -} http_strerror_tab[] = { - HTTP_ERRNO_MAP(HTTP_STRERROR_GEN) -}; -#undef HTTP_STRERROR_GEN - -int http_message_needs_eof(const http_parser *parser); - -/* Our URL parser. - * - * This is designed to be shared by http_parser_execute() for URL validation, - * hence it has a state transition + byte-for-byte interface. In addition, it - * is meant to be embedded in http_parser_parse_url(), which does the dirty - * work of turning state transitions URL components for its API. - * - * This function should only be invoked with non-space characters. It is - * assumed that the caller cares about (and can detect) the transition between - * URL and non-URL states by looking for these. - */ -static enum state -parse_url_char(enum state s, const char ch) -{ - if (ch == ' ' || ch == '\r' || ch == '\n') { - return s_dead; - } - -#if HTTP_PARSER_STRICT - if (ch == '\t' || ch == '\f') { - return s_dead; - } -#endif - - switch (s) { - case s_req_spaces_before_url: - /* Proxied requests are followed by scheme of an absolute URI (alpha). - * All methods except CONNECT are followed by '/' or '*'. - */ - - if (ch == '/' || ch == '*') { - return s_req_path; - } - - if (IS_ALPHA(ch)) { - return s_req_schema; - } - - break; - - case s_req_schema: - if (IS_ALPHA(ch)) { - return s; - } - - if (ch == ':') { - return s_req_schema_slash; - } - - break; - - case s_req_schema_slash: - if (ch == '/') { - return s_req_schema_slash_slash; - } - - break; - - case s_req_schema_slash_slash: - if (ch == '/') { - return s_req_server_start; - } - - break; - - case s_req_server_with_at: - if (ch == '@') { - return s_dead; - } - - /* fall through */ - case s_req_server_start: - case s_req_server: - if (ch == '/') { - return s_req_path; - } - - if (ch == '?') { - return s_req_query_string_start; - } - - if (ch == '@') { - return s_req_server_with_at; - } - - if (IS_USERINFO_CHAR(ch) || ch == '[' || ch == ']') { - return s_req_server; - } - - break; - - case s_req_path: - if (IS_URL_CHAR(ch)) { - return s; - } - - switch (ch) { - case '?': - return s_req_query_string_start; - - case '#': - return s_req_fragment_start; - } - - break; - - case s_req_query_string_start: - case s_req_query_string: - if (IS_URL_CHAR(ch)) { - return s_req_query_string; - } - - switch (ch) { - case '?': - /* allow extra '?' in query string */ - return s_req_query_string; - - case '#': - return s_req_fragment_start; - } - - break; - - case s_req_fragment_start: - if (IS_URL_CHAR(ch)) { - return s_req_fragment; - } - - switch (ch) { - case '?': - return s_req_fragment; - - case '#': - return s; - } - - break; - - case s_req_fragment: - if (IS_URL_CHAR(ch)) { - return s; - } - - switch (ch) { - case '?': - case '#': - return s; - } - - break; - - default: - break; - } - - /* We should never fall out of the switch above unless there's an error */ - return s_dead; -} - -size_t http_parser_execute (http_parser *parser, - const http_parser_settings *settings, - const char *data, - size_t len) -{ - char c, ch; - int8_t unhex_val; - const char *p = data; - const char *header_field_mark = 0; - const char *header_value_mark = 0; - const char *url_mark = 0; - const char *body_mark = 0; - const char *status_mark = 0; - enum state p_state = (enum state) parser->state; - const unsigned int lenient = parser->lenient_http_headers; - uint32_t nread = parser->nread; - - /* We're in an error state. Don't bother doing anything. */ - if (HTTP_PARSER_ERRNO(parser) != HPE_OK) { - return 0; - } - - if (len == 0) { - switch (CURRENT_STATE()) { - case s_body_identity_eof: - /* Use of CALLBACK_NOTIFY() here would erroneously return 1 byte read if - * we got paused. - */ - CALLBACK_NOTIFY_NOADVANCE(message_complete); - return 0; - - case s_dead: - case s_start_req_or_res: - case s_start_res: - case s_start_req: - return 0; - - default: - SET_ERRNO(HPE_INVALID_EOF_STATE); - return 1; - } - } - - - if (CURRENT_STATE() == s_header_field) - header_field_mark = data; - if (CURRENT_STATE() == s_header_value) - header_value_mark = data; - switch (CURRENT_STATE()) { - case s_req_path: - case s_req_schema: - case s_req_schema_slash: - case s_req_schema_slash_slash: - case s_req_server_start: - case s_req_server: - case s_req_server_with_at: - case s_req_query_string_start: - case s_req_query_string: - case s_req_fragment_start: - case s_req_fragment: - url_mark = data; - break; - case s_res_status: - status_mark = data; - break; - default: - break; - } - - for (p=data; p != data + len; p++) { - ch = *p; - - if (PARSING_HEADER(CURRENT_STATE())) - COUNT_HEADER_SIZE(1); - -reexecute: - switch (CURRENT_STATE()) { - - case s_dead: - /* this state is used after a 'Connection: close' message - * the parser will error out if it reads another message - */ - if (LIKELY(ch == CR || ch == LF)) - break; - - SET_ERRNO(HPE_CLOSED_CONNECTION); - goto error; - - case s_start_req_or_res: - { - if (ch == CR || ch == LF) - break; - parser->flags = 0; - parser->content_length = ULLONG_MAX; - - if (ch == 'H') { - UPDATE_STATE(s_res_or_resp_H); - - CALLBACK_NOTIFY(message_begin); - } else { - parser->type = HTTP_REQUEST; - UPDATE_STATE(s_start_req); - REEXECUTE(); - } - - break; - } - - case s_res_or_resp_H: - if (ch == 'T') { - parser->type = HTTP_RESPONSE; - UPDATE_STATE(s_res_HT); - } else { - if (UNLIKELY(ch != 'E')) { - SET_ERRNO(HPE_INVALID_CONSTANT); - goto error; - } - - parser->type = HTTP_REQUEST; - parser->method = HTTP_HEAD; - parser->index = 2; - UPDATE_STATE(s_req_method); - } - break; - - case s_start_res: - { - if (ch == CR || ch == LF) - break; - parser->flags = 0; - parser->content_length = ULLONG_MAX; - - if (ch == 'H') { - UPDATE_STATE(s_res_H); - } else { - SET_ERRNO(HPE_INVALID_CONSTANT); - goto error; - } - - CALLBACK_NOTIFY(message_begin); - break; - } - - case s_res_H: - STRICT_CHECK(ch != 'T'); - UPDATE_STATE(s_res_HT); - break; - - case s_res_HT: - STRICT_CHECK(ch != 'T'); - UPDATE_STATE(s_res_HTT); - break; - - case s_res_HTT: - STRICT_CHECK(ch != 'P'); - UPDATE_STATE(s_res_HTTP); - break; - - case s_res_HTTP: - STRICT_CHECK(ch != '/'); - UPDATE_STATE(s_res_http_major); - break; - - case s_res_http_major: - if (UNLIKELY(!IS_NUM(ch))) { - SET_ERRNO(HPE_INVALID_VERSION); - goto error; - } - - parser->http_major = ch - '0'; - UPDATE_STATE(s_res_http_dot); - break; - - case s_res_http_dot: - { - if (UNLIKELY(ch != '.')) { - SET_ERRNO(HPE_INVALID_VERSION); - goto error; - } - - UPDATE_STATE(s_res_http_minor); - break; - } - - case s_res_http_minor: - if (UNLIKELY(!IS_NUM(ch))) { - SET_ERRNO(HPE_INVALID_VERSION); - goto error; - } - - parser->http_minor = ch - '0'; - UPDATE_STATE(s_res_http_end); - break; - - case s_res_http_end: - { - if (UNLIKELY(ch != ' ')) { - SET_ERRNO(HPE_INVALID_VERSION); - goto error; - } - - UPDATE_STATE(s_res_first_status_code); - break; - } - - case s_res_first_status_code: - { - if (!IS_NUM(ch)) { - if (ch == ' ') { - break; - } - - SET_ERRNO(HPE_INVALID_STATUS); - goto error; - } - parser->status_code = ch - '0'; - UPDATE_STATE(s_res_status_code); - break; - } - - case s_res_status_code: - { - if (!IS_NUM(ch)) { - switch (ch) { - case ' ': - UPDATE_STATE(s_res_status_start); - break; - case CR: - case LF: - UPDATE_STATE(s_res_status_start); - REEXECUTE(); - break; - default: - SET_ERRNO(HPE_INVALID_STATUS); - goto error; - } - break; - } - - parser->status_code *= 10; - parser->status_code += ch - '0'; - - if (UNLIKELY(parser->status_code > 999)) { - SET_ERRNO(HPE_INVALID_STATUS); - goto error; - } - - break; - } - - case s_res_status_start: - { - MARK(status); - UPDATE_STATE(s_res_status); - parser->index = 0; - - if (ch == CR || ch == LF) - REEXECUTE(); - - break; - } - - case s_res_status: - if (ch == CR) { - UPDATE_STATE(s_res_line_almost_done); - CALLBACK_DATA(status); - break; - } - - if (ch == LF) { - UPDATE_STATE(s_header_field_start); - CALLBACK_DATA(status); - break; - } - - break; - - case s_res_line_almost_done: - STRICT_CHECK(ch != LF); - UPDATE_STATE(s_header_field_start); - break; - - case s_start_req: - { - if (ch == CR || ch == LF) - break; - parser->flags = 0; - parser->content_length = ULLONG_MAX; - - if (UNLIKELY(!IS_ALPHA(ch))) { - SET_ERRNO(HPE_INVALID_METHOD); - goto error; - } - - parser->method = (enum http_method) 0; - parser->index = 1; - switch (ch) { - case 'A': parser->method = HTTP_ACL; break; - case 'B': parser->method = HTTP_BIND; break; - case 'C': parser->method = HTTP_CONNECT; /* or COPY, CHECKOUT */ break; - case 'D': parser->method = HTTP_DELETE; break; - case 'G': parser->method = HTTP_GET; break; - case 'H': parser->method = HTTP_HEAD; break; - case 'L': parser->method = HTTP_LOCK; /* or LINK */ break; - case 'M': parser->method = HTTP_MKCOL; /* or MOVE, MKACTIVITY, MERGE, M-SEARCH, MKCALENDAR */ break; - case 'N': parser->method = HTTP_NOTIFY; break; - case 'O': parser->method = HTTP_OPTIONS; break; - case 'P': parser->method = HTTP_POST; - /* or PROPFIND|PROPPATCH|PUT|PATCH|PURGE */ - break; - case 'R': parser->method = HTTP_REPORT; /* or REBIND */ break; - case 'S': parser->method = HTTP_SUBSCRIBE; /* or SEARCH, SOURCE */ break; - case 'T': parser->method = HTTP_TRACE; break; - case 'U': parser->method = HTTP_UNLOCK; /* or UNSUBSCRIBE, UNBIND, UNLINK */ break; - default: - SET_ERRNO(HPE_INVALID_METHOD); - goto error; - } - UPDATE_STATE(s_req_method); - - CALLBACK_NOTIFY(message_begin); - - break; - } - - case s_req_method: - { - const char *matcher; - if (UNLIKELY(ch == '\0')) { - SET_ERRNO(HPE_INVALID_METHOD); - goto error; - } - - matcher = method_strings[parser->method]; - if (ch == ' ' && matcher[parser->index] == '\0') { - UPDATE_STATE(s_req_spaces_before_url); - } else if (ch == matcher[parser->index]) { - ; /* nada */ - } else if ((ch >= 'A' && ch <= 'Z') || ch == '-') { - - switch (parser->method << 16 | parser->index << 8 | ch) { -#define XX(meth, pos, ch, new_meth) \ - case (HTTP_##meth << 16 | pos << 8 | ch): \ - parser->method = HTTP_##new_meth; break; - - XX(POST, 1, 'U', PUT) - XX(POST, 1, 'A', PATCH) - XX(POST, 1, 'R', PROPFIND) - XX(PUT, 2, 'R', PURGE) - XX(CONNECT, 1, 'H', CHECKOUT) - XX(CONNECT, 2, 'P', COPY) - XX(MKCOL, 1, 'O', MOVE) - XX(MKCOL, 1, 'E', MERGE) - XX(MKCOL, 1, '-', MSEARCH) - XX(MKCOL, 2, 'A', MKACTIVITY) - XX(MKCOL, 3, 'A', MKCALENDAR) - XX(SUBSCRIBE, 1, 'E', SEARCH) - XX(SUBSCRIBE, 1, 'O', SOURCE) - XX(REPORT, 2, 'B', REBIND) - XX(PROPFIND, 4, 'P', PROPPATCH) - XX(LOCK, 1, 'I', LINK) - XX(UNLOCK, 2, 'S', UNSUBSCRIBE) - XX(UNLOCK, 2, 'B', UNBIND) - XX(UNLOCK, 3, 'I', UNLINK) -#undef XX - default: - SET_ERRNO(HPE_INVALID_METHOD); - goto error; - } - } else { - SET_ERRNO(HPE_INVALID_METHOD); - goto error; - } - - ++parser->index; - break; - } - - case s_req_spaces_before_url: - { - if (ch == ' ') break; - - MARK(url); - if (parser->method == HTTP_CONNECT) { - UPDATE_STATE(s_req_server_start); - } - - UPDATE_STATE(parse_url_char(CURRENT_STATE(), ch)); - if (UNLIKELY(CURRENT_STATE() == s_dead)) { - SET_ERRNO(HPE_INVALID_URL); - goto error; - } - - break; - } - - case s_req_schema: - case s_req_schema_slash: - case s_req_schema_slash_slash: - case s_req_server_start: - { - switch (ch) { - /* No whitespace allowed here */ - case ' ': - case CR: - case LF: - SET_ERRNO(HPE_INVALID_URL); - goto error; - default: - UPDATE_STATE(parse_url_char(CURRENT_STATE(), ch)); - if (UNLIKELY(CURRENT_STATE() == s_dead)) { - SET_ERRNO(HPE_INVALID_URL); - goto error; - } - } - - break; - } - - case s_req_server: - case s_req_server_with_at: - case s_req_path: - case s_req_query_string_start: - case s_req_query_string: - case s_req_fragment_start: - case s_req_fragment: - { - switch (ch) { - case ' ': - UPDATE_STATE(s_req_http_start); - CALLBACK_DATA(url); - break; - case CR: - case LF: - parser->http_major = 0; - parser->http_minor = 9; - UPDATE_STATE((ch == CR) ? - s_req_line_almost_done : - s_header_field_start); - CALLBACK_DATA(url); - break; - default: - UPDATE_STATE(parse_url_char(CURRENT_STATE(), ch)); - if (UNLIKELY(CURRENT_STATE() == s_dead)) { - SET_ERRNO(HPE_INVALID_URL); - goto error; - } - } - break; - } - - case s_req_http_start: - switch (ch) { - case ' ': - break; - case 'H': - UPDATE_STATE(s_req_http_H); - break; - case 'I': - if (parser->method == HTTP_SOURCE) { - UPDATE_STATE(s_req_http_I); - break; - } - /* fall through */ - default: - SET_ERRNO(HPE_INVALID_CONSTANT); - goto error; - } - break; - - case s_req_http_H: - STRICT_CHECK(ch != 'T'); - UPDATE_STATE(s_req_http_HT); - break; - - case s_req_http_HT: - STRICT_CHECK(ch != 'T'); - UPDATE_STATE(s_req_http_HTT); - break; - - case s_req_http_HTT: - STRICT_CHECK(ch != 'P'); - UPDATE_STATE(s_req_http_HTTP); - break; - - case s_req_http_I: - STRICT_CHECK(ch != 'C'); - UPDATE_STATE(s_req_http_IC); - break; - - case s_req_http_IC: - STRICT_CHECK(ch != 'E'); - UPDATE_STATE(s_req_http_HTTP); /* Treat "ICE" as "HTTP". */ - break; - - case s_req_http_HTTP: - STRICT_CHECK(ch != '/'); - UPDATE_STATE(s_req_http_major); - break; - - case s_req_http_major: - if (UNLIKELY(!IS_NUM(ch))) { - SET_ERRNO(HPE_INVALID_VERSION); - goto error; - } - - parser->http_major = ch - '0'; - UPDATE_STATE(s_req_http_dot); - break; - - case s_req_http_dot: - { - if (UNLIKELY(ch != '.')) { - SET_ERRNO(HPE_INVALID_VERSION); - goto error; - } - - UPDATE_STATE(s_req_http_minor); - break; - } - - case s_req_http_minor: - if (UNLIKELY(!IS_NUM(ch))) { - SET_ERRNO(HPE_INVALID_VERSION); - goto error; - } - - parser->http_minor = ch - '0'; - UPDATE_STATE(s_req_http_end); - break; - - case s_req_http_end: - { - if (ch == CR) { - UPDATE_STATE(s_req_line_almost_done); - break; - } - - if (ch == LF) { - UPDATE_STATE(s_header_field_start); - break; - } - - SET_ERRNO(HPE_INVALID_VERSION); - goto error; - break; - } - - /* end of request line */ - case s_req_line_almost_done: - { - if (UNLIKELY(ch != LF)) { - SET_ERRNO(HPE_LF_EXPECTED); - goto error; - } - - UPDATE_STATE(s_header_field_start); - break; - } - - case s_header_field_start: - { - if (ch == CR) { - UPDATE_STATE(s_headers_almost_done); - break; - } - - if (ch == LF) { - /* they might be just sending \n instead of \r\n so this would be - * the second \n to denote the end of headers*/ - UPDATE_STATE(s_headers_almost_done); - REEXECUTE(); - } - - c = TOKEN(ch); - - if (UNLIKELY(!c)) { - SET_ERRNO(HPE_INVALID_HEADER_TOKEN); - goto error; - } - - MARK(header_field); - - parser->index = 0; - UPDATE_STATE(s_header_field); - - switch (c) { - case 'c': - parser->header_state = h_C; - break; - - case 'p': - parser->header_state = h_matching_proxy_connection; - break; - - case 't': - parser->header_state = h_matching_transfer_encoding; - break; - - case 'u': - parser->header_state = h_matching_upgrade; - break; - - default: - parser->header_state = h_general; - break; - } - break; - } - - case s_header_field: - { - const char* start = p; - for (; p != data + len; p++) { - ch = *p; - c = TOKEN(ch); - - if (!c) - break; - - switch (parser->header_state) { - case h_general: { - size_t left = data + len - p; - const char* pe = p + MIN(left, max_header_size); - while (p+1 < pe && TOKEN(p[1])) { - p++; - } - break; - } - - case h_C: - parser->index++; - parser->header_state = (c == 'o' ? h_CO : h_general); - break; - - case h_CO: - parser->index++; - parser->header_state = (c == 'n' ? h_CON : h_general); - break; - - case h_CON: - parser->index++; - switch (c) { - case 'n': - parser->header_state = h_matching_connection; - break; - case 't': - parser->header_state = h_matching_content_length; - break; - default: - parser->header_state = h_general; - break; - } - break; - - /* connection */ - - case h_matching_connection: - parser->index++; - if (parser->index > sizeof(CONNECTION)-1 - || c != CONNECTION[parser->index]) { - parser->header_state = h_general; - } else if (parser->index == sizeof(CONNECTION)-2) { - parser->header_state = h_connection; - } - break; - - /* proxy-connection */ - - case h_matching_proxy_connection: - parser->index++; - if (parser->index > sizeof(PROXY_CONNECTION)-1 - || c != PROXY_CONNECTION[parser->index]) { - parser->header_state = h_general; - } else if (parser->index == sizeof(PROXY_CONNECTION)-2) { - parser->header_state = h_connection; - } - break; - - /* content-length */ - - case h_matching_content_length: - parser->index++; - if (parser->index > sizeof(CONTENT_LENGTH)-1 - || c != CONTENT_LENGTH[parser->index]) { - parser->header_state = h_general; - } else if (parser->index == sizeof(CONTENT_LENGTH)-2) { - parser->header_state = h_content_length; - } - break; - - /* transfer-encoding */ - - case h_matching_transfer_encoding: - parser->index++; - if (parser->index > sizeof(TRANSFER_ENCODING)-1 - || c != TRANSFER_ENCODING[parser->index]) { - parser->header_state = h_general; - } else if (parser->index == sizeof(TRANSFER_ENCODING)-2) { - parser->header_state = h_transfer_encoding; - parser->flags |= F_TRANSFER_ENCODING; - } - break; - - /* upgrade */ - - case h_matching_upgrade: - parser->index++; - if (parser->index > sizeof(UPGRADE)-1 - || c != UPGRADE[parser->index]) { - parser->header_state = h_general; - } else if (parser->index == sizeof(UPGRADE)-2) { - parser->header_state = h_upgrade; - } - break; - - case h_connection: - case h_content_length: - case h_transfer_encoding: - case h_upgrade: - if (ch != ' ') parser->header_state = h_general; - break; - - default: - assert(0 && "Unknown header_state"); - break; - } - } - - if (p == data + len) { - --p; - COUNT_HEADER_SIZE(p - start); - break; - } - - COUNT_HEADER_SIZE(p - start); - - if (ch == ':') { - UPDATE_STATE(s_header_value_discard_ws); - CALLBACK_DATA(header_field); - break; - } - - SET_ERRNO(HPE_INVALID_HEADER_TOKEN); - goto error; - } - - case s_header_value_discard_ws: - if (ch == ' ' || ch == '\t') break; - - if (ch == CR) { - UPDATE_STATE(s_header_value_discard_ws_almost_done); - break; - } - - if (ch == LF) { - UPDATE_STATE(s_header_value_discard_lws); - break; - } - - /* fall through */ - - case s_header_value_start: - { - MARK(header_value); - - UPDATE_STATE(s_header_value); - parser->index = 0; - - c = LOWER(ch); - - switch (parser->header_state) { - case h_upgrade: - parser->flags |= F_UPGRADE; - parser->header_state = h_general; - break; - - case h_transfer_encoding: - /* looking for 'Transfer-Encoding: chunked' */ - if ('c' == c) { - parser->header_state = h_matching_transfer_encoding_chunked; - } else { - parser->header_state = h_matching_transfer_encoding_token; - } - break; - - /* Multi-value `Transfer-Encoding` header */ - case h_matching_transfer_encoding_token_start: - break; - - case h_content_length: - if (UNLIKELY(!IS_NUM(ch))) { - SET_ERRNO(HPE_INVALID_CONTENT_LENGTH); - goto error; - } - - if (parser->flags & F_CONTENTLENGTH) { - SET_ERRNO(HPE_UNEXPECTED_CONTENT_LENGTH); - goto error; - } - - parser->flags |= F_CONTENTLENGTH; - parser->content_length = ch - '0'; - parser->header_state = h_content_length_num; - break; - - /* when obsolete line folding is encountered for content length - * continue to the s_header_value state */ - case h_content_length_ws: - break; - - case h_connection: - /* looking for 'Connection: keep-alive' */ - if (c == 'k') { - parser->header_state = h_matching_connection_keep_alive; - /* looking for 'Connection: close' */ - } else if (c == 'c') { - parser->header_state = h_matching_connection_close; - } else if (c == 'u') { - parser->header_state = h_matching_connection_upgrade; - } else { - parser->header_state = h_matching_connection_token; - } - break; - - /* Multi-value `Connection` header */ - case h_matching_connection_token_start: - break; - - default: - parser->header_state = h_general; - break; - } - break; - } - - case s_header_value: - { - const char* start = p; - enum header_states h_state = (enum header_states) parser->header_state; - for (; p != data + len; p++) { - ch = *p; - if (ch == CR) { - UPDATE_STATE(s_header_almost_done); - parser->header_state = h_state; - CALLBACK_DATA(header_value); - break; - } - - if (ch == LF) { - UPDATE_STATE(s_header_almost_done); - COUNT_HEADER_SIZE(p - start); - parser->header_state = h_state; - CALLBACK_DATA_NOADVANCE(header_value); - REEXECUTE(); - } - - if (!lenient && !IS_HEADER_CHAR(ch)) { - SET_ERRNO(HPE_INVALID_HEADER_TOKEN); - goto error; - } - - c = LOWER(ch); - - switch (h_state) { - case h_general: - { - size_t left = data + len - p; - const char* pe = p + MIN(left, max_header_size); - - for (; p != pe; p++) { - ch = *p; - if (ch == CR || ch == LF) { - --p; - break; - } - if (!lenient && !IS_HEADER_CHAR(ch)) { - SET_ERRNO(HPE_INVALID_HEADER_TOKEN); - goto error; - } - } - if (p == data + len) - --p; - break; - } - - case h_connection: - case h_transfer_encoding: - assert(0 && "Shouldn't get here."); - break; - - case h_content_length: - if (ch == ' ') break; - h_state = h_content_length_num; - /* fall through */ - - case h_content_length_num: - { - uint64_t t; - - if (ch == ' ') { - h_state = h_content_length_ws; - break; - } - - if (UNLIKELY(!IS_NUM(ch))) { - SET_ERRNO(HPE_INVALID_CONTENT_LENGTH); - parser->header_state = h_state; - goto error; - } - - t = parser->content_length; - t *= 10; - t += ch - '0'; - - /* Overflow? Test against a conservative limit for simplicity. */ - if (UNLIKELY((ULLONG_MAX - 10) / 10 < parser->content_length)) { - SET_ERRNO(HPE_INVALID_CONTENT_LENGTH); - parser->header_state = h_state; - goto error; - } - - parser->content_length = t; - break; - } - - case h_content_length_ws: - if (ch == ' ') break; - SET_ERRNO(HPE_INVALID_CONTENT_LENGTH); - parser->header_state = h_state; - goto error; - - /* Transfer-Encoding: chunked */ - case h_matching_transfer_encoding_token_start: - /* looking for 'Transfer-Encoding: chunked' */ - if ('c' == c) { - h_state = h_matching_transfer_encoding_chunked; - } else if (STRICT_TOKEN(c)) { - /* TODO(indutny): similar code below does this, but why? - * At the very least it seems to be inconsistent given that - * h_matching_transfer_encoding_token does not check for - * `STRICT_TOKEN` - */ - h_state = h_matching_transfer_encoding_token; - } else if (c == ' ' || c == '\t') { - /* Skip lws */ - } else { - h_state = h_general; - } - break; - - case h_matching_transfer_encoding_chunked: - parser->index++; - if (parser->index > sizeof(CHUNKED)-1 - || c != CHUNKED[parser->index]) { - h_state = h_matching_transfer_encoding_token; - } else if (parser->index == sizeof(CHUNKED)-2) { - h_state = h_transfer_encoding_chunked; - } - break; - - case h_matching_transfer_encoding_token: - if (ch == ',') { - h_state = h_matching_transfer_encoding_token_start; - parser->index = 0; - } - break; - - case h_matching_connection_token_start: - /* looking for 'Connection: keep-alive' */ - if (c == 'k') { - h_state = h_matching_connection_keep_alive; - /* looking for 'Connection: close' */ - } else if (c == 'c') { - h_state = h_matching_connection_close; - } else if (c == 'u') { - h_state = h_matching_connection_upgrade; - } else if (STRICT_TOKEN(c)) { - h_state = h_matching_connection_token; - } else if (c == ' ' || c == '\t') { - /* Skip lws */ - } else { - h_state = h_general; - } - break; - - /* looking for 'Connection: keep-alive' */ - case h_matching_connection_keep_alive: - parser->index++; - if (parser->index > sizeof(KEEP_ALIVE)-1 - || c != KEEP_ALIVE[parser->index]) { - h_state = h_matching_connection_token; - } else if (parser->index == sizeof(KEEP_ALIVE)-2) { - h_state = h_connection_keep_alive; - } - break; - - /* looking for 'Connection: close' */ - case h_matching_connection_close: - parser->index++; - if (parser->index > sizeof(CLOSE)-1 || c != CLOSE[parser->index]) { - h_state = h_matching_connection_token; - } else if (parser->index == sizeof(CLOSE)-2) { - h_state = h_connection_close; - } - break; - - /* looking for 'Connection: upgrade' */ - case h_matching_connection_upgrade: - parser->index++; - if (parser->index > sizeof(UPGRADE) - 1 || - c != UPGRADE[parser->index]) { - h_state = h_matching_connection_token; - } else if (parser->index == sizeof(UPGRADE)-2) { - h_state = h_connection_upgrade; - } - break; - - case h_matching_connection_token: - if (ch == ',') { - h_state = h_matching_connection_token_start; - parser->index = 0; - } - break; - - case h_transfer_encoding_chunked: - if (ch != ' ') h_state = h_matching_transfer_encoding_token; - break; - - case h_connection_keep_alive: - case h_connection_close: - case h_connection_upgrade: - if (ch == ',') { - if (h_state == h_connection_keep_alive) { - parser->flags |= F_CONNECTION_KEEP_ALIVE; - } else if (h_state == h_connection_close) { - parser->flags |= F_CONNECTION_CLOSE; - } else if (h_state == h_connection_upgrade) { - parser->flags |= F_CONNECTION_UPGRADE; - } - h_state = h_matching_connection_token_start; - parser->index = 0; - } else if (ch != ' ') { - h_state = h_matching_connection_token; - } - break; - - default: - UPDATE_STATE(s_header_value); - h_state = h_general; - break; - } - } - parser->header_state = h_state; - - if (p == data + len) - --p; - - COUNT_HEADER_SIZE(p - start); - break; - } - - case s_header_almost_done: - { - if (UNLIKELY(ch != LF)) { - SET_ERRNO(HPE_LF_EXPECTED); - goto error; - } - - UPDATE_STATE(s_header_value_lws); - break; - } - - case s_header_value_lws: - { - if (ch == ' ' || ch == '\t') { - if (parser->header_state == h_content_length_num) { - /* treat obsolete line folding as space */ - parser->header_state = h_content_length_ws; - } - UPDATE_STATE(s_header_value_start); - REEXECUTE(); - } - - /* finished the header */ - switch (parser->header_state) { - case h_connection_keep_alive: - parser->flags |= F_CONNECTION_KEEP_ALIVE; - break; - case h_connection_close: - parser->flags |= F_CONNECTION_CLOSE; - break; - case h_transfer_encoding_chunked: - parser->flags |= F_CHUNKED; - break; - case h_connection_upgrade: - parser->flags |= F_CONNECTION_UPGRADE; - break; - default: - break; - } - - UPDATE_STATE(s_header_field_start); - REEXECUTE(); - } - - case s_header_value_discard_ws_almost_done: - { - STRICT_CHECK(ch != LF); - UPDATE_STATE(s_header_value_discard_lws); - break; - } - - case s_header_value_discard_lws: - { - if (ch == ' ' || ch == '\t') { - UPDATE_STATE(s_header_value_discard_ws); - break; - } else { - switch (parser->header_state) { - case h_connection_keep_alive: - parser->flags |= F_CONNECTION_KEEP_ALIVE; - break; - case h_connection_close: - parser->flags |= F_CONNECTION_CLOSE; - break; - case h_connection_upgrade: - parser->flags |= F_CONNECTION_UPGRADE; - break; - case h_transfer_encoding_chunked: - parser->flags |= F_CHUNKED; - break; - case h_content_length: - /* do not allow empty content length */ - SET_ERRNO(HPE_INVALID_CONTENT_LENGTH); - goto error; - break; - default: - break; - } - - /* header value was empty */ - MARK(header_value); - UPDATE_STATE(s_header_field_start); - CALLBACK_DATA_NOADVANCE(header_value); - REEXECUTE(); - } - } - - case s_headers_almost_done: - { - STRICT_CHECK(ch != LF); - - if (parser->flags & F_TRAILING) { - /* End of a chunked request */ - UPDATE_STATE(s_message_done); - CALLBACK_NOTIFY_NOADVANCE(chunk_complete); - REEXECUTE(); - } - - /* Cannot us transfer-encoding and a content-length header together - per the HTTP specification. (RFC 7230 Section 3.3.3) */ - if ((parser->flags & F_TRANSFER_ENCODING) && - (parser->flags & F_CONTENTLENGTH)) { - /* Allow it for lenient parsing as long as `Transfer-Encoding` is - * not `chunked` - */ - if (!lenient || (parser->flags & F_CHUNKED)) { - SET_ERRNO(HPE_UNEXPECTED_CONTENT_LENGTH); - goto error; - } - } - - UPDATE_STATE(s_headers_done); - - /* Set this here so that on_headers_complete() callbacks can see it */ - if ((parser->flags & F_UPGRADE) && - (parser->flags & F_CONNECTION_UPGRADE)) { - /* For responses, "Upgrade: foo" and "Connection: upgrade" are - * mandatory only when it is a 101 Switching Protocols response, - * otherwise it is purely informational, to announce support. - */ - parser->upgrade = - (parser->type == HTTP_REQUEST || parser->status_code == 101); - } else { - parser->upgrade = (parser->method == HTTP_CONNECT); - } - - /* Here we call the headers_complete callback. This is somewhat - * different than other callbacks because if the user returns 1, we - * will interpret that as saying that this message has no body. This - * is needed for the annoying case of recieving a response to a HEAD - * request. - * - * We'd like to use CALLBACK_NOTIFY_NOADVANCE() here but we cannot, so - * we have to simulate it by handling a change in errno below. - */ - if (settings->on_headers_complete) { - switch (settings->on_headers_complete(parser)) { - case 0: - break; - - case 2: - parser->upgrade = 1; - - /* fall through */ - case 1: - parser->flags |= F_SKIPBODY; - break; - - default: - SET_ERRNO(HPE_CB_headers_complete); - RETURN(p - data); /* Error */ - } - } - - if (HTTP_PARSER_ERRNO(parser) != HPE_OK) { - RETURN(p - data); - } - - REEXECUTE(); - } - - case s_headers_done: - { - int hasBody; - STRICT_CHECK(ch != LF); - - parser->nread = 0; - nread = 0; - - hasBody = parser->flags & F_CHUNKED || - (parser->content_length > 0 && parser->content_length != ULLONG_MAX); - if (parser->upgrade && (parser->method == HTTP_CONNECT || - (parser->flags & F_SKIPBODY) || !hasBody)) { - /* Exit, the rest of the message is in a different protocol. */ - UPDATE_STATE(NEW_MESSAGE()); - CALLBACK_NOTIFY(message_complete); - RETURN((p - data) + 1); - } - - if (parser->flags & F_SKIPBODY) { - UPDATE_STATE(NEW_MESSAGE()); - CALLBACK_NOTIFY(message_complete); - } else if (parser->flags & F_CHUNKED) { - /* chunked encoding - ignore Content-Length header, - * prepare for a chunk */ - UPDATE_STATE(s_chunk_size_start); - } else if (parser->flags & F_TRANSFER_ENCODING) { - if (parser->type == HTTP_REQUEST && !lenient) { - /* RFC 7230 3.3.3 */ - - /* If a Transfer-Encoding header field - * is present in a request and the chunked transfer coding is not - * the final encoding, the message body length cannot be determined - * reliably; the server MUST respond with the 400 (Bad Request) - * status code and then close the connection. - */ - SET_ERRNO(HPE_INVALID_TRANSFER_ENCODING); - RETURN(p - data); /* Error */ - } else { - /* RFC 7230 3.3.3 */ - - /* If a Transfer-Encoding header field is present in a response and - * the chunked transfer coding is not the final encoding, the - * message body length is determined by reading the connection until - * it is closed by the server. - */ - UPDATE_STATE(s_body_identity_eof); - } - } else { - if (parser->content_length == 0) { - /* Content-Length header given but zero: Content-Length: 0\r\n */ - UPDATE_STATE(NEW_MESSAGE()); - CALLBACK_NOTIFY(message_complete); - } else if (parser->content_length != ULLONG_MAX) { - /* Content-Length header given and non-zero */ - UPDATE_STATE(s_body_identity); - } else { - if (!http_message_needs_eof(parser)) { - /* Assume content-length 0 - read the next */ - UPDATE_STATE(NEW_MESSAGE()); - CALLBACK_NOTIFY(message_complete); - } else { - /* Read body until EOF */ - UPDATE_STATE(s_body_identity_eof); - } - } - } - - break; - } - - case s_body_identity: - { - uint64_t to_read = MIN(parser->content_length, - (uint64_t) ((data + len) - p)); - - assert(parser->content_length != 0 - && parser->content_length != ULLONG_MAX); - - /* The difference between advancing content_length and p is because - * the latter will automaticaly advance on the next loop iteration. - * Further, if content_length ends up at 0, we want to see the last - * byte again for our message complete callback. - */ - MARK(body); - parser->content_length -= to_read; - p += to_read - 1; - - if (parser->content_length == 0) { - UPDATE_STATE(s_message_done); - - /* Mimic CALLBACK_DATA_NOADVANCE() but with one extra byte. - * - * The alternative to doing this is to wait for the next byte to - * trigger the data callback, just as in every other case. The - * problem with this is that this makes it difficult for the test - * harness to distinguish between complete-on-EOF and - * complete-on-length. It's not clear that this distinction is - * important for applications, but let's keep it for now. - */ - CALLBACK_DATA_(body, p - body_mark + 1, p - data); - REEXECUTE(); - } - - break; - } - - /* read until EOF */ - case s_body_identity_eof: - MARK(body); - p = data + len - 1; - - break; - - case s_message_done: - UPDATE_STATE(NEW_MESSAGE()); - CALLBACK_NOTIFY(message_complete); - if (parser->upgrade) { - /* Exit, the rest of the message is in a different protocol. */ - RETURN((p - data) + 1); - } - break; - - case s_chunk_size_start: - { - assert(nread == 1); - assert(parser->flags & F_CHUNKED); - - unhex_val = unhex[(unsigned char)ch]; - if (UNLIKELY(unhex_val == -1)) { - SET_ERRNO(HPE_INVALID_CHUNK_SIZE); - goto error; - } - - parser->content_length = unhex_val; - UPDATE_STATE(s_chunk_size); - break; - } - - case s_chunk_size: - { - uint64_t t; - - assert(parser->flags & F_CHUNKED); - - if (ch == CR) { - UPDATE_STATE(s_chunk_size_almost_done); - break; - } - - unhex_val = unhex[(unsigned char)ch]; - - if (unhex_val == -1) { - if (ch == ';' || ch == ' ') { - UPDATE_STATE(s_chunk_parameters); - break; - } - - SET_ERRNO(HPE_INVALID_CHUNK_SIZE); - goto error; - } - - t = parser->content_length; - t *= 16; - t += unhex_val; - - /* Overflow? Test against a conservative limit for simplicity. */ - if (UNLIKELY((ULLONG_MAX - 16) / 16 < parser->content_length)) { - SET_ERRNO(HPE_INVALID_CONTENT_LENGTH); - goto error; - } - - parser->content_length = t; - break; - } - - case s_chunk_parameters: - { - assert(parser->flags & F_CHUNKED); - /* just ignore this shit. TODO check for overflow */ - if (ch == CR) { - UPDATE_STATE(s_chunk_size_almost_done); - break; - } - break; - } - - case s_chunk_size_almost_done: - { - assert(parser->flags & F_CHUNKED); - STRICT_CHECK(ch != LF); - - parser->nread = 0; - nread = 0; - - if (parser->content_length == 0) { - parser->flags |= F_TRAILING; - UPDATE_STATE(s_header_field_start); - } else { - UPDATE_STATE(s_chunk_data); - } - CALLBACK_NOTIFY(chunk_header); - break; - } - - case s_chunk_data: - { - uint64_t to_read = MIN(parser->content_length, - (uint64_t) ((data + len) - p)); - - assert(parser->flags & F_CHUNKED); - assert(parser->content_length != 0 - && parser->content_length != ULLONG_MAX); - - /* See the explanation in s_body_identity for why the content - * length and data pointers are managed this way. - */ - MARK(body); - parser->content_length -= to_read; - p += to_read - 1; - - if (parser->content_length == 0) { - UPDATE_STATE(s_chunk_data_almost_done); - } - - break; - } - - case s_chunk_data_almost_done: - assert(parser->flags & F_CHUNKED); - assert(parser->content_length == 0); - STRICT_CHECK(ch != CR); - UPDATE_STATE(s_chunk_data_done); - CALLBACK_DATA(body); - break; - - case s_chunk_data_done: - assert(parser->flags & F_CHUNKED); - STRICT_CHECK(ch != LF); - parser->nread = 0; - nread = 0; - UPDATE_STATE(s_chunk_size_start); - CALLBACK_NOTIFY(chunk_complete); - break; - - default: - assert(0 && "unhandled state"); - SET_ERRNO(HPE_INVALID_INTERNAL_STATE); - goto error; - } - } - - /* Run callbacks for any marks that we have leftover after we ran out of - * bytes. There should be at most one of these set, so it's OK to invoke - * them in series (unset marks will not result in callbacks). - * - * We use the NOADVANCE() variety of callbacks here because 'p' has already - * overflowed 'data' and this allows us to correct for the off-by-one that - * we'd otherwise have (since CALLBACK_DATA() is meant to be run with a 'p' - * value that's in-bounds). - */ - - assert(((header_field_mark ? 1 : 0) + - (header_value_mark ? 1 : 0) + - (url_mark ? 1 : 0) + - (body_mark ? 1 : 0) + - (status_mark ? 1 : 0)) <= 1); - - CALLBACK_DATA_NOADVANCE(header_field); - CALLBACK_DATA_NOADVANCE(header_value); - CALLBACK_DATA_NOADVANCE(url); - CALLBACK_DATA_NOADVANCE(body); - CALLBACK_DATA_NOADVANCE(status); - - RETURN(len); - -error: - if (HTTP_PARSER_ERRNO(parser) == HPE_OK) { - SET_ERRNO(HPE_UNKNOWN); - } - - RETURN(p - data); -} - - -/* Does the parser need to see an EOF to find the end of the message? */ -int -http_message_needs_eof (const http_parser *parser) -{ - if (parser->type == HTTP_REQUEST) { - return 0; - } - - /* See RFC 2616 section 4.4 */ - if (parser->status_code / 100 == 1 || /* 1xx e.g. Continue */ - parser->status_code == 204 || /* No Content */ - parser->status_code == 304 || /* Not Modified */ - parser->flags & F_SKIPBODY) { /* response to a HEAD request */ - return 0; - } - - /* RFC 7230 3.3.3, see `s_headers_almost_done` */ - if ((parser->flags & F_TRANSFER_ENCODING) && - (parser->flags & F_CHUNKED) == 0) { - return 1; - } - - if ((parser->flags & F_CHUNKED) || parser->content_length != ULLONG_MAX) { - return 0; - } - - return 1; -} - - -int -http_should_keep_alive (const http_parser *parser) -{ - if (parser->http_major > 0 && parser->http_minor > 0) { - /* HTTP/1.1 */ - if (parser->flags & F_CONNECTION_CLOSE) { - return 0; - } - } else { - /* HTTP/1.0 or earlier */ - if (!(parser->flags & F_CONNECTION_KEEP_ALIVE)) { - return 0; - } - } - - return !http_message_needs_eof(parser); -} - - -const char * -http_method_str (enum http_method m) -{ - return ELEM_AT(method_strings, m, ""); -} - -const char * -http_status_str (enum http_status s) -{ - switch (s) { -#define XX(num, name, string) case HTTP_STATUS_##name: return #string; - HTTP_STATUS_MAP(XX) -#undef XX - default: return ""; - } -} - -void -http_parser_init (http_parser *parser, enum http_parser_type t) -{ - void *data = parser->data; /* preserve application data */ - memset(parser, 0, sizeof(*parser)); - parser->data = data; - parser->type = t; - parser->state = (t == HTTP_REQUEST ? s_start_req : (t == HTTP_RESPONSE ? s_start_res : s_start_req_or_res)); - parser->http_errno = HPE_OK; -} - -void -http_parser_settings_init(http_parser_settings *settings) -{ - memset(settings, 0, sizeof(*settings)); -} - -const char * -http_errno_name(enum http_errno err) { - assert(((size_t) err) < ARRAY_SIZE(http_strerror_tab)); - return http_strerror_tab[err].name; -} - -const char * -http_errno_description(enum http_errno err) { - assert(((size_t) err) < ARRAY_SIZE(http_strerror_tab)); - return http_strerror_tab[err].description; -} - -static enum http_host_state -http_parse_host_char(enum http_host_state s, const char ch) { - switch(s) { - case s_http_userinfo: - case s_http_userinfo_start: - if (ch == '@') { - return s_http_host_start; - } - - if (IS_USERINFO_CHAR(ch)) { - return s_http_userinfo; - } - break; - - case s_http_host_start: - if (ch == '[') { - return s_http_host_v6_start; - } - - if (IS_HOST_CHAR(ch)) { - return s_http_host; - } - - break; - - case s_http_host: - if (IS_HOST_CHAR(ch)) { - return s_http_host; - } - - /* fall through */ - case s_http_host_v6_end: - if (ch == ':') { - return s_http_host_port_start; - } - - break; - - case s_http_host_v6: - if (ch == ']') { - return s_http_host_v6_end; - } - - /* fall through */ - case s_http_host_v6_start: - if (IS_HEX(ch) || ch == ':' || ch == '.') { - return s_http_host_v6; - } - - if (s == s_http_host_v6 && ch == '%') { - return s_http_host_v6_zone_start; - } - break; - - case s_http_host_v6_zone: - if (ch == ']') { - return s_http_host_v6_end; - } - - /* fall through */ - case s_http_host_v6_zone_start: - /* RFC 6874 Zone ID consists of 1*( unreserved / pct-encoded) */ - if (IS_ALPHANUM(ch) || ch == '%' || ch == '.' || ch == '-' || ch == '_' || - ch == '~') { - return s_http_host_v6_zone; - } - break; - - case s_http_host_port: - case s_http_host_port_start: - if (IS_NUM(ch)) { - return s_http_host_port; - } - - break; - - default: - break; - } - return s_http_host_dead; -} - -static int -http_parse_host(const char * buf, struct http_parser_url *u, int found_at) { - enum http_host_state s; - - const char *p; - size_t buflen = u->field_data[UF_HOST].off + u->field_data[UF_HOST].len; - - assert(u->field_set & (1 << UF_HOST)); - - u->field_data[UF_HOST].len = 0; - - s = found_at ? s_http_userinfo_start : s_http_host_start; - - for (p = buf + u->field_data[UF_HOST].off; p < buf + buflen; p++) { - enum http_host_state new_s = http_parse_host_char(s, *p); - - if (new_s == s_http_host_dead) { - return 1; - } - - switch(new_s) { - case s_http_host: - if (s != s_http_host) { - u->field_data[UF_HOST].off = (uint16_t)(p - buf); - } - u->field_data[UF_HOST].len++; - break; - - case s_http_host_v6: - if (s != s_http_host_v6) { - u->field_data[UF_HOST].off = (uint16_t)(p - buf); - } - u->field_data[UF_HOST].len++; - break; - - case s_http_host_v6_zone_start: - case s_http_host_v6_zone: - u->field_data[UF_HOST].len++; - break; - - case s_http_host_port: - if (s != s_http_host_port) { - u->field_data[UF_PORT].off = (uint16_t)(p - buf); - u->field_data[UF_PORT].len = 0; - u->field_set |= (1 << UF_PORT); - } - u->field_data[UF_PORT].len++; - break; - - case s_http_userinfo: - if (s != s_http_userinfo) { - u->field_data[UF_USERINFO].off = (uint16_t)(p - buf); - u->field_data[UF_USERINFO].len = 0; - u->field_set |= (1 << UF_USERINFO); - } - u->field_data[UF_USERINFO].len++; - break; - - default: - break; - } - s = new_s; - } - - /* Make sure we don't end somewhere unexpected */ - switch (s) { - case s_http_host_start: - case s_http_host_v6_start: - case s_http_host_v6: - case s_http_host_v6_zone_start: - case s_http_host_v6_zone: - case s_http_host_port_start: - case s_http_userinfo: - case s_http_userinfo_start: - return 1; - default: - break; - } - - return 0; -} - -void -http_parser_url_init(struct http_parser_url *u) { - memset(u, 0, sizeof(*u)); -} - -int -http_parser_parse_url(const char *buf, size_t buflen, int is_connect, - struct http_parser_url *u) -{ - enum state s; - const char *p; - enum http_parser_url_fields uf, old_uf; - int found_at = 0; - - if (buflen == 0) { - return 1; - } - - u->port = u->field_set = 0; - s = is_connect ? s_req_server_start : s_req_spaces_before_url; - old_uf = UF_MAX; - - for (p = buf; p < buf + buflen; p++) { - s = parse_url_char(s, *p); - - /* Figure out the next field that we're operating on */ - switch (s) { - case s_dead: - return 1; - - /* Skip delimeters */ - case s_req_schema_slash: - case s_req_schema_slash_slash: - case s_req_server_start: - case s_req_query_string_start: - case s_req_fragment_start: - continue; - - case s_req_schema: - uf = UF_SCHEMA; - break; - - case s_req_server_with_at: - found_at = 1; - - /* fall through */ - case s_req_server: - uf = UF_HOST; - break; - - case s_req_path: - uf = UF_PATH; - break; - - case s_req_query_string: - uf = UF_QUERY; - break; - - case s_req_fragment: - uf = UF_FRAGMENT; - break; - - default: - assert(!"Unexpected state"); - return 1; - } - - /* Nothing's changed; soldier on */ - if (uf == old_uf) { - u->field_data[uf].len++; - continue; - } - - u->field_data[uf].off = (uint16_t)(p - buf); - u->field_data[uf].len = 1; - - u->field_set |= (1 << uf); - old_uf = uf; - } - - /* host must be present if there is a schema */ - /* parsing http:///toto will fail */ - if ((u->field_set & (1 << UF_SCHEMA)) && - (u->field_set & (1 << UF_HOST)) == 0) { - return 1; - } - - if (u->field_set & (1 << UF_HOST)) { - if (http_parse_host(buf, u, found_at) != 0) { - return 1; - } - } - - /* CONNECT requests can only contain "hostname:port" */ - if (is_connect && u->field_set != ((1 << UF_HOST)|(1 << UF_PORT))) { - return 1; - } - - if (u->field_set & (1 << UF_PORT)) { - uint16_t off; - uint16_t len; - const char* p; - const char* end; - unsigned long v; - - off = u->field_data[UF_PORT].off; - len = u->field_data[UF_PORT].len; - end = buf + off + len; - - /* NOTE: The characters are already validated and are in the [0-9] range */ - assert(off + len <= buflen && "Port number overflow"); - v = 0; - for (p = buf + off; p < end; p++) { - v *= 10; - v += *p - '0'; - - /* Ports have a max value of 2^16 */ - if (v > 0xffff) { - return 1; - } - } - - u->port = (uint16_t) v; - } - - return 0; -} - -void -http_parser_pause(http_parser *parser, int paused) { - /* Users should only be pausing/unpausing a parser that is not in an error - * state. In non-debug builds, there's not much that we can do about this - * other than ignore it. - */ - if (HTTP_PARSER_ERRNO(parser) == HPE_OK || - HTTP_PARSER_ERRNO(parser) == HPE_PAUSED) { - uint32_t nread = parser->nread; /* used by the SET_ERRNO macro */ - SET_ERRNO((paused) ? HPE_PAUSED : HPE_OK); - } else { - assert(0 && "Attempting to pause parser in error state"); - } -} - -int -http_body_is_final(const struct http_parser *parser) { - return parser->state == s_message_done; -} - -unsigned long -http_parser_version(void) { - return HTTP_PARSER_VERSION_MAJOR * 0x10000 | - HTTP_PARSER_VERSION_MINOR * 0x00100 | - HTTP_PARSER_VERSION_PATCH * 0x00001; -} - -void -http_parser_set_max_header_size(uint32_t size) { - max_header_size = size; -} diff --git a/src/3rdparty/http-parser/http_parser.h b/src/3rdparty/http-parser/http_parser.h deleted file mode 100644 index 983d88a9..00000000 --- a/src/3rdparty/http-parser/http_parser.h +++ /dev/null @@ -1,442 +0,0 @@ -/* Copyright Joyent, Inc. and other Node contributors. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ -#ifndef http_parser_h -#define http_parser_h -#ifdef __cplusplus -extern "C" { -#endif - -/* Also update SONAME in the Makefile whenever you change these. */ -#define HTTP_PARSER_VERSION_MAJOR 2 -#define HTTP_PARSER_VERSION_MINOR 9 -#define HTTP_PARSER_VERSION_PATCH 3 - -#include -#if defined(_WIN32) && !defined(__MINGW32__) && \ - (!defined(_MSC_VER) || _MSC_VER<1600) && !defined(__WINE__) -#include -typedef __int8 int8_t; -typedef unsigned __int8 uint8_t; -typedef __int16 int16_t; -typedef unsigned __int16 uint16_t; -typedef __int32 int32_t; -typedef unsigned __int32 uint32_t; -typedef __int64 int64_t; -typedef unsigned __int64 uint64_t; -#else -#include -#endif - -/* Compile with -DHTTP_PARSER_STRICT=0 to make less checks, but run - * faster - */ -#ifndef HTTP_PARSER_STRICT -# define HTTP_PARSER_STRICT 1 -#endif - -/* Maximium header size allowed. If the macro is not defined - * before including this header then the default is used. To - * change the maximum header size, define the macro in the build - * environment (e.g. -DHTTP_MAX_HEADER_SIZE=). To remove - * the effective limit on the size of the header, define the macro - * to a very large number (e.g. -DHTTP_MAX_HEADER_SIZE=0x7fffffff) - */ -#ifndef HTTP_MAX_HEADER_SIZE -# define HTTP_MAX_HEADER_SIZE (80*1024) -#endif - -typedef struct http_parser http_parser; -typedef struct http_parser_settings http_parser_settings; - - -/* Callbacks should return non-zero to indicate an error. The parser will - * then halt execution. - * - * The one exception is on_headers_complete. In a HTTP_RESPONSE parser - * returning '1' from on_headers_complete will tell the parser that it - * should not expect a body. This is used when receiving a response to a - * HEAD request which may contain 'Content-Length' or 'Transfer-Encoding: - * chunked' headers that indicate the presence of a body. - * - * Returning `2` from on_headers_complete will tell parser that it should not - * expect neither a body nor any futher responses on this connection. This is - * useful for handling responses to a CONNECT request which may not contain - * `Upgrade` or `Connection: upgrade` headers. - * - * http_data_cb does not return data chunks. It will be called arbitrarily - * many times for each string. E.G. you might get 10 callbacks for "on_url" - * each providing just a few characters more data. - */ -typedef int (*http_data_cb) (http_parser*, const char *at, size_t length); -typedef int (*http_cb) (http_parser*); - - -/* Status Codes */ -#define HTTP_STATUS_MAP(XX) \ - XX(100, CONTINUE, Continue) \ - XX(101, SWITCHING_PROTOCOLS, Switching Protocols) \ - XX(102, PROCESSING, Processing) \ - XX(200, OK, OK) \ - XX(201, CREATED, Created) \ - XX(202, ACCEPTED, Accepted) \ - XX(203, NON_AUTHORITATIVE_INFORMATION, Non-Authoritative Information) \ - XX(204, NO_CONTENT, No Content) \ - XX(205, RESET_CONTENT, Reset Content) \ - XX(206, PARTIAL_CONTENT, Partial Content) \ - XX(207, MULTI_STATUS, Multi-Status) \ - XX(208, ALREADY_REPORTED, Already Reported) \ - XX(226, IM_USED, IM Used) \ - XX(300, MULTIPLE_CHOICES, Multiple Choices) \ - XX(301, MOVED_PERMANENTLY, Moved Permanently) \ - XX(302, FOUND, Found) \ - XX(303, SEE_OTHER, See Other) \ - XX(304, NOT_MODIFIED, Not Modified) \ - XX(305, USE_PROXY, Use Proxy) \ - XX(307, TEMPORARY_REDIRECT, Temporary Redirect) \ - XX(308, PERMANENT_REDIRECT, Permanent Redirect) \ - XX(400, BAD_REQUEST, Bad Request) \ - XX(401, UNAUTHORIZED, Unauthorized) \ - XX(402, PAYMENT_REQUIRED, Payment Required) \ - XX(403, FORBIDDEN, Forbidden) \ - XX(404, NOT_FOUND, Not Found) \ - XX(405, METHOD_NOT_ALLOWED, Method Not Allowed) \ - XX(406, NOT_ACCEPTABLE, Not Acceptable) \ - XX(407, PROXY_AUTHENTICATION_REQUIRED, Proxy Authentication Required) \ - XX(408, REQUEST_TIMEOUT, Request Timeout) \ - XX(409, CONFLICT, Conflict) \ - XX(410, GONE, Gone) \ - XX(411, LENGTH_REQUIRED, Length Required) \ - XX(412, PRECONDITION_FAILED, Precondition Failed) \ - XX(413, PAYLOAD_TOO_LARGE, Payload Too Large) \ - XX(414, URI_TOO_LONG, URI Too Long) \ - XX(415, UNSUPPORTED_MEDIA_TYPE, Unsupported Media Type) \ - XX(416, RANGE_NOT_SATISFIABLE, Range Not Satisfiable) \ - XX(417, EXPECTATION_FAILED, Expectation Failed) \ - XX(421, MISDIRECTED_REQUEST, Misdirected Request) \ - XX(422, UNPROCESSABLE_ENTITY, Unprocessable Entity) \ - XX(423, LOCKED, Locked) \ - XX(424, FAILED_DEPENDENCY, Failed Dependency) \ - XX(426, UPGRADE_REQUIRED, Upgrade Required) \ - XX(428, PRECONDITION_REQUIRED, Precondition Required) \ - XX(429, TOO_MANY_REQUESTS, Too Many Requests) \ - XX(431, REQUEST_HEADER_FIELDS_TOO_LARGE, Request Header Fields Too Large) \ - XX(451, UNAVAILABLE_FOR_LEGAL_REASONS, Unavailable For Legal Reasons) \ - XX(500, INTERNAL_SERVER_ERROR, Internal Server Error) \ - XX(501, NOT_IMPLEMENTED, Not Implemented) \ - XX(502, BAD_GATEWAY, Bad Gateway) \ - XX(503, SERVICE_UNAVAILABLE, Service Unavailable) \ - XX(504, GATEWAY_TIMEOUT, Gateway Timeout) \ - XX(505, HTTP_VERSION_NOT_SUPPORTED, HTTP Version Not Supported) \ - XX(506, VARIANT_ALSO_NEGOTIATES, Variant Also Negotiates) \ - XX(507, INSUFFICIENT_STORAGE, Insufficient Storage) \ - XX(508, LOOP_DETECTED, Loop Detected) \ - XX(510, NOT_EXTENDED, Not Extended) \ - XX(511, NETWORK_AUTHENTICATION_REQUIRED, Network Authentication Required) \ - -enum http_status - { -#define XX(num, name, string) HTTP_STATUS_##name = num, - HTTP_STATUS_MAP(XX) -#undef XX - }; - - -/* Request Methods */ -#define HTTP_METHOD_MAP(XX) \ - XX(0, DELETE, DELETE) \ - XX(1, GET, GET) \ - XX(2, HEAD, HEAD) \ - XX(3, POST, POST) \ - XX(4, PUT, PUT) \ - /* pathological */ \ - XX(5, CONNECT, CONNECT) \ - XX(6, OPTIONS, OPTIONS) \ - XX(7, TRACE, TRACE) \ - /* WebDAV */ \ - XX(8, COPY, COPY) \ - XX(9, LOCK, LOCK) \ - XX(10, MKCOL, MKCOL) \ - XX(11, MOVE, MOVE) \ - XX(12, PROPFIND, PROPFIND) \ - XX(13, PROPPATCH, PROPPATCH) \ - XX(14, SEARCH, SEARCH) \ - XX(15, UNLOCK, UNLOCK) \ - XX(16, BIND, BIND) \ - XX(17, REBIND, REBIND) \ - XX(18, UNBIND, UNBIND) \ - XX(19, ACL, ACL) \ - /* subversion */ \ - XX(20, REPORT, REPORT) \ - XX(21, MKACTIVITY, MKACTIVITY) \ - XX(22, CHECKOUT, CHECKOUT) \ - XX(23, MERGE, MERGE) \ - /* upnp */ \ - XX(24, MSEARCH, M-SEARCH) \ - XX(25, NOTIFY, NOTIFY) \ - XX(26, SUBSCRIBE, SUBSCRIBE) \ - XX(27, UNSUBSCRIBE, UNSUBSCRIBE) \ - /* RFC-5789 */ \ - XX(28, PATCH, PATCH) \ - XX(29, PURGE, PURGE) \ - /* CalDAV */ \ - XX(30, MKCALENDAR, MKCALENDAR) \ - /* RFC-2068, section 19.6.1.2 */ \ - XX(31, LINK, LINK) \ - XX(32, UNLINK, UNLINK) \ - /* icecast */ \ - XX(33, SOURCE, SOURCE) \ - -enum http_method - { -#define XX(num, name, string) HTTP_##name = num, - HTTP_METHOD_MAP(XX) -#undef XX - }; - - -enum http_parser_type { HTTP_REQUEST, HTTP_RESPONSE, HTTP_BOTH }; - - -/* Flag values for http_parser.flags field */ -enum flags - { F_CHUNKED = 1 << 0 - , F_CONNECTION_KEEP_ALIVE = 1 << 1 - , F_CONNECTION_CLOSE = 1 << 2 - , F_CONNECTION_UPGRADE = 1 << 3 - , F_TRAILING = 1 << 4 - , F_UPGRADE = 1 << 5 - , F_SKIPBODY = 1 << 6 - , F_CONTENTLENGTH = 1 << 7 - , F_TRANSFER_ENCODING = 1 << 8 - }; - - -/* Map for errno-related constants - * - * The provided argument should be a macro that takes 2 arguments. - */ -#define HTTP_ERRNO_MAP(XX) \ - /* No error */ \ - XX(OK, "success") \ - \ - /* Callback-related errors */ \ - XX(CB_message_begin, "the on_message_begin callback failed") \ - XX(CB_url, "the on_url callback failed") \ - XX(CB_header_field, "the on_header_field callback failed") \ - XX(CB_header_value, "the on_header_value callback failed") \ - XX(CB_headers_complete, "the on_headers_complete callback failed") \ - XX(CB_body, "the on_body callback failed") \ - XX(CB_message_complete, "the on_message_complete callback failed") \ - XX(CB_status, "the on_status callback failed") \ - XX(CB_chunk_header, "the on_chunk_header callback failed") \ - XX(CB_chunk_complete, "the on_chunk_complete callback failed") \ - \ - /* Parsing-related errors */ \ - XX(INVALID_EOF_STATE, "stream ended at an unexpected time") \ - XX(HEADER_OVERFLOW, \ - "too many header bytes seen; overflow detected") \ - XX(CLOSED_CONNECTION, \ - "data received after completed connection: close message") \ - XX(INVALID_VERSION, "invalid HTTP version") \ - XX(INVALID_STATUS, "invalid HTTP status code") \ - XX(INVALID_METHOD, "invalid HTTP method") \ - XX(INVALID_URL, "invalid URL") \ - XX(INVALID_HOST, "invalid host") \ - XX(INVALID_PORT, "invalid port") \ - XX(INVALID_PATH, "invalid path") \ - XX(INVALID_QUERY_STRING, "invalid query string") \ - XX(INVALID_FRAGMENT, "invalid fragment") \ - XX(LF_EXPECTED, "LF character expected") \ - XX(INVALID_HEADER_TOKEN, "invalid character in header") \ - XX(INVALID_CONTENT_LENGTH, \ - "invalid character in content-length header") \ - XX(UNEXPECTED_CONTENT_LENGTH, \ - "unexpected content-length header") \ - XX(INVALID_CHUNK_SIZE, \ - "invalid character in chunk size header") \ - XX(INVALID_TRANSFER_ENCODING, \ - "request has invalid transfer-encoding") \ - XX(INVALID_CONSTANT, "invalid constant string") \ - XX(INVALID_INTERNAL_STATE, "encountered unexpected internal state")\ - XX(STRICT, "strict mode assertion failed") \ - XX(PAUSED, "parser is paused") \ - XX(UNKNOWN, "an unknown error occurred") - - -/* Define HPE_* values for each errno value above */ -#define HTTP_ERRNO_GEN(n, s) HPE_##n, -enum http_errno { - HTTP_ERRNO_MAP(HTTP_ERRNO_GEN) -}; -#undef HTTP_ERRNO_GEN - - -/* Get an http_errno value from an http_parser */ -#define HTTP_PARSER_ERRNO(p) ((enum http_errno) (p)->http_errno) - - -struct http_parser { - /** PRIVATE **/ - unsigned int type : 2; /* enum http_parser_type */ - unsigned int state : 7; /* enum state from http_parser.c */ - unsigned int header_state : 7; /* enum header_state from http_parser.c */ - unsigned int index : 7; /* index into current matcher */ - unsigned int lenient_http_headers : 1; - unsigned int flags : 16; /* F_* values from 'flags' enum; semi-public */ - - uint32_t nread; /* # bytes read in various scenarios */ - uint64_t content_length; /* # bytes in body (0 if no Content-Length header) */ - - /** READ-ONLY **/ - unsigned short http_major; - unsigned short http_minor; - unsigned int status_code : 16; /* responses only */ - unsigned int method : 8; /* requests only */ - unsigned int http_errno : 7; - - /* 1 = Upgrade header was present and the parser has exited because of that. - * 0 = No upgrade header present. - * Should be checked when http_parser_execute() returns in addition to - * error checking. - */ - unsigned int upgrade : 1; - - /** PUBLIC **/ - void *data; /* A pointer to get hook to the "connection" or "socket" object */ -}; - - -struct http_parser_settings { - http_cb on_message_begin; - http_data_cb on_url; - http_data_cb on_status; - http_data_cb on_header_field; - http_data_cb on_header_value; - http_cb on_headers_complete; - http_data_cb on_body; - http_cb on_message_complete; - /* When on_chunk_header is called, the current chunk length is stored - * in parser->content_length. - */ - http_cb on_chunk_header; - http_cb on_chunk_complete; -}; - - -enum http_parser_url_fields - { UF_SCHEMA = 0 - , UF_HOST = 1 - , UF_PORT = 2 - , UF_PATH = 3 - , UF_QUERY = 4 - , UF_FRAGMENT = 5 - , UF_USERINFO = 6 - , UF_MAX = 7 - }; - - -/* Result structure for http_parser_parse_url(). - * - * Callers should index into field_data[] with UF_* values iff field_set - * has the relevant (1 << UF_*) bit set. As a courtesy to clients (and - * because we probably have padding left over), we convert any port to - * a uint16_t. - */ -struct http_parser_url { - uint16_t field_set; /* Bitmask of (1 << UF_*) values */ - uint16_t port; /* Converted UF_PORT string */ - - struct { - uint16_t off; /* Offset into buffer in which field starts */ - uint16_t len; /* Length of run in buffer */ - } field_data[UF_MAX]; -}; - - -/* Returns the library version. Bits 16-23 contain the major version number, - * bits 8-15 the minor version number and bits 0-7 the patch level. - * Usage example: - * - * unsigned long version = http_parser_version(); - * unsigned major = (version >> 16) & 255; - * unsigned minor = (version >> 8) & 255; - * unsigned patch = version & 255; - * printf("http_parser v%u.%u.%u\n", major, minor, patch); - */ -unsigned long http_parser_version(void); - -void http_parser_init(http_parser *parser, enum http_parser_type type); - - -/* Initialize http_parser_settings members to 0 - */ -void http_parser_settings_init(http_parser_settings *settings); - - -/* Executes the parser. Returns number of parsed bytes. Sets - * `parser->http_errno` on error. */ -size_t http_parser_execute(http_parser *parser, - const http_parser_settings *settings, - const char *data, - size_t len); - - -/* If http_should_keep_alive() in the on_headers_complete or - * on_message_complete callback returns 0, then this should be - * the last message on the connection. - * If you are the server, respond with the "Connection: close" header. - * If you are the client, close the connection. - */ -int http_should_keep_alive(const http_parser *parser); - -/* Returns a string version of the HTTP method. */ -const char *http_method_str(enum http_method m); - -/* Returns a string version of the HTTP status code. */ -const char *http_status_str(enum http_status s); - -/* Return a string name of the given error */ -const char *http_errno_name(enum http_errno err); - -/* Return a string description of the given error */ -const char *http_errno_description(enum http_errno err); - -/* Initialize all http_parser_url members to 0 */ -void http_parser_url_init(struct http_parser_url *u); - -/* Parse a URL; return nonzero on failure */ -int http_parser_parse_url(const char *buf, size_t buflen, - int is_connect, - struct http_parser_url *u); - -/* Pause or un-pause the parser; a nonzero value pauses */ -void http_parser_pause(http_parser *parser, int paused); - -/* Checks if this is the final chunk of the body. */ -int http_body_is_final(const http_parser *parser); - -/* Change the maximum header size provided at compile time. */ -void http_parser_set_max_header_size(uint32_t size); - -#ifdef __cplusplus -} -#endif -#endif diff --git a/src/3rdparty/llhttp/LICENSE-MIT b/src/3rdparty/llhttp/LICENSE-MIT new file mode 100644 index 00000000..6c1512dd --- /dev/null +++ b/src/3rdparty/llhttp/LICENSE-MIT @@ -0,0 +1,22 @@ +This software is licensed under the MIT License. + +Copyright Fedor Indutny, 2018. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit +persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/src/3rdparty/llhttp/README.md b/src/3rdparty/llhttp/README.md new file mode 100644 index 00000000..ff72e07f --- /dev/null +++ b/src/3rdparty/llhttp/README.md @@ -0,0 +1,135 @@ +# llhttp +[![CI](https://github.com/nodejs/llhttp/workflows/CI/badge.svg)](https://github.com/nodejs/llhttp/actions?query=workflow%3ACI) + +Port of [http_parser][0] to [llparse][1]. + +## Why? + +Let's face it, [http_parser][0] is practically unmaintainable. Even +introduction of a single new method results in a significant code churn. + +This project aims to: + +* Make it maintainable +* Verifiable +* Improving benchmarks where possible + +More details in [Fedor Indutny's talk at JSConf EU 2019](https://youtu.be/x3k_5Mi66sY) + +## How? + +Over time, different approaches for improving [http_parser][0]'s code base +were tried. However, all of them failed due to resulting significant performance +degradation. + +This project is a port of [http_parser][0] to TypeScript. [llparse][1] is used +to generate the output C source file, which could be compiled and +linked with the embedder's program (like [Node.js][7]). + +## Performance + +So far llhttp outperforms http_parser: + +| | input size | bandwidth | reqs/sec | time | +|:----------------|-----------:|-------------:|-----------:|--------:| +| **llhttp** | 8192.00 mb | 1777.24 mb/s | 3583799.39 req/sec | 4.61 s | +| **http_parser** | 8192.00 mb | 694.66 mb/s | 1406180.33 req/sec | 11.79 s | + +llhttp is faster by approximately **156%**. + +## Maintenance + +llhttp project has about 1400 lines of TypeScript code describing the parser +itself and around 450 lines of C code and headers providing the helper methods. +The whole [http_parser][0] is implemented in approximately 2500 lines of C, and +436 lines of headers. + +All optimizations and multi-character matching in llhttp are generated +automatically, and thus doesn't add any extra maintenance cost. On the contrary, +most of http_parser's code is hand-optimized and unrolled. Instead describing +"how" it should parse the HTTP requests/responses, a maintainer should +implement the new features in [http_parser][0] cautiously, considering +possible performance degradation and manually optimizing the new code. + +## Verification + +The state machine graph is encoded explicitly in llhttp. The [llparse][1] +automatically checks the graph for absence of loops and correct reporting of the +input ranges (spans) like header names and values. In the future, additional +checks could be performed to get even stricter verification of the llhttp. + +## Usage + +```C +#include "llhttp.h" + +llhttp_t parser; +llhttp_settings_t settings; + +/* Initialize user callbacks and settings */ +llhttp_settings_init(&settings); + +/* Set user callback */ +settings.on_message_complete = handle_on_message_complete; + +/* Initialize the parser in HTTP_BOTH mode, meaning that it will select between + * HTTP_REQUEST and HTTP_RESPONSE parsing automatically while reading the first + * input. + */ +llhttp_init(&parser, HTTP_BOTH, &settings); + +/* Parse request! */ +const char* request = "GET / HTTP/1.1\r\n\r\n"; +int request_len = strlen(request); + +enum llhttp_errno err = llhttp_execute(&parser, request, request_len); +if (err == HPE_OK) { + /* Successfully parsed! */ +} else { + fprintf(stderr, "Parse error: %s %s\n", llhttp_errno_name(err), + parser.reason); +} +``` + +--- + +### Bindings to other languages + +* Python: [pallas/pyllhttp][8] +* Ruby: [metabahn/llhttp][9] + +#### LICENSE + +This software is licensed under the MIT License. + +Copyright Fedor Indutny, 2018. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit +persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +USE OR OTHER DEALINGS IN THE SOFTWARE. + +[0]: https://github.com/nodejs/http-parser +[1]: https://github.com/nodejs/llparse +[2]: https://en.wikipedia.org/wiki/Register_allocation#Spilling +[3]: https://en.wikipedia.org/wiki/Tail_call +[4]: https://llvm.org/docs/LangRef.html +[5]: https://llvm.org/docs/LangRef.html#call-instruction +[6]: https://clang.llvm.org/ +[7]: https://github.com/nodejs/node +[8]: https://github.com/pallas/pyllhttp +[9]: https://github.com/metabahn/llhttp diff --git a/src/3rdparty/llhttp/api.c b/src/3rdparty/llhttp/api.c new file mode 100644 index 00000000..0f9b5677 --- /dev/null +++ b/src/3rdparty/llhttp/api.c @@ -0,0 +1,275 @@ +#include +#include +#include + +#include "llhttp.h" + +#define CALLBACK_MAYBE(PARSER, NAME, ...) \ + do { \ + const llhttp_settings_t* settings; \ + settings = (const llhttp_settings_t*) (PARSER)->settings; \ + if (settings == NULL || settings->NAME == NULL) { \ + err = 0; \ + break; \ + } \ + err = settings->NAME(__VA_ARGS__); \ + } while (0) + +void llhttp_init(llhttp_t* parser, llhttp_type_t type, + const llhttp_settings_t* settings) { + llhttp__internal_init(parser); + + parser->type = type; + parser->settings = (void*) settings; +} + + +void llhttp_reset(llhttp_t* parser) { + llhttp_type_t type = parser->type; + const llhttp_settings_t* settings = parser->settings; + void* data = parser->data; + uint8_t lenient_flags = parser->lenient_flags; + + llhttp__internal_init(parser); + + parser->type = type; + parser->settings = (void*) settings; + parser->data = data; + parser->lenient_flags = lenient_flags; +} + + +llhttp_errno_t llhttp_execute(llhttp_t* parser, const char* data, size_t len) { + return llhttp__internal_execute(parser, data, data + len); +} + + +void llhttp_settings_init(llhttp_settings_t* settings) { + memset(settings, 0, sizeof(*settings)); +} + + +llhttp_errno_t llhttp_finish(llhttp_t* parser) { + int err; + + /* We're in an error state. Don't bother doing anything. */ + if (parser->error != 0) { + return 0; + } + + switch (parser->finish) { + case HTTP_FINISH_SAFE_WITH_CB: + CALLBACK_MAYBE(parser, on_message_complete, parser); + if (err != HPE_OK) return err; + + /* FALLTHROUGH */ + case HTTP_FINISH_SAFE: + return HPE_OK; + case HTTP_FINISH_UNSAFE: + parser->reason = "Invalid EOF state"; + return HPE_INVALID_EOF_STATE; + default: + abort(); + } +} + + +void llhttp_pause(llhttp_t* parser) { + if (parser->error != HPE_OK) { + return; + } + + parser->error = HPE_PAUSED; + parser->reason = "Paused"; +} + + +void llhttp_resume(llhttp_t* parser) { + if (parser->error != HPE_PAUSED) { + return; + } + + parser->error = 0; +} + + +void llhttp_resume_after_upgrade(llhttp_t* parser) { + if (parser->error != HPE_PAUSED_UPGRADE) { + return; + } + + parser->error = 0; +} + + +llhttp_errno_t llhttp_get_errno(const llhttp_t* parser) { + return parser->error; +} + + +const char* llhttp_get_error_reason(const llhttp_t* parser) { + return parser->reason; +} + + +void llhttp_set_error_reason(llhttp_t* parser, const char* reason) { + parser->reason = reason; +} + + +const char* llhttp_get_error_pos(const llhttp_t* parser) { + return parser->error_pos; +} + + +const char* llhttp_errno_name(llhttp_errno_t err) { +#define HTTP_ERRNO_GEN(CODE, NAME, _) case HPE_##NAME: return "HPE_" #NAME; + switch (err) { + HTTP_ERRNO_MAP(HTTP_ERRNO_GEN) + default: abort(); + } +#undef HTTP_ERRNO_GEN +} + + +const char* llhttp_method_name(llhttp_method_t method) { +#define HTTP_METHOD_GEN(NUM, NAME, STRING) case HTTP_##NAME: return #STRING; + switch (method) { + HTTP_METHOD_MAP(HTTP_METHOD_GEN) + default: abort(); + } +#undef HTTP_METHOD_GEN +} + + +void llhttp_set_lenient_headers(llhttp_t* parser, int enabled) { + if (enabled) { + parser->lenient_flags |= LENIENT_HEADERS; + } else { + parser->lenient_flags &= ~LENIENT_HEADERS; + } +} + +void llhttp_set_lenient_chunked_length(llhttp_t* parser, int enabled) { + if (enabled) { + parser->lenient_flags |= LENIENT_CHUNKED_LENGTH; + } else { + parser->lenient_flags &= ~LENIENT_CHUNKED_LENGTH; + } +} + + +/* Callbacks */ + + +int llhttp__on_message_begin(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_message_begin, s); + return err; +} + + +int llhttp__on_url(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_url, s, p, endp - p); + return err; +} + + +int llhttp__on_url_complete(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_url_complete, s); + return err; +} + + +int llhttp__on_status(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_status, s, p, endp - p); + return err; +} + + +int llhttp__on_status_complete(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_status_complete, s); + return err; +} + + +int llhttp__on_header_field(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_header_field, s, p, endp - p); + return err; +} + + +int llhttp__on_header_field_complete(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_header_field_complete, s); + return err; +} + + +int llhttp__on_header_value(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_header_value, s, p, endp - p); + return err; +} + + +int llhttp__on_header_value_complete(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_header_value_complete, s); + return err; +} + + +int llhttp__on_headers_complete(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_headers_complete, s); + return err; +} + + +int llhttp__on_message_complete(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_message_complete, s); + return err; +} + + +int llhttp__on_body(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_body, s, p, endp - p); + return err; +} + + +int llhttp__on_chunk_header(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_chunk_header, s); + return err; +} + + +int llhttp__on_chunk_complete(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_chunk_complete, s); + return err; +} + + +/* Private */ + + +void llhttp__debug(llhttp_t* s, const char* p, const char* endp, + const char* msg) { + if (p == endp) { + fprintf(stderr, "p=%p type=%d flags=%02x next=null debug=%s\n", s, s->type, + s->flags, msg); + } else { + fprintf(stderr, "p=%p type=%d flags=%02x next=%02x debug=%s\n", s, + s->type, s->flags, *p, msg); + } +} diff --git a/src/3rdparty/llhttp/api.h b/src/3rdparty/llhttp/api.h new file mode 100644 index 00000000..f62c57fe --- /dev/null +++ b/src/3rdparty/llhttp/api.h @@ -0,0 +1,187 @@ +#ifndef INCLUDE_LLHTTP_API_H_ +#define INCLUDE_LLHTTP_API_H_ +#ifdef __cplusplus +extern "C" { +#endif +#include + +typedef llhttp__internal_t llhttp_t; +typedef struct llhttp_settings_s llhttp_settings_t; + +typedef int (*llhttp_data_cb)(llhttp_t*, const char *at, size_t length); +typedef int (*llhttp_cb)(llhttp_t*); + +struct llhttp_settings_s { + /* Possible return values 0, -1, `HPE_PAUSED` */ + llhttp_cb on_message_begin; + + llhttp_data_cb on_url; + llhttp_data_cb on_status; + llhttp_data_cb on_header_field; + llhttp_data_cb on_header_value; + + /* Possible return values: + * 0 - Proceed normally + * 1 - Assume that request/response has no body, and proceed to parsing the + * next message + * 2 - Assume absence of body (as above) and make `llhttp_execute()` return + * `HPE_PAUSED_UPGRADE` + * -1 - Error + * `HPE_PAUSED` + */ + llhttp_cb on_headers_complete; + + llhttp_data_cb on_body; + + /* Possible return values 0, -1, `HPE_PAUSED` */ + llhttp_cb on_message_complete; + + /* When on_chunk_header is called, the current chunk length is stored + * in parser->content_length. + * Possible return values 0, -1, `HPE_PAUSED` + */ + llhttp_cb on_chunk_header; + llhttp_cb on_chunk_complete; + + llhttp_cb on_url_complete; + llhttp_cb on_status_complete; + llhttp_cb on_header_field_complete; + llhttp_cb on_header_value_complete; +}; + +/* Initialize the parser with specific type and user settings. + * + * NOTE: lifetime of `settings` has to be at least the same as the lifetime of + * the `parser` here. In practice, `settings` has to be either a static + * variable or be allocated with `malloc`, `new`, etc. + */ +void llhttp_init(llhttp_t* parser, llhttp_type_t type, + const llhttp_settings_t* settings); + +/* Reset an already initialized parser back to the start state, preserving the + * existing parser type, callback settings, user data, and lenient flags. + */ +void llhttp_reset(llhttp_t* parser); + +/* Initialize the settings object */ +void llhttp_settings_init(llhttp_settings_t* settings); + +/* Parse full or partial request/response, invoking user callbacks along the + * way. + * + * If any of `llhttp_data_cb` returns errno not equal to `HPE_OK` - the parsing + * interrupts, and such errno is returned from `llhttp_execute()`. If + * `HPE_PAUSED` was used as a errno, the execution can be resumed with + * `llhttp_resume()` call. + * + * In a special case of CONNECT/Upgrade request/response `HPE_PAUSED_UPGRADE` + * is returned after fully parsing the request/response. If the user wishes to + * continue parsing, they need to invoke `llhttp_resume_after_upgrade()`. + * + * NOTE: if this function ever returns a non-pause type error, it will continue + * to return the same error upon each successive call up until `llhttp_init()` + * is called. + */ +llhttp_errno_t llhttp_execute(llhttp_t* parser, const char* data, size_t len); + +/* This method should be called when the other side has no further bytes to + * send (e.g. shutdown of readable side of the TCP connection.) + * + * Requests without `Content-Length` and other messages might require treating + * all incoming bytes as the part of the body, up to the last byte of the + * connection. This method will invoke `on_message_complete()` callback if the + * request was terminated safely. Otherwise a error code would be returned. + */ +llhttp_errno_t llhttp_finish(llhttp_t* parser); + +/* Returns `1` if the incoming message is parsed until the last byte, and has + * to be completed by calling `llhttp_finish()` on EOF + */ +int llhttp_message_needs_eof(const llhttp_t* parser); + +/* Returns `1` if there might be any other messages following the last that was + * successfully parsed. + */ +int llhttp_should_keep_alive(const llhttp_t* parser); + +/* Make further calls of `llhttp_execute()` return `HPE_PAUSED` and set + * appropriate error reason. + * + * Important: do not call this from user callbacks! User callbacks must return + * `HPE_PAUSED` if pausing is required. + */ +void llhttp_pause(llhttp_t* parser); + +/* Might be called to resume the execution after the pause in user's callback. + * See `llhttp_execute()` above for details. + * + * Call this only if `llhttp_execute()` returns `HPE_PAUSED`. + */ +void llhttp_resume(llhttp_t* parser); + +/* Might be called to resume the execution after the pause in user's callback. + * See `llhttp_execute()` above for details. + * + * Call this only if `llhttp_execute()` returns `HPE_PAUSED_UPGRADE` + */ +void llhttp_resume_after_upgrade(llhttp_t* parser); + +/* Returns the latest return error */ +llhttp_errno_t llhttp_get_errno(const llhttp_t* parser); + +/* Returns the verbal explanation of the latest returned error. + * + * Note: User callback should set error reason when returning the error. See + * `llhttp_set_error_reason()` for details. + */ +const char* llhttp_get_error_reason(const llhttp_t* parser); + +/* Assign verbal description to the returned error. Must be called in user + * callbacks right before returning the errno. + * + * Note: `HPE_USER` error code might be useful in user callbacks. + */ +void llhttp_set_error_reason(llhttp_t* parser, const char* reason); + +/* Returns the pointer to the last parsed byte before the returned error. The + * pointer is relative to the `data` argument of `llhttp_execute()`. + * + * Note: this method might be useful for counting the number of parsed bytes. + */ +const char* llhttp_get_error_pos(const llhttp_t* parser); + +/* Returns textual name of error code */ +const char* llhttp_errno_name(llhttp_errno_t err); + +/* Returns textual name of HTTP method */ +const char* llhttp_method_name(llhttp_method_t method); + + +/* Enables/disables lenient header value parsing (disabled by default). + * + * Lenient parsing disables header value token checks, extending llhttp's + * protocol support to highly non-compliant clients/server. No + * `HPE_INVALID_HEADER_TOKEN` will be raised for incorrect header values when + * lenient parsing is "on". + * + * **(USE AT YOUR OWN RISK)** + */ +void llhttp_set_lenient_headers(llhttp_t* parser, int enabled); + + +/* Enables/disables lenient handling of conflicting `Transfer-Encoding` and + * `Content-Length` headers (disabled by default). + * + * Normally `llhttp` would error when `Transfer-Encoding` is present in + * conjunction with `Content-Length`. This error is important to prevent HTTP + * request smuggling, but may be less desirable for small number of cases + * involving legacy servers. + * + * **(USE AT YOUR OWN RISK)** + */ +void llhttp_set_lenient_chunked_length(llhttp_t* parser, int enabled); + +#ifdef __cplusplus +} /* extern "C" */ +#endif +#endif /* INCLUDE_LLHTTP_API_H_ */ diff --git a/src/3rdparty/llhttp/http.c b/src/3rdparty/llhttp/http.c new file mode 100644 index 00000000..e6279f3b --- /dev/null +++ b/src/3rdparty/llhttp/http.c @@ -0,0 +1,149 @@ +#include +#ifndef LLHTTP__TEST +# include "llhttp.h" +#else +# define llhttp_t llparse_t +#endif /* */ + +int llhttp_message_needs_eof(const llhttp_t* parser); +int llhttp_should_keep_alive(const llhttp_t* parser); + +int llhttp__before_headers_complete(llhttp_t* parser, const char* p, + const char* endp) { + /* Set this here so that on_headers_complete() callbacks can see it */ + if ((parser->flags & F_UPGRADE) && + (parser->flags & F_CONNECTION_UPGRADE)) { + /* For responses, "Upgrade: foo" and "Connection: upgrade" are + * mandatory only when it is a 101 Switching Protocols response, + * otherwise it is purely informational, to announce support. + */ + parser->upgrade = + (parser->type == HTTP_REQUEST || parser->status_code == 101); + } else { + parser->upgrade = (parser->method == HTTP_CONNECT); + } + return 0; +} + + +/* Return values: + * 0 - No body, `restart`, message_complete + * 1 - CONNECT request, `restart`, message_complete, and pause + * 2 - chunk_size_start + * 3 - body_identity + * 4 - body_identity_eof + * 5 - invalid transfer-encoding for request + */ +int llhttp__after_headers_complete(llhttp_t* parser, const char* p, + const char* endp) { + int hasBody; + + hasBody = parser->flags & F_CHUNKED || parser->content_length > 0; + if (parser->upgrade && (parser->method == HTTP_CONNECT || + (parser->flags & F_SKIPBODY) || !hasBody)) { + /* Exit, the rest of the message is in a different protocol. */ + return 1; + } + + if (parser->flags & F_SKIPBODY) { + return 0; + } else if (parser->flags & F_CHUNKED) { + /* chunked encoding - ignore Content-Length header, prepare for a chunk */ + return 2; + } else if (parser->flags & F_TRANSFER_ENCODING) { + if (parser->type == HTTP_REQUEST && + (parser->lenient_flags & LENIENT_CHUNKED_LENGTH) == 0) { + /* RFC 7230 3.3.3 */ + + /* If a Transfer-Encoding header field + * is present in a request and the chunked transfer coding is not + * the final encoding, the message body length cannot be determined + * reliably; the server MUST respond with the 400 (Bad Request) + * status code and then close the connection. + */ + return 5; + } else { + /* RFC 7230 3.3.3 */ + + /* If a Transfer-Encoding header field is present in a response and + * the chunked transfer coding is not the final encoding, the + * message body length is determined by reading the connection until + * it is closed by the server. + */ + return 4; + } + } else { + if (!(parser->flags & F_CONTENT_LENGTH)) { + if (!llhttp_message_needs_eof(parser)) { + /* Assume content-length 0 - read the next */ + return 0; + } else { + /* Read body until EOF */ + return 4; + } + } else if (parser->content_length == 0) { + /* Content-Length header given but zero: Content-Length: 0\r\n */ + return 0; + } else { + /* Content-Length header given and non-zero */ + return 3; + } + } +} + + +int llhttp__after_message_complete(llhttp_t* parser, const char* p, + const char* endp) { + int should_keep_alive; + + should_keep_alive = llhttp_should_keep_alive(parser); + parser->finish = HTTP_FINISH_SAFE; + parser->flags = 0; + + /* NOTE: this is ignored in loose parsing mode */ + return should_keep_alive; +} + + +int llhttp_message_needs_eof(const llhttp_t* parser) { + if (parser->type == HTTP_REQUEST) { + return 0; + } + + /* See RFC 2616 section 4.4 */ + if (parser->status_code / 100 == 1 || /* 1xx e.g. Continue */ + parser->status_code == 204 || /* No Content */ + parser->status_code == 304 || /* Not Modified */ + (parser->flags & F_SKIPBODY)) { /* response to a HEAD request */ + return 0; + } + + /* RFC 7230 3.3.3, see `llhttp__after_headers_complete` */ + if ((parser->flags & F_TRANSFER_ENCODING) && + (parser->flags & F_CHUNKED) == 0) { + return 1; + } + + if (parser->flags & (F_CHUNKED | F_CONTENT_LENGTH)) { + return 0; + } + + return 1; +} + + +int llhttp_should_keep_alive(const llhttp_t* parser) { + if (parser->http_major > 0 && parser->http_minor > 0) { + /* HTTP/1.1 */ + if (parser->flags & F_CONNECTION_CLOSE) { + return 0; + } + } else { + /* HTTP/1.0 or earlier */ + if (!(parser->flags & F_CONNECTION_KEEP_ALIVE)) { + return 0; + } + } + + return !llhttp_message_needs_eof(parser); +} diff --git a/src/3rdparty/llhttp/llhttp.c b/src/3rdparty/llhttp/llhttp.c new file mode 100644 index 00000000..33ddcb29 --- /dev/null +++ b/src/3rdparty/llhttp/llhttp.c @@ -0,0 +1,14871 @@ +#if LLHTTP_STRICT_MODE + +#include +#include +#include + +#ifdef __SSE4_2__ + #ifdef _MSC_VER + #include + #else /* !_MSC_VER */ + #include + #endif /* _MSC_VER */ +#endif /* __SSE4_2__ */ + +#ifdef _MSC_VER + #define ALIGN(n) _declspec(align(n)) +#else /* !_MSC_VER */ + #define ALIGN(n) __attribute__((aligned(n))) +#endif /* _MSC_VER */ + +#include "llhttp.h" + +typedef int (*llhttp__internal__span_cb)( + llhttp__internal_t*, const char*, const char*); + +static const unsigned char llparse_blob0[] = { + 0xd, 0xa +}; +static const unsigned char llparse_blob1[] = { + 'o', 'n' +}; +static const unsigned char llparse_blob2[] = { + 'e', 'c', 't', 'i', 'o', 'n' +}; +static const unsigned char llparse_blob3[] = { + 'l', 'o', 's', 'e' +}; +static const unsigned char llparse_blob4[] = { + 'e', 'e', 'p', '-', 'a', 'l', 'i', 'v', 'e' +}; +static const unsigned char llparse_blob5[] = { + 'p', 'g', 'r', 'a', 'd', 'e' +}; +static const unsigned char llparse_blob6[] = { + 'c', 'h', 'u', 'n', 'k', 'e', 'd' +}; +#ifdef __SSE4_2__ +static const unsigned char ALIGN(16) llparse_blob7[] = { + 0x9, 0x9, ' ', '~', 0x80, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0 +}; +#endif /* __SSE4_2__ */ +#ifdef __SSE4_2__ +static const unsigned char ALIGN(16) llparse_blob8[] = { + '!', '!', '#', '\'', '*', '+', '-', '.', '0', '9', 'A', + 'Z', '^', 'z', '|', '|' +}; +#endif /* __SSE4_2__ */ +#ifdef __SSE4_2__ +static const unsigned char ALIGN(16) llparse_blob9[] = { + '~', '~', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0 +}; +#endif /* __SSE4_2__ */ +static const unsigned char llparse_blob10[] = { + 'e', 'n', 't', '-', 'l', 'e', 'n', 'g', 't', 'h' +}; +static const unsigned char llparse_blob11[] = { + 'r', 'o', 'x', 'y', '-', 'c', 'o', 'n', 'n', 'e', 'c', + 't', 'i', 'o', 'n' +}; +static const unsigned char llparse_blob12[] = { + 'r', 'a', 'n', 's', 'f', 'e', 'r', '-', 'e', 'n', 'c', + 'o', 'd', 'i', 'n', 'g' +}; +static const unsigned char llparse_blob13[] = { + 'p', 'g', 'r', 'a', 'd', 'e' +}; +static const unsigned char llparse_blob14[] = { + 'T', 'T', 'P', '/' +}; +static const unsigned char llparse_blob15[] = { + 0xd, 0xa, 0xd, 0xa, 'S', 'M', 0xd, 0xa, 0xd, 0xa +}; +static const unsigned char llparse_blob16[] = { + 'C', 'E', '/' +}; +static const unsigned char llparse_blob17[] = { + 'T', 'S', 'P', '/' +}; +static const unsigned char llparse_blob18[] = { + 'N', 'O', 'U', 'N', 'C', 'E' +}; +static const unsigned char llparse_blob19[] = { + 'I', 'N', 'D' +}; +static const unsigned char llparse_blob20[] = { + 'E', 'C', 'K', 'O', 'U', 'T' +}; +static const unsigned char llparse_blob21[] = { + 'N', 'E', 'C', 'T' +}; +static const unsigned char llparse_blob22[] = { + 'E', 'T', 'E' +}; +static const unsigned char llparse_blob23[] = { + 'C', 'R', 'I', 'B', 'E' +}; +static const unsigned char llparse_blob24[] = { + 'L', 'U', 'S', 'H' +}; +static const unsigned char llparse_blob25[] = { + 'E', 'T' +}; +static const unsigned char llparse_blob26[] = { + 'P', 'A', 'R', 'A', 'M', 'E', 'T', 'E', 'R' +}; +static const unsigned char llparse_blob27[] = { + 'E', 'A', 'D' +}; +static const unsigned char llparse_blob28[] = { + 'N', 'K' +}; +static const unsigned char llparse_blob29[] = { + 'C', 'K' +}; +static const unsigned char llparse_blob30[] = { + 'S', 'E', 'A', 'R', 'C', 'H' +}; +static const unsigned char llparse_blob31[] = { + 'R', 'G', 'E' +}; +static const unsigned char llparse_blob32[] = { + 'C', 'T', 'I', 'V', 'I', 'T', 'Y' +}; +static const unsigned char llparse_blob33[] = { + 'L', 'E', 'N', 'D', 'A', 'R' +}; +static const unsigned char llparse_blob34[] = { + 'V', 'E' +}; +static const unsigned char llparse_blob35[] = { + 'O', 'T', 'I', 'F', 'Y' +}; +static const unsigned char llparse_blob36[] = { + 'P', 'T', 'I', 'O', 'N', 'S' +}; +static const unsigned char llparse_blob37[] = { + 'C', 'H' +}; +static const unsigned char llparse_blob38[] = { + 'S', 'E' +}; +static const unsigned char llparse_blob39[] = { + 'A', 'Y' +}; +static const unsigned char llparse_blob40[] = { + 'S', 'T' +}; +static const unsigned char llparse_blob41[] = { + 'I', 'N', 'D' +}; +static const unsigned char llparse_blob42[] = { + 'A', 'T', 'C', 'H' +}; +static const unsigned char llparse_blob43[] = { + 'G', 'E' +}; +static const unsigned char llparse_blob44[] = { + 'I', 'N', 'D' +}; +static const unsigned char llparse_blob45[] = { + 'O', 'R', 'D' +}; +static const unsigned char llparse_blob46[] = { + 'I', 'R', 'E', 'C', 'T' +}; +static const unsigned char llparse_blob47[] = { + 'O', 'R', 'T' +}; +static const unsigned char llparse_blob48[] = { + 'R', 'C', 'H' +}; +static const unsigned char llparse_blob49[] = { + 'P', 'A', 'R', 'A', 'M', 'E', 'T', 'E', 'R' +}; +static const unsigned char llparse_blob50[] = { + 'U', 'R', 'C', 'E' +}; +static const unsigned char llparse_blob51[] = { + 'B', 'S', 'C', 'R', 'I', 'B', 'E' +}; +static const unsigned char llparse_blob52[] = { + 'A', 'R', 'D', 'O', 'W', 'N' +}; +static const unsigned char llparse_blob53[] = { + 'A', 'C', 'E' +}; +static const unsigned char llparse_blob54[] = { + 'I', 'N', 'D' +}; +static const unsigned char llparse_blob55[] = { + 'N', 'K' +}; +static const unsigned char llparse_blob56[] = { + 'C', 'K' +}; +static const unsigned char llparse_blob57[] = { + 'U', 'B', 'S', 'C', 'R', 'I', 'B', 'E' +}; +static const unsigned char llparse_blob58[] = { + 'H', 'T', 'T', 'P', '/' +}; +static const unsigned char llparse_blob59[] = { + 'A', 'D' +}; +static const unsigned char llparse_blob60[] = { + 'T', 'P', '/' +}; + +enum llparse_match_status_e { + kMatchComplete, + kMatchPause, + kMatchMismatch +}; +typedef enum llparse_match_status_e llparse_match_status_t; + +struct llparse_match_s { + llparse_match_status_t status; + const unsigned char* current; +}; +typedef struct llparse_match_s llparse_match_t; + +static llparse_match_t llparse__match_sequence_id( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp, + const unsigned char* seq, uint32_t seq_len) { + uint32_t index; + llparse_match_t res; + + index = s->_index; + for (; p != endp; p++) { + unsigned char current; + + current = *p; + if (current == seq[index]) { + if (++index == seq_len) { + res.status = kMatchComplete; + goto reset; + } + } else { + res.status = kMatchMismatch; + goto reset; + } + } + s->_index = index; + res.status = kMatchPause; + res.current = p; + return res; +reset: + s->_index = 0; + res.current = p; + return res; +} + +static llparse_match_t llparse__match_sequence_to_lower( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp, + const unsigned char* seq, uint32_t seq_len) { + uint32_t index; + llparse_match_t res; + + index = s->_index; + for (; p != endp; p++) { + unsigned char current; + + current = ((*p) >= 'A' && (*p) <= 'Z' ? (*p | 0x20) : (*p)); + if (current == seq[index]) { + if (++index == seq_len) { + res.status = kMatchComplete; + goto reset; + } + } else { + res.status = kMatchMismatch; + goto reset; + } + } + s->_index = index; + res.status = kMatchPause; + res.current = p; + return res; +reset: + s->_index = 0; + res.current = p; + return res; +} + +static llparse_match_t llparse__match_sequence_to_lower_unsafe( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp, + const unsigned char* seq, uint32_t seq_len) { + uint32_t index; + llparse_match_t res; + + index = s->_index; + for (; p != endp; p++) { + unsigned char current; + + current = ((*p) | 0x20); + if (current == seq[index]) { + if (++index == seq_len) { + res.status = kMatchComplete; + goto reset; + } + } else { + res.status = kMatchMismatch; + goto reset; + } + } + s->_index = index; + res.status = kMatchPause; + res.current = p; + return res; +reset: + s->_index = 0; + res.current = p; + return res; +} + +enum llparse_state_e { + s_error, + s_n_llhttp__internal__n_closed, + s_n_llhttp__internal__n_invoke_llhttp__after_message_complete, + s_n_llhttp__internal__n_pause_1, + s_n_llhttp__internal__n_invoke_is_equal_upgrade, + s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2, + s_n_llhttp__internal__n_chunk_data_almost_done, + s_n_llhttp__internal__n_consume_content_length, + s_n_llhttp__internal__n_span_start_llhttp__on_body, + s_n_llhttp__internal__n_invoke_is_equal_content_length, + s_n_llhttp__internal__n_chunk_size_almost_done, + s_n_llhttp__internal__n_chunk_parameters, + s_n_llhttp__internal__n_chunk_size_otherwise, + s_n_llhttp__internal__n_chunk_size, + s_n_llhttp__internal__n_chunk_size_digit, + s_n_llhttp__internal__n_invoke_update_content_length, + s_n_llhttp__internal__n_consume_content_length_1, + s_n_llhttp__internal__n_span_start_llhttp__on_body_1, + s_n_llhttp__internal__n_eof, + s_n_llhttp__internal__n_span_start_llhttp__on_body_2, + s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete, + s_n_llhttp__internal__n_headers_almost_done, + s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete, + s_n_llhttp__internal__n_span_start_llhttp__on_header_value, + s_n_llhttp__internal__n_header_value_discard_lws, + s_n_llhttp__internal__n_header_value_discard_ws_almost_done, + s_n_llhttp__internal__n_header_value_lws, + s_n_llhttp__internal__n_header_value_almost_done, + s_n_llhttp__internal__n_header_value_lenient, + s_n_llhttp__internal__n_header_value_otherwise, + s_n_llhttp__internal__n_header_value_connection_token, + s_n_llhttp__internal__n_header_value_connection_ws, + s_n_llhttp__internal__n_header_value_connection_1, + s_n_llhttp__internal__n_header_value_connection_2, + s_n_llhttp__internal__n_header_value_connection_3, + s_n_llhttp__internal__n_header_value_connection, + s_n_llhttp__internal__n_error_23, + s_n_llhttp__internal__n_error_24, + s_n_llhttp__internal__n_header_value_content_length_ws, + s_n_llhttp__internal__n_header_value_content_length, + s_n_llhttp__internal__n_header_value_te_chunked_last, + s_n_llhttp__internal__n_header_value_te_token_ows, + s_n_llhttp__internal__n_header_value, + s_n_llhttp__internal__n_header_value_te_token, + s_n_llhttp__internal__n_header_value_te_chunked, + s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1, + s_n_llhttp__internal__n_header_value_discard_ws, + s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete, + s_n_llhttp__internal__n_header_field_general_otherwise, + s_n_llhttp__internal__n_header_field_general, + s_n_llhttp__internal__n_header_field_colon, + s_n_llhttp__internal__n_header_field_3, + s_n_llhttp__internal__n_header_field_4, + s_n_llhttp__internal__n_header_field_2, + s_n_llhttp__internal__n_header_field_1, + s_n_llhttp__internal__n_header_field_5, + s_n_llhttp__internal__n_header_field_6, + s_n_llhttp__internal__n_header_field_7, + s_n_llhttp__internal__n_header_field, + s_n_llhttp__internal__n_span_start_llhttp__on_header_field, + s_n_llhttp__internal__n_header_field_start, + s_n_llhttp__internal__n_url_to_http_09, + s_n_llhttp__internal__n_url_skip_to_http09, + s_n_llhttp__internal__n_url_skip_lf_to_http09_1, + s_n_llhttp__internal__n_url_skip_lf_to_http09, + s_n_llhttp__internal__n_req_pri_upgrade, + s_n_llhttp__internal__n_req_http_complete_1, + s_n_llhttp__internal__n_req_http_complete, + s_n_llhttp__internal__n_req_http_minor, + s_n_llhttp__internal__n_req_http_dot, + s_n_llhttp__internal__n_req_http_major, + s_n_llhttp__internal__n_req_http_start_1, + s_n_llhttp__internal__n_req_http_start_2, + s_n_llhttp__internal__n_req_http_start_3, + s_n_llhttp__internal__n_req_http_start, + s_n_llhttp__internal__n_url_to_http, + s_n_llhttp__internal__n_url_skip_to_http, + s_n_llhttp__internal__n_url_fragment, + s_n_llhttp__internal__n_span_end_stub_query_3, + s_n_llhttp__internal__n_url_query, + s_n_llhttp__internal__n_url_query_or_fragment, + s_n_llhttp__internal__n_url_path, + s_n_llhttp__internal__n_span_start_stub_path_2, + s_n_llhttp__internal__n_span_start_stub_path, + s_n_llhttp__internal__n_span_start_stub_path_1, + s_n_llhttp__internal__n_url_server_with_at, + s_n_llhttp__internal__n_url_server, + s_n_llhttp__internal__n_url_schema_delim_1, + s_n_llhttp__internal__n_url_schema_delim, + s_n_llhttp__internal__n_span_end_stub_schema, + s_n_llhttp__internal__n_url_schema, + s_n_llhttp__internal__n_url_start, + s_n_llhttp__internal__n_span_start_llhttp__on_url_1, + s_n_llhttp__internal__n_url_entry_normal, + s_n_llhttp__internal__n_span_start_llhttp__on_url, + s_n_llhttp__internal__n_url_entry_connect, + s_n_llhttp__internal__n_req_spaces_before_url, + s_n_llhttp__internal__n_req_first_space_before_url, + s_n_llhttp__internal__n_start_req_2, + s_n_llhttp__internal__n_start_req_3, + s_n_llhttp__internal__n_start_req_1, + s_n_llhttp__internal__n_start_req_4, + s_n_llhttp__internal__n_start_req_6, + s_n_llhttp__internal__n_start_req_8, + s_n_llhttp__internal__n_start_req_9, + s_n_llhttp__internal__n_start_req_7, + s_n_llhttp__internal__n_start_req_5, + s_n_llhttp__internal__n_start_req_12, + s_n_llhttp__internal__n_start_req_13, + s_n_llhttp__internal__n_start_req_11, + s_n_llhttp__internal__n_start_req_10, + s_n_llhttp__internal__n_start_req_14, + s_n_llhttp__internal__n_start_req_17, + s_n_llhttp__internal__n_start_req_16, + s_n_llhttp__internal__n_start_req_15, + s_n_llhttp__internal__n_start_req_18, + s_n_llhttp__internal__n_start_req_20, + s_n_llhttp__internal__n_start_req_21, + s_n_llhttp__internal__n_start_req_19, + s_n_llhttp__internal__n_start_req_23, + s_n_llhttp__internal__n_start_req_24, + s_n_llhttp__internal__n_start_req_26, + s_n_llhttp__internal__n_start_req_28, + s_n_llhttp__internal__n_start_req_29, + s_n_llhttp__internal__n_start_req_27, + s_n_llhttp__internal__n_start_req_25, + s_n_llhttp__internal__n_start_req_30, + s_n_llhttp__internal__n_start_req_22, + s_n_llhttp__internal__n_start_req_31, + s_n_llhttp__internal__n_start_req_32, + s_n_llhttp__internal__n_start_req_35, + s_n_llhttp__internal__n_start_req_36, + s_n_llhttp__internal__n_start_req_34, + s_n_llhttp__internal__n_start_req_37, + s_n_llhttp__internal__n_start_req_38, + s_n_llhttp__internal__n_start_req_42, + s_n_llhttp__internal__n_start_req_43, + s_n_llhttp__internal__n_start_req_41, + s_n_llhttp__internal__n_start_req_40, + s_n_llhttp__internal__n_start_req_39, + s_n_llhttp__internal__n_start_req_45, + s_n_llhttp__internal__n_start_req_44, + s_n_llhttp__internal__n_start_req_33, + s_n_llhttp__internal__n_start_req_48, + s_n_llhttp__internal__n_start_req_49, + s_n_llhttp__internal__n_start_req_50, + s_n_llhttp__internal__n_start_req_51, + s_n_llhttp__internal__n_start_req_47, + s_n_llhttp__internal__n_start_req_46, + s_n_llhttp__internal__n_start_req_54, + s_n_llhttp__internal__n_start_req_56, + s_n_llhttp__internal__n_start_req_57, + s_n_llhttp__internal__n_start_req_55, + s_n_llhttp__internal__n_start_req_53, + s_n_llhttp__internal__n_start_req_58, + s_n_llhttp__internal__n_start_req_59, + s_n_llhttp__internal__n_start_req_52, + s_n_llhttp__internal__n_start_req_61, + s_n_llhttp__internal__n_start_req_62, + s_n_llhttp__internal__n_start_req_60, + s_n_llhttp__internal__n_start_req_65, + s_n_llhttp__internal__n_start_req_67, + s_n_llhttp__internal__n_start_req_68, + s_n_llhttp__internal__n_start_req_66, + s_n_llhttp__internal__n_start_req_69, + s_n_llhttp__internal__n_start_req_64, + s_n_llhttp__internal__n_start_req_63, + s_n_llhttp__internal__n_start_req, + s_n_llhttp__internal__n_invoke_llhttp__on_status_complete, + s_n_llhttp__internal__n_res_line_almost_done, + s_n_llhttp__internal__n_res_status, + s_n_llhttp__internal__n_span_start_llhttp__on_status, + s_n_llhttp__internal__n_res_status_start, + s_n_llhttp__internal__n_res_status_code_otherwise, + s_n_llhttp__internal__n_res_status_code, + s_n_llhttp__internal__n_res_http_end, + s_n_llhttp__internal__n_res_http_minor, + s_n_llhttp__internal__n_res_http_dot, + s_n_llhttp__internal__n_res_http_major, + s_n_llhttp__internal__n_start_res, + s_n_llhttp__internal__n_req_or_res_method_2, + s_n_llhttp__internal__n_req_or_res_method_3, + s_n_llhttp__internal__n_req_or_res_method_1, + s_n_llhttp__internal__n_req_or_res_method, + s_n_llhttp__internal__n_start_req_or_res, + s_n_llhttp__internal__n_invoke_load_type, + s_n_llhttp__internal__n_start, +}; +typedef enum llparse_state_e llparse_state_t; + +int llhttp__on_url( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__on_header_field( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__on_header_value( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__on_body( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__on_status( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_update_finish( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->finish = 2; + return 0; +} + +int llhttp__on_message_begin( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_load_type( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return state->type; +} + +int llhttp__internal__c_store_method( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp, + int match) { + state->method = match; + return 0; +} + +int llhttp__internal__c_is_equal_method( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return state->method == 5; +} + +int llhttp__internal__c_update_http_major( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->http_major = 0; + return 0; +} + +int llhttp__internal__c_update_http_minor( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->http_minor = 9; + return 0; +} + +int llhttp__on_url_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_test_flags( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->flags & 128) == 128; +} + +int llhttp__on_chunk_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__on_message_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_is_equal_upgrade( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return state->upgrade == 1; +} + +int llhttp__after_message_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_update_finish_1( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->finish = 0; + return 0; +} + +int llhttp__internal__c_test_flags_1( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->flags & 544) == 544; +} + +int llhttp__internal__c_test_lenient_flags( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->lenient_flags & 2) == 2; +} + +int llhttp__before_headers_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__on_headers_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__after_headers_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_update_content_length( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->content_length = 0; + return 0; +} + +int llhttp__internal__c_mul_add_content_length( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp, + int match) { + /* Multiplication overflow */ + if (state->content_length > 0xffffffffffffffffULL / 16) { + return 1; + } + + state->content_length *= 16; + + /* Addition overflow */ + if (match >= 0) { + if (state->content_length > 0xffffffffffffffffULL - match) { + return 1; + } + } else { + if (state->content_length < 0ULL - match) { + return 1; + } + } + state->content_length += match; + return 0; +} + +int llhttp__on_chunk_header( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_is_equal_content_length( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return state->content_length == 0; +} + +int llhttp__internal__c_or_flags( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags |= 128; + return 0; +} + +int llhttp__internal__c_update_finish_3( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->finish = 1; + return 0; +} + +int llhttp__internal__c_or_flags_1( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags |= 64; + return 0; +} + +int llhttp__internal__c_update_upgrade( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->upgrade = 1; + return 0; +} + +int llhttp__internal__c_store_header_state( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp, + int match) { + state->header_state = match; + return 0; +} + +int llhttp__on_header_field_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_load_header_state( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return state->header_state; +} + +int llhttp__internal__c_or_flags_3( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags |= 1; + return 0; +} + +int llhttp__internal__c_update_header_state( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->header_state = 1; + return 0; +} + +int llhttp__on_header_value_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_or_flags_4( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags |= 2; + return 0; +} + +int llhttp__internal__c_or_flags_5( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags |= 4; + return 0; +} + +int llhttp__internal__c_or_flags_6( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags |= 8; + return 0; +} + +int llhttp__internal__c_update_header_state_2( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->header_state = 6; + return 0; +} + +int llhttp__internal__c_test_lenient_flags_1( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->lenient_flags & 1) == 1; +} + +int llhttp__internal__c_update_header_state_4( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->header_state = 0; + return 0; +} + +int llhttp__internal__c_update_header_state_5( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->header_state = 5; + return 0; +} + +int llhttp__internal__c_update_header_state_6( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->header_state = 7; + return 0; +} + +int llhttp__internal__c_test_flags_2( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->flags & 32) == 32; +} + +int llhttp__internal__c_mul_add_content_length_1( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp, + int match) { + /* Multiplication overflow */ + if (state->content_length > 0xffffffffffffffffULL / 10) { + return 1; + } + + state->content_length *= 10; + + /* Addition overflow */ + if (match >= 0) { + if (state->content_length > 0xffffffffffffffffULL - match) { + return 1; + } + } else { + if (state->content_length < 0ULL - match) { + return 1; + } + } + state->content_length += match; + return 0; +} + +int llhttp__internal__c_or_flags_15( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags |= 32; + return 0; +} + +int llhttp__internal__c_or_flags_16( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags |= 512; + return 0; +} + +int llhttp__internal__c_and_flags( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags &= -9; + return 0; +} + +int llhttp__internal__c_update_header_state_7( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->header_state = 8; + return 0; +} + +int llhttp__internal__c_or_flags_17( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags |= 16; + return 0; +} + +int llhttp__internal__c_load_method( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return state->method; +} + +int llhttp__internal__c_store_http_major( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp, + int match) { + state->http_major = match; + return 0; +} + +int llhttp__internal__c_store_http_minor( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp, + int match) { + state->http_minor = match; + return 0; +} + +int llhttp__internal__c_update_status_code( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->status_code = 0; + return 0; +} + +int llhttp__internal__c_mul_add_status_code( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp, + int match) { + /* Multiplication overflow */ + if (state->status_code > 0xffff / 10) { + return 1; + } + + state->status_code *= 10; + + /* Addition overflow */ + if (match >= 0) { + if (state->status_code > 0xffff - match) { + return 1; + } + } else { + if (state->status_code < 0 - match) { + return 1; + } + } + state->status_code += match; + + /* Enforce maximum */ + if (state->status_code > 999) { + return 1; + } + return 0; +} + +int llhttp__on_status_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_update_type( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->type = 1; + return 0; +} + +int llhttp__internal__c_update_type_1( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->type = 2; + return 0; +} + +int llhttp__internal_init(llhttp__internal_t* state) { + memset(state, 0, sizeof(*state)); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_start; + return 0; +} + +static llparse_state_t llhttp__internal__run( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + int match; + switch ((llparse_state_t) (intptr_t) state->_current) { + case s_n_llhttp__internal__n_closed: + s_n_llhttp__internal__n_closed: { + if (p == endp) { + return s_n_llhttp__internal__n_closed; + } + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_closed; + } + case 13: { + p++; + goto s_n_llhttp__internal__n_closed; + } + default: { + p++; + goto s_n_llhttp__internal__n_error_4; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_llhttp__after_message_complete: + s_n_llhttp__internal__n_invoke_llhttp__after_message_complete: { + switch (llhttp__after_message_complete(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_update_finish_2; + default: + goto s_n_llhttp__internal__n_invoke_update_finish_1; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_pause_1: + s_n_llhttp__internal__n_pause_1: { + state->error = 0x16; + state->reason = "Pause on CONNECT/Upgrade"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__after_message_complete; + return s_error; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_is_equal_upgrade: + s_n_llhttp__internal__n_invoke_is_equal_upgrade: { + switch (llhttp__internal__c_is_equal_upgrade(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_llhttp__after_message_complete; + default: + goto s_n_llhttp__internal__n_pause_1; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2: + s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2: { + switch (llhttp__on_message_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_is_equal_upgrade; + case 21: + goto s_n_llhttp__internal__n_pause_5; + default: + goto s_n_llhttp__internal__n_error_13; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_chunk_data_almost_done: + s_n_llhttp__internal__n_chunk_data_almost_done: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_chunk_data_almost_done; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob0, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete; + } + case kMatchPause: { + return s_n_llhttp__internal__n_chunk_data_almost_done; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_8; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_consume_content_length: + s_n_llhttp__internal__n_consume_content_length: { + size_t avail; + size_t need; + + avail = endp - p; + need = state->content_length; + if (avail >= need) { + p += need; + state->content_length = 0; + goto s_n_llhttp__internal__n_span_end_llhttp__on_body; + } + + state->content_length -= avail; + return s_n_llhttp__internal__n_consume_content_length; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_llhttp__on_body: + s_n_llhttp__internal__n_span_start_llhttp__on_body: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_body; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_body; + goto s_n_llhttp__internal__n_consume_content_length; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_is_equal_content_length: + s_n_llhttp__internal__n_invoke_is_equal_content_length: { + switch (llhttp__internal__c_is_equal_content_length(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_span_start_llhttp__on_body; + default: + goto s_n_llhttp__internal__n_invoke_or_flags; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_chunk_size_almost_done: + s_n_llhttp__internal__n_chunk_size_almost_done: { + if (p == endp) { + return s_n_llhttp__internal__n_chunk_size_almost_done; + } + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_header; + } + default: { + goto s_n_llhttp__internal__n_error_9; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_chunk_parameters: + s_n_llhttp__internal__n_chunk_parameters: { + if (p == endp) { + return s_n_llhttp__internal__n_chunk_parameters; + } + switch (*p) { + case 13: { + p++; + goto s_n_llhttp__internal__n_chunk_size_almost_done; + } + default: { + p++; + goto s_n_llhttp__internal__n_chunk_parameters; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_chunk_size_otherwise: + s_n_llhttp__internal__n_chunk_size_otherwise: { + if (p == endp) { + return s_n_llhttp__internal__n_chunk_size_otherwise; + } + switch (*p) { + case 13: { + p++; + goto s_n_llhttp__internal__n_chunk_size_almost_done; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_chunk_parameters; + } + case ';': { + p++; + goto s_n_llhttp__internal__n_chunk_parameters; + } + default: { + goto s_n_llhttp__internal__n_error_10; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_chunk_size: + s_n_llhttp__internal__n_chunk_size: { + if (p == endp) { + return s_n_llhttp__internal__n_chunk_size; + } + switch (*p) { + case '0': { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '1': { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '2': { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '3': { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '4': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '5': { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '6': { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '7': { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '8': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '9': { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'A': { + p++; + match = 10; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'B': { + p++; + match = 11; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'C': { + p++; + match = 12; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'D': { + p++; + match = 13; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'E': { + p++; + match = 14; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'F': { + p++; + match = 15; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'a': { + p++; + match = 10; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'b': { + p++; + match = 11; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'c': { + p++; + match = 12; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'd': { + p++; + match = 13; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'e': { + p++; + match = 14; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'f': { + p++; + match = 15; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + default: { + goto s_n_llhttp__internal__n_chunk_size_otherwise; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_chunk_size_digit: + s_n_llhttp__internal__n_chunk_size_digit: { + if (p == endp) { + return s_n_llhttp__internal__n_chunk_size_digit; + } + switch (*p) { + case '0': { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '1': { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '2': { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '3': { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '4': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '5': { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '6': { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '7': { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '8': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '9': { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'A': { + p++; + match = 10; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'B': { + p++; + match = 11; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'C': { + p++; + match = 12; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'D': { + p++; + match = 13; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'E': { + p++; + match = 14; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'F': { + p++; + match = 15; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'a': { + p++; + match = 10; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'b': { + p++; + match = 11; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'c': { + p++; + match = 12; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'd': { + p++; + match = 13; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'e': { + p++; + match = 14; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'f': { + p++; + match = 15; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + default: { + goto s_n_llhttp__internal__n_error_12; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_update_content_length: + s_n_llhttp__internal__n_invoke_update_content_length: { + switch (llhttp__internal__c_update_content_length(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_chunk_size_digit; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_consume_content_length_1: + s_n_llhttp__internal__n_consume_content_length_1: { + size_t avail; + size_t need; + + avail = endp - p; + need = state->content_length; + if (avail >= need) { + p += need; + state->content_length = 0; + goto s_n_llhttp__internal__n_span_end_llhttp__on_body_1; + } + + state->content_length -= avail; + return s_n_llhttp__internal__n_consume_content_length_1; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_llhttp__on_body_1: + s_n_llhttp__internal__n_span_start_llhttp__on_body_1: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_body_1; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_body; + goto s_n_llhttp__internal__n_consume_content_length_1; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_eof: + s_n_llhttp__internal__n_eof: { + if (p == endp) { + return s_n_llhttp__internal__n_eof; + } + p++; + goto s_n_llhttp__internal__n_eof; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_llhttp__on_body_2: + s_n_llhttp__internal__n_span_start_llhttp__on_body_2: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_body_2; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_body; + goto s_n_llhttp__internal__n_eof; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete: + s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete: { + switch (llhttp__after_headers_complete(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_1; + case 2: + goto s_n_llhttp__internal__n_invoke_update_content_length; + case 3: + goto s_n_llhttp__internal__n_span_start_llhttp__on_body_1; + case 4: + goto s_n_llhttp__internal__n_invoke_update_finish_3; + case 5: + goto s_n_llhttp__internal__n_error_14; + default: + goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_headers_almost_done: + s_n_llhttp__internal__n_headers_almost_done: { + if (p == endp) { + return s_n_llhttp__internal__n_headers_almost_done; + } + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_invoke_test_flags; + } + default: { + goto s_n_llhttp__internal__n_error_17; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete: + s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete: { + switch (llhttp__on_header_value_complete(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_field_start; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_llhttp__on_header_value: + s_n_llhttp__internal__n_span_start_llhttp__on_header_value: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_header_value; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_header_value; + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_discard_lws: + s_n_llhttp__internal__n_header_value_discard_lws: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_discard_lws; + } + switch (*p) { + case 9: { + p++; + goto s_n_llhttp__internal__n_header_value_discard_ws; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_header_value_discard_ws; + } + default: { + goto s_n_llhttp__internal__n_invoke_load_header_state; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_discard_ws_almost_done: + s_n_llhttp__internal__n_header_value_discard_ws_almost_done: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_discard_ws_almost_done; + } + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_header_value_discard_lws; + } + default: { + goto s_n_llhttp__internal__n_error_19; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_lws: + s_n_llhttp__internal__n_header_value_lws: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_lws; + } + switch (*p) { + case 9: { + goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1; + } + case ' ': { + goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1; + } + default: { + goto s_n_llhttp__internal__n_invoke_load_header_state_3; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_almost_done: + s_n_llhttp__internal__n_header_value_almost_done: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_almost_done; + } + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_header_value_lws; + } + default: { + goto s_n_llhttp__internal__n_error_20; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_lenient: + s_n_llhttp__internal__n_header_value_lenient: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_lenient; + } + switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_1; + } + case 13: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_3; + } + default: { + p++; + goto s_n_llhttp__internal__n_header_value_lenient; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_otherwise: + s_n_llhttp__internal__n_header_value_otherwise: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_otherwise; + } + switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_1; + } + case 13: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_2; + } + default: { + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_1; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_connection_token: + s_n_llhttp__internal__n_header_value_connection_token: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + if (p == endp) { + return s_n_llhttp__internal__n_header_value_connection_token; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_header_value_connection_token; + } + case 2: { + p++; + goto s_n_llhttp__internal__n_header_value_connection; + } + default: { + goto s_n_llhttp__internal__n_header_value_otherwise; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_connection_ws: + s_n_llhttp__internal__n_header_value_connection_ws: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_connection_ws; + } + switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_header_value_otherwise; + } + case 13: { + goto s_n_llhttp__internal__n_header_value_otherwise; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_header_value_connection_ws; + } + case ',': { + p++; + goto s_n_llhttp__internal__n_invoke_load_header_state_4; + } + default: { + goto s_n_llhttp__internal__n_invoke_update_header_state_4; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_connection_1: + s_n_llhttp__internal__n_header_value_connection_1: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_value_connection_1; + } + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob3, 4); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_invoke_update_header_state_2; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_value_connection_1; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_header_value_connection_token; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_connection_2: + s_n_llhttp__internal__n_header_value_connection_2: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_value_connection_2; + } + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob4, 9); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_invoke_update_header_state_5; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_value_connection_2; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_header_value_connection_token; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_connection_3: + s_n_llhttp__internal__n_header_value_connection_3: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_value_connection_3; + } + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob5, 6); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_invoke_update_header_state_6; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_value_connection_3; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_header_value_connection_token; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_connection: + s_n_llhttp__internal__n_header_value_connection: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_connection; + } + switch (((*p) >= 'A' && (*p) <= 'Z' ? (*p | 0x20) : (*p))) { + case 9: { + p++; + goto s_n_llhttp__internal__n_header_value_connection; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_header_value_connection; + } + case 'c': { + p++; + goto s_n_llhttp__internal__n_header_value_connection_1; + } + case 'k': { + p++; + goto s_n_llhttp__internal__n_header_value_connection_2; + } + case 'u': { + p++; + goto s_n_llhttp__internal__n_header_value_connection_3; + } + default: { + goto s_n_llhttp__internal__n_header_value_connection_token; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_error_23: + s_n_llhttp__internal__n_error_23: { + state->error = 0xb; + state->reason = "Content-Length overflow"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_error_24: + s_n_llhttp__internal__n_error_24: { + state->error = 0xb; + state->reason = "Invalid character in Content-Length"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_content_length_ws: + s_n_llhttp__internal__n_header_value_content_length_ws: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_content_length_ws; + } + switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_invoke_or_flags_15; + } + case 13: { + goto s_n_llhttp__internal__n_invoke_or_flags_15; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_header_value_content_length_ws; + } + default: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_5; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_content_length: + s_n_llhttp__internal__n_header_value_content_length: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_content_length; + } + switch (*p) { + case '0': { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + case '1': { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + case '2': { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + case '3': { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + case '4': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + case '5': { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + case '6': { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + case '7': { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + case '8': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + case '9': { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + default: { + goto s_n_llhttp__internal__n_header_value_content_length_ws; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_te_chunked_last: + s_n_llhttp__internal__n_header_value_te_chunked_last: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_te_chunked_last; + } + switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_invoke_update_header_state_7; + } + case 13: { + goto s_n_llhttp__internal__n_invoke_update_header_state_7; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_header_value_te_chunked_last; + } + default: { + goto s_n_llhttp__internal__n_header_value_te_chunked; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_te_token_ows: + s_n_llhttp__internal__n_header_value_te_token_ows: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_te_token_ows; + } + switch (*p) { + case 9: { + p++; + goto s_n_llhttp__internal__n_header_value_te_token_ows; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_header_value_te_token_ows; + } + default: { + goto s_n_llhttp__internal__n_header_value_te_chunked; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value: + s_n_llhttp__internal__n_header_value: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + if (p == endp) { + return s_n_llhttp__internal__n_header_value; + } + #ifdef __SSE4_2__ + if (endp - p >= 16) { + __m128i ranges; + __m128i input; + int avail; + int match_len; + + /* Load input */ + input = _mm_loadu_si128((__m128i const*) p); + ranges = _mm_loadu_si128((__m128i const*) llparse_blob7); + + /* Find first character that does not match `ranges` */ + match_len = _mm_cmpestri(ranges, 6, + input, 16, + _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES | + _SIDD_NEGATIVE_POLARITY); + + if (match_len != 0) { + p += match_len; + goto s_n_llhttp__internal__n_header_value; + } + goto s_n_llhttp__internal__n_header_value_otherwise; + } + #endif /* __SSE4_2__ */ + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_header_value; + } + default: { + goto s_n_llhttp__internal__n_header_value_otherwise; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_te_token: + s_n_llhttp__internal__n_header_value_te_token: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + if (p == endp) { + return s_n_llhttp__internal__n_header_value_te_token; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_header_value_te_token; + } + case 2: { + p++; + goto s_n_llhttp__internal__n_header_value_te_token_ows; + } + default: { + goto s_n_llhttp__internal__n_invoke_update_header_state_8; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_te_chunked: + s_n_llhttp__internal__n_header_value_te_chunked: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_value_te_chunked; + } + match_seq = llparse__match_sequence_to_lower_unsafe(state, p, endp, llparse_blob6, 7); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_header_value_te_chunked_last; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_value_te_chunked; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_header_value_te_token; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1: + s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_header_value; + goto s_n_llhttp__internal__n_invoke_load_header_state_2; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_discard_ws: + s_n_llhttp__internal__n_header_value_discard_ws: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_discard_ws; + } + switch (*p) { + case 9: { + p++; + goto s_n_llhttp__internal__n_header_value_discard_ws; + } + case 10: { + p++; + goto s_n_llhttp__internal__n_header_value_discard_lws; + } + case 13: { + p++; + goto s_n_llhttp__internal__n_header_value_discard_ws_almost_done; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_header_value_discard_ws; + } + default: { + goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete: + s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete: { + switch (llhttp__on_header_field_complete(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_discard_ws; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_general_otherwise: + s_n_llhttp__internal__n_header_field_general_otherwise: { + if (p == endp) { + return s_n_llhttp__internal__n_header_field_general_otherwise; + } + switch (*p) { + case ':': { + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_field_1; + } + default: { + goto s_n_llhttp__internal__n_error_25; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_general: + s_n_llhttp__internal__n_header_field_general: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + if (p == endp) { + return s_n_llhttp__internal__n_header_field_general; + } + #ifdef __SSE4_2__ + if (endp - p >= 16) { + __m128i ranges; + __m128i input; + int avail; + int match_len; + + /* Load input */ + input = _mm_loadu_si128((__m128i const*) p); + ranges = _mm_loadu_si128((__m128i const*) llparse_blob8); + + /* Find first character that does not match `ranges` */ + match_len = _mm_cmpestri(ranges, 16, + input, 16, + _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES | + _SIDD_NEGATIVE_POLARITY); + + if (match_len != 0) { + p += match_len; + goto s_n_llhttp__internal__n_header_field_general; + } + ranges = _mm_loadu_si128((__m128i const*) llparse_blob9); + + /* Find first character that does not match `ranges` */ + match_len = _mm_cmpestri(ranges, 2, + input, 16, + _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES | + _SIDD_NEGATIVE_POLARITY); + + if (match_len != 0) { + p += match_len; + goto s_n_llhttp__internal__n_header_field_general; + } + goto s_n_llhttp__internal__n_header_field_general_otherwise; + } + #endif /* __SSE4_2__ */ + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_header_field_general; + } + default: { + goto s_n_llhttp__internal__n_header_field_general_otherwise; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_colon: + s_n_llhttp__internal__n_header_field_colon: { + if (p == endp) { + return s_n_llhttp__internal__n_header_field_colon; + } + switch (*p) { + case ' ': { + p++; + goto s_n_llhttp__internal__n_header_field_colon; + } + case ':': { + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_field; + } + default: { + goto s_n_llhttp__internal__n_invoke_update_header_state_9; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_3: + s_n_llhttp__internal__n_header_field_3: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_field_3; + } + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob2, 6); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_store_header_state; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_field_3; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_invoke_update_header_state_10; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_4: + s_n_llhttp__internal__n_header_field_4: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_field_4; + } + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob10, 10); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_store_header_state; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_field_4; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_invoke_update_header_state_10; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_2: + s_n_llhttp__internal__n_header_field_2: { + if (p == endp) { + return s_n_llhttp__internal__n_header_field_2; + } + switch (((*p) >= 'A' && (*p) <= 'Z' ? (*p | 0x20) : (*p))) { + case 'n': { + p++; + goto s_n_llhttp__internal__n_header_field_3; + } + case 't': { + p++; + goto s_n_llhttp__internal__n_header_field_4; + } + default: { + goto s_n_llhttp__internal__n_invoke_update_header_state_10; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_1: + s_n_llhttp__internal__n_header_field_1: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_field_1; + } + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob1, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_header_field_2; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_field_1; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_invoke_update_header_state_10; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_5: + s_n_llhttp__internal__n_header_field_5: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_field_5; + } + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob11, 15); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_store_header_state; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_field_5; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_invoke_update_header_state_10; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_6: + s_n_llhttp__internal__n_header_field_6: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_field_6; + } + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob12, 16); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_store_header_state; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_field_6; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_invoke_update_header_state_10; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_7: + s_n_llhttp__internal__n_header_field_7: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_field_7; + } + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob13, 6); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_store_header_state; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_field_7; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_invoke_update_header_state_10; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field: + s_n_llhttp__internal__n_header_field: { + if (p == endp) { + return s_n_llhttp__internal__n_header_field; + } + switch (((*p) >= 'A' && (*p) <= 'Z' ? (*p | 0x20) : (*p))) { + case 'c': { + p++; + goto s_n_llhttp__internal__n_header_field_1; + } + case 'p': { + p++; + goto s_n_llhttp__internal__n_header_field_5; + } + case 't': { + p++; + goto s_n_llhttp__internal__n_header_field_6; + } + case 'u': { + p++; + goto s_n_llhttp__internal__n_header_field_7; + } + default: { + goto s_n_llhttp__internal__n_invoke_update_header_state_10; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_llhttp__on_header_field: + s_n_llhttp__internal__n_span_start_llhttp__on_header_field: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_header_field; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_header_field; + goto s_n_llhttp__internal__n_header_field; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_start: + s_n_llhttp__internal__n_header_field_start: { + if (p == endp) { + return s_n_llhttp__internal__n_header_field_start; + } + switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_headers_almost_done; + } + case 13: { + p++; + goto s_n_llhttp__internal__n_headers_almost_done; + } + default: { + goto s_n_llhttp__internal__n_span_start_llhttp__on_header_field; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_to_http_09: + s_n_llhttp__internal__n_url_to_http_09: { + if (p == endp) { + return s_n_llhttp__internal__n_url_to_http_09; + } + switch (*p) { + case 9: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + case 12: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + default: { + goto s_n_llhttp__internal__n_invoke_update_http_major; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_skip_to_http09: + s_n_llhttp__internal__n_url_skip_to_http09: { + if (p == endp) { + return s_n_llhttp__internal__n_url_skip_to_http09; + } + switch (*p) { + case 9: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + case 12: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + default: { + p++; + goto s_n_llhttp__internal__n_url_to_http_09; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_skip_lf_to_http09_1: + s_n_llhttp__internal__n_url_skip_lf_to_http09_1: { + if (p == endp) { + return s_n_llhttp__internal__n_url_skip_lf_to_http09_1; + } + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_url_to_http_09; + } + default: { + goto s_n_llhttp__internal__n_error_26; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_skip_lf_to_http09: + s_n_llhttp__internal__n_url_skip_lf_to_http09: { + if (p == endp) { + return s_n_llhttp__internal__n_url_skip_lf_to_http09; + } + switch (*p) { + case 9: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + case 12: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + case 13: { + p++; + goto s_n_llhttp__internal__n_url_skip_lf_to_http09_1; + } + default: { + goto s_n_llhttp__internal__n_error_26; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_pri_upgrade: + s_n_llhttp__internal__n_req_pri_upgrade: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_req_pri_upgrade; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob15, 10); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_error_29; + } + case kMatchPause: { + return s_n_llhttp__internal__n_req_pri_upgrade; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_30; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_http_complete_1: + s_n_llhttp__internal__n_req_http_complete_1: { + if (p == endp) { + return s_n_llhttp__internal__n_req_http_complete_1; + } + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_header_field_start; + } + default: { + goto s_n_llhttp__internal__n_error_28; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_http_complete: + s_n_llhttp__internal__n_req_http_complete: { + if (p == endp) { + return s_n_llhttp__internal__n_req_http_complete; + } + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_header_field_start; + } + case 13: { + p++; + goto s_n_llhttp__internal__n_req_http_complete_1; + } + default: { + goto s_n_llhttp__internal__n_error_28; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_http_minor: + s_n_llhttp__internal__n_req_http_minor: { + if (p == endp) { + return s_n_llhttp__internal__n_req_http_minor; + } + switch (*p) { + case '0': { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + case '1': { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + case '2': { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + case '3': { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + case '4': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + case '5': { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + case '6': { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + case '7': { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + case '8': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + case '9': { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + default: { + goto s_n_llhttp__internal__n_error_31; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_http_dot: + s_n_llhttp__internal__n_req_http_dot: { + if (p == endp) { + return s_n_llhttp__internal__n_req_http_dot; + } + switch (*p) { + case '.': { + p++; + goto s_n_llhttp__internal__n_req_http_minor; + } + default: { + goto s_n_llhttp__internal__n_error_32; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_http_major: + s_n_llhttp__internal__n_req_http_major: { + if (p == endp) { + return s_n_llhttp__internal__n_req_http_major; + } + switch (*p) { + case '0': { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + case '1': { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + case '2': { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + case '3': { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + case '4': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + case '5': { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + case '6': { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + case '7': { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + case '8': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + case '9': { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + default: { + goto s_n_llhttp__internal__n_error_33; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_http_start_1: + s_n_llhttp__internal__n_req_http_start_1: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_req_http_start_1; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob14, 4); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_invoke_load_method; + } + case kMatchPause: { + return s_n_llhttp__internal__n_req_http_start_1; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_36; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_http_start_2: + s_n_llhttp__internal__n_req_http_start_2: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_req_http_start_2; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob16, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_invoke_load_method_2; + } + case kMatchPause: { + return s_n_llhttp__internal__n_req_http_start_2; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_36; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_http_start_3: + s_n_llhttp__internal__n_req_http_start_3: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_req_http_start_3; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob17, 4); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_invoke_load_method_3; + } + case kMatchPause: { + return s_n_llhttp__internal__n_req_http_start_3; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_36; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_http_start: + s_n_llhttp__internal__n_req_http_start: { + if (p == endp) { + return s_n_llhttp__internal__n_req_http_start; + } + switch (*p) { + case ' ': { + p++; + goto s_n_llhttp__internal__n_req_http_start; + } + case 'H': { + p++; + goto s_n_llhttp__internal__n_req_http_start_1; + } + case 'I': { + p++; + goto s_n_llhttp__internal__n_req_http_start_2; + } + case 'R': { + p++; + goto s_n_llhttp__internal__n_req_http_start_3; + } + default: { + goto s_n_llhttp__internal__n_error_36; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_to_http: + s_n_llhttp__internal__n_url_to_http: { + if (p == endp) { + return s_n_llhttp__internal__n_url_to_http; + } + switch (*p) { + case 9: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + case 12: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + default: { + goto s_n_llhttp__internal__n_invoke_llhttp__on_url_complete_1; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_skip_to_http: + s_n_llhttp__internal__n_url_skip_to_http: { + if (p == endp) { + return s_n_llhttp__internal__n_url_skip_to_http; + } + switch (*p) { + case 9: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + case 12: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + default: { + p++; + goto s_n_llhttp__internal__n_url_to_http; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_fragment: + s_n_llhttp__internal__n_url_fragment: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 3, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + if (p == endp) { + return s_n_llhttp__internal__n_url_fragment; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + case 2: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_6; + } + case 3: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_7; + } + case 4: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_8; + } + case 5: { + p++; + goto s_n_llhttp__internal__n_url_fragment; + } + default: { + goto s_n_llhttp__internal__n_error_37; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_end_stub_query_3: + s_n_llhttp__internal__n_span_end_stub_query_3: { + if (p == endp) { + return s_n_llhttp__internal__n_span_end_stub_query_3; + } + p++; + goto s_n_llhttp__internal__n_url_fragment; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_query: + s_n_llhttp__internal__n_url_query: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 3, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 5, 5, 6, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + if (p == endp) { + return s_n_llhttp__internal__n_url_query; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + case 2: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_9; + } + case 3: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_10; + } + case 4: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_11; + } + case 5: { + p++; + goto s_n_llhttp__internal__n_url_query; + } + case 6: { + goto s_n_llhttp__internal__n_span_end_stub_query_3; + } + default: { + goto s_n_llhttp__internal__n_error_38; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_query_or_fragment: + s_n_llhttp__internal__n_url_query_or_fragment: { + if (p == endp) { + return s_n_llhttp__internal__n_url_query_or_fragment; + } + switch (*p) { + case 9: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + case 10: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_3; + } + case 12: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + case 13: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_4; + } + case ' ': { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_5; + } + case '#': { + p++; + goto s_n_llhttp__internal__n_url_fragment; + } + case '?': { + p++; + goto s_n_llhttp__internal__n_url_query; + } + default: { + goto s_n_llhttp__internal__n_error_39; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_path: + s_n_llhttp__internal__n_url_path: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 2, 2, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + if (p == endp) { + return s_n_llhttp__internal__n_url_path; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + case 2: { + p++; + goto s_n_llhttp__internal__n_url_path; + } + default: { + goto s_n_llhttp__internal__n_url_query_or_fragment; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_stub_path_2: + s_n_llhttp__internal__n_span_start_stub_path_2: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_stub_path_2; + } + p++; + goto s_n_llhttp__internal__n_url_path; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_stub_path: + s_n_llhttp__internal__n_span_start_stub_path: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_stub_path; + } + p++; + goto s_n_llhttp__internal__n_url_path; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_stub_path_1: + s_n_llhttp__internal__n_span_start_stub_path_1: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_stub_path_1; + } + p++; + goto s_n_llhttp__internal__n_url_path; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_server_with_at: + s_n_llhttp__internal__n_url_server_with_at: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 3, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 5, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 5, 0, 7, + 8, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 5, 0, 5, + 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 5, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + if (p == endp) { + return s_n_llhttp__internal__n_url_server_with_at; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + case 2: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_12; + } + case 3: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_13; + } + case 4: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_14; + } + case 5: { + p++; + goto s_n_llhttp__internal__n_url_server; + } + case 6: { + goto s_n_llhttp__internal__n_span_start_stub_path_1; + } + case 7: { + p++; + goto s_n_llhttp__internal__n_url_query; + } + case 8: { + p++; + goto s_n_llhttp__internal__n_error_40; + } + default: { + goto s_n_llhttp__internal__n_error_41; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_server: + s_n_llhttp__internal__n_url_server: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 3, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 5, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 5, 0, 7, + 8, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 5, 0, 5, + 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 5, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + if (p == endp) { + return s_n_llhttp__internal__n_url_server; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + case 2: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url; + } + case 3: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_1; + } + case 4: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_2; + } + case 5: { + p++; + goto s_n_llhttp__internal__n_url_server; + } + case 6: { + goto s_n_llhttp__internal__n_span_start_stub_path; + } + case 7: { + p++; + goto s_n_llhttp__internal__n_url_query; + } + case 8: { + p++; + goto s_n_llhttp__internal__n_url_server_with_at; + } + default: { + goto s_n_llhttp__internal__n_error_42; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_schema_delim_1: + s_n_llhttp__internal__n_url_schema_delim_1: { + if (p == endp) { + return s_n_llhttp__internal__n_url_schema_delim_1; + } + switch (*p) { + case '/': { + p++; + goto s_n_llhttp__internal__n_url_server; + } + default: { + goto s_n_llhttp__internal__n_error_44; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_schema_delim: + s_n_llhttp__internal__n_url_schema_delim: { + if (p == endp) { + return s_n_llhttp__internal__n_url_schema_delim; + } + switch (*p) { + case 9: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + case 10: { + p++; + goto s_n_llhttp__internal__n_error_43; + } + case 12: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + case 13: { + p++; + goto s_n_llhttp__internal__n_error_43; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_error_43; + } + case '/': { + p++; + goto s_n_llhttp__internal__n_url_schema_delim_1; + } + default: { + goto s_n_llhttp__internal__n_error_44; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_end_stub_schema: + s_n_llhttp__internal__n_span_end_stub_schema: { + if (p == endp) { + return s_n_llhttp__internal__n_span_end_stub_schema; + } + p++; + goto s_n_llhttp__internal__n_url_schema_delim; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_schema: + s_n_llhttp__internal__n_url_schema: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 2, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + if (p == endp) { + return s_n_llhttp__internal__n_url_schema; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + case 2: { + p++; + goto s_n_llhttp__internal__n_error_43; + } + case 3: { + goto s_n_llhttp__internal__n_span_end_stub_schema; + } + case 4: { + p++; + goto s_n_llhttp__internal__n_url_schema; + } + default: { + goto s_n_llhttp__internal__n_error_45; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_start: + s_n_llhttp__internal__n_url_start: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 2, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + if (p == endp) { + return s_n_llhttp__internal__n_url_start; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + case 2: { + p++; + goto s_n_llhttp__internal__n_error_43; + } + case 3: { + goto s_n_llhttp__internal__n_span_start_stub_path_2; + } + case 4: { + goto s_n_llhttp__internal__n_url_schema; + } + default: { + goto s_n_llhttp__internal__n_error_46; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_llhttp__on_url_1: + s_n_llhttp__internal__n_span_start_llhttp__on_url_1: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_url_1; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_url; + goto s_n_llhttp__internal__n_url_start; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_entry_normal: + s_n_llhttp__internal__n_url_entry_normal: { + if (p == endp) { + return s_n_llhttp__internal__n_url_entry_normal; + } + switch (*p) { + case 9: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + case 12: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + default: { + goto s_n_llhttp__internal__n_span_start_llhttp__on_url_1; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_llhttp__on_url: + s_n_llhttp__internal__n_span_start_llhttp__on_url: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_url; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_url; + goto s_n_llhttp__internal__n_url_server; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_entry_connect: + s_n_llhttp__internal__n_url_entry_connect: { + if (p == endp) { + return s_n_llhttp__internal__n_url_entry_connect; + } + switch (*p) { + case 9: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + case 12: { + p++; + goto s_n_llhttp__internal__n_error_1; + } + default: { + goto s_n_llhttp__internal__n_span_start_llhttp__on_url; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_spaces_before_url: + s_n_llhttp__internal__n_req_spaces_before_url: { + if (p == endp) { + return s_n_llhttp__internal__n_req_spaces_before_url; + } + switch (*p) { + case ' ': { + p++; + goto s_n_llhttp__internal__n_req_spaces_before_url; + } + default: { + goto s_n_llhttp__internal__n_invoke_is_equal_method; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_first_space_before_url: + s_n_llhttp__internal__n_req_first_space_before_url: { + if (p == endp) { + return s_n_llhttp__internal__n_req_first_space_before_url; + } + switch (*p) { + case ' ': { + p++; + goto s_n_llhttp__internal__n_req_spaces_before_url; + } + default: { + goto s_n_llhttp__internal__n_error_47; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_2: + s_n_llhttp__internal__n_start_req_2: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_2; + } + switch (*p) { + case 'L': { + p++; + match = 19; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_3: + s_n_llhttp__internal__n_start_req_3: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_3; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob18, 6); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 36; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_3; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_1: + s_n_llhttp__internal__n_start_req_1: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_1; + } + switch (*p) { + case 'C': { + p++; + goto s_n_llhttp__internal__n_start_req_2; + } + case 'N': { + p++; + goto s_n_llhttp__internal__n_start_req_3; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_4: + s_n_llhttp__internal__n_start_req_4: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_4; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob19, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 16; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_4; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_6: + s_n_llhttp__internal__n_start_req_6: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_6; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob20, 6); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 22; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_6; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_8: + s_n_llhttp__internal__n_start_req_8: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_8; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob21, 4); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_8; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_9: + s_n_llhttp__internal__n_start_req_9: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_9; + } + switch (*p) { + case 'Y': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_7: + s_n_llhttp__internal__n_start_req_7: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_7; + } + switch (*p) { + case 'N': { + p++; + goto s_n_llhttp__internal__n_start_req_8; + } + case 'P': { + p++; + goto s_n_llhttp__internal__n_start_req_9; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_5: + s_n_llhttp__internal__n_start_req_5: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_5; + } + switch (*p) { + case 'H': { + p++; + goto s_n_llhttp__internal__n_start_req_6; + } + case 'O': { + p++; + goto s_n_llhttp__internal__n_start_req_7; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_12: + s_n_llhttp__internal__n_start_req_12: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_12; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob22, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_12; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_13: + s_n_llhttp__internal__n_start_req_13: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_13; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob23, 5); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 35; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_13; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_11: + s_n_llhttp__internal__n_start_req_11: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_11; + } + switch (*p) { + case 'L': { + p++; + goto s_n_llhttp__internal__n_start_req_12; + } + case 'S': { + p++; + goto s_n_llhttp__internal__n_start_req_13; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_10: + s_n_llhttp__internal__n_start_req_10: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_10; + } + switch (*p) { + case 'E': { + p++; + goto s_n_llhttp__internal__n_start_req_11; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_14: + s_n_llhttp__internal__n_start_req_14: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_14; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob24, 4); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 45; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_14; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_17: + s_n_llhttp__internal__n_start_req_17: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_17; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob26, 9); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 41; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_17; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_16: + s_n_llhttp__internal__n_start_req_16: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_16; + } + switch (*p) { + case '_': { + p++; + goto s_n_llhttp__internal__n_start_req_17; + } + default: { + match = 1; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_15: + s_n_llhttp__internal__n_start_req_15: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_15; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob25, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_start_req_16; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_15; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_18: + s_n_llhttp__internal__n_start_req_18: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_18; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob27, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_18; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_20: + s_n_llhttp__internal__n_start_req_20: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_20; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob28, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 31; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_20; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_21: + s_n_llhttp__internal__n_start_req_21: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_21; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob29, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_21; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_19: + s_n_llhttp__internal__n_start_req_19: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_19; + } + switch (*p) { + case 'I': { + p++; + goto s_n_llhttp__internal__n_start_req_20; + } + case 'O': { + p++; + goto s_n_llhttp__internal__n_start_req_21; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_23: + s_n_llhttp__internal__n_start_req_23: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_23; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob30, 6); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 24; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_23; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_24: + s_n_llhttp__internal__n_start_req_24: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_24; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob31, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 23; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_24; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_26: + s_n_llhttp__internal__n_start_req_26: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_26; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob32, 7); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 21; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_26; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_28: + s_n_llhttp__internal__n_start_req_28: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_28; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob33, 6); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 30; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_28; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_29: + s_n_llhttp__internal__n_start_req_29: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_29; + } + switch (*p) { + case 'L': { + p++; + match = 10; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_27: + s_n_llhttp__internal__n_start_req_27: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_27; + } + switch (*p) { + case 'A': { + p++; + goto s_n_llhttp__internal__n_start_req_28; + } + case 'O': { + p++; + goto s_n_llhttp__internal__n_start_req_29; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_25: + s_n_llhttp__internal__n_start_req_25: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_25; + } + switch (*p) { + case 'A': { + p++; + goto s_n_llhttp__internal__n_start_req_26; + } + case 'C': { + p++; + goto s_n_llhttp__internal__n_start_req_27; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_30: + s_n_llhttp__internal__n_start_req_30: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_30; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob34, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 11; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_30; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_22: + s_n_llhttp__internal__n_start_req_22: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_22; + } + switch (*p) { + case '-': { + p++; + goto s_n_llhttp__internal__n_start_req_23; + } + case 'E': { + p++; + goto s_n_llhttp__internal__n_start_req_24; + } + case 'K': { + p++; + goto s_n_llhttp__internal__n_start_req_25; + } + case 'O': { + p++; + goto s_n_llhttp__internal__n_start_req_30; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_31: + s_n_llhttp__internal__n_start_req_31: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_31; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob35, 5); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 25; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_31; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_32: + s_n_llhttp__internal__n_start_req_32: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_32; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob36, 6); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_32; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_35: + s_n_llhttp__internal__n_start_req_35: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_35; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob37, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 28; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_35; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_36: + s_n_llhttp__internal__n_start_req_36: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_36; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob38, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 39; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_36; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_34: + s_n_llhttp__internal__n_start_req_34: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_34; + } + switch (*p) { + case 'T': { + p++; + goto s_n_llhttp__internal__n_start_req_35; + } + case 'U': { + p++; + goto s_n_llhttp__internal__n_start_req_36; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_37: + s_n_llhttp__internal__n_start_req_37: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_37; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob39, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 38; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_37; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_38: + s_n_llhttp__internal__n_start_req_38: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_38; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob40, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_38; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_42: + s_n_llhttp__internal__n_start_req_42: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_42; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob41, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 12; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_42; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_43: + s_n_llhttp__internal__n_start_req_43: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_43; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob42, 4); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 13; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_43; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_41: + s_n_llhttp__internal__n_start_req_41: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_41; + } + switch (*p) { + case 'F': { + p++; + goto s_n_llhttp__internal__n_start_req_42; + } + case 'P': { + p++; + goto s_n_llhttp__internal__n_start_req_43; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_40: + s_n_llhttp__internal__n_start_req_40: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_40; + } + switch (*p) { + case 'P': { + p++; + goto s_n_llhttp__internal__n_start_req_41; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_39: + s_n_llhttp__internal__n_start_req_39: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_39; + } + switch (*p) { + case 'I': { + p++; + match = 34; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case 'O': { + p++; + goto s_n_llhttp__internal__n_start_req_40; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_45: + s_n_llhttp__internal__n_start_req_45: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_45; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob43, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 29; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_45; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_44: + s_n_llhttp__internal__n_start_req_44: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_44; + } + switch (*p) { + case 'R': { + p++; + goto s_n_llhttp__internal__n_start_req_45; + } + case 'T': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_33: + s_n_llhttp__internal__n_start_req_33: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_33; + } + switch (*p) { + case 'A': { + p++; + goto s_n_llhttp__internal__n_start_req_34; + } + case 'L': { + p++; + goto s_n_llhttp__internal__n_start_req_37; + } + case 'O': { + p++; + goto s_n_llhttp__internal__n_start_req_38; + } + case 'R': { + p++; + goto s_n_llhttp__internal__n_start_req_39; + } + case 'U': { + p++; + goto s_n_llhttp__internal__n_start_req_44; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_48: + s_n_llhttp__internal__n_start_req_48: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_48; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob44, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 17; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_48; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_49: + s_n_llhttp__internal__n_start_req_49: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_49; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob45, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 44; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_49; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_50: + s_n_llhttp__internal__n_start_req_50: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_50; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob46, 5); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 43; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_50; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_51: + s_n_llhttp__internal__n_start_req_51: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_51; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob47, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 20; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_51; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_47: + s_n_llhttp__internal__n_start_req_47: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_47; + } + switch (*p) { + case 'B': { + p++; + goto s_n_llhttp__internal__n_start_req_48; + } + case 'C': { + p++; + goto s_n_llhttp__internal__n_start_req_49; + } + case 'D': { + p++; + goto s_n_llhttp__internal__n_start_req_50; + } + case 'P': { + p++; + goto s_n_llhttp__internal__n_start_req_51; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_46: + s_n_llhttp__internal__n_start_req_46: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_46; + } + switch (*p) { + case 'E': { + p++; + goto s_n_llhttp__internal__n_start_req_47; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_54: + s_n_llhttp__internal__n_start_req_54: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_54; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob48, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 14; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_54; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_56: + s_n_llhttp__internal__n_start_req_56: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_56; + } + switch (*p) { + case 'P': { + p++; + match = 37; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_57: + s_n_llhttp__internal__n_start_req_57: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_57; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob49, 9); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 42; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_57; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_55: + s_n_llhttp__internal__n_start_req_55: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_55; + } + switch (*p) { + case 'U': { + p++; + goto s_n_llhttp__internal__n_start_req_56; + } + case '_': { + p++; + goto s_n_llhttp__internal__n_start_req_57; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_53: + s_n_llhttp__internal__n_start_req_53: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_53; + } + switch (*p) { + case 'A': { + p++; + goto s_n_llhttp__internal__n_start_req_54; + } + case 'T': { + p++; + goto s_n_llhttp__internal__n_start_req_55; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_58: + s_n_llhttp__internal__n_start_req_58: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_58; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob50, 4); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 33; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_58; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_59: + s_n_llhttp__internal__n_start_req_59: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_59; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob51, 7); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 26; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_59; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_52: + s_n_llhttp__internal__n_start_req_52: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_52; + } + switch (*p) { + case 'E': { + p++; + goto s_n_llhttp__internal__n_start_req_53; + } + case 'O': { + p++; + goto s_n_llhttp__internal__n_start_req_58; + } + case 'U': { + p++; + goto s_n_llhttp__internal__n_start_req_59; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_61: + s_n_llhttp__internal__n_start_req_61: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_61; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob52, 6); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 40; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_61; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_62: + s_n_llhttp__internal__n_start_req_62: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_62; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob53, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_62; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_60: + s_n_llhttp__internal__n_start_req_60: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_60; + } + switch (*p) { + case 'E': { + p++; + goto s_n_llhttp__internal__n_start_req_61; + } + case 'R': { + p++; + goto s_n_llhttp__internal__n_start_req_62; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_65: + s_n_llhttp__internal__n_start_req_65: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_65; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob54, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 18; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_65; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_67: + s_n_llhttp__internal__n_start_req_67: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_67; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob55, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 32; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_67; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_68: + s_n_llhttp__internal__n_start_req_68: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_68; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob56, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 15; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_68; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_66: + s_n_llhttp__internal__n_start_req_66: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_66; + } + switch (*p) { + case 'I': { + p++; + goto s_n_llhttp__internal__n_start_req_67; + } + case 'O': { + p++; + goto s_n_llhttp__internal__n_start_req_68; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_69: + s_n_llhttp__internal__n_start_req_69: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_69; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob57, 8); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 27; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_69; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_64: + s_n_llhttp__internal__n_start_req_64: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_64; + } + switch (*p) { + case 'B': { + p++; + goto s_n_llhttp__internal__n_start_req_65; + } + case 'L': { + p++; + goto s_n_llhttp__internal__n_start_req_66; + } + case 'S': { + p++; + goto s_n_llhttp__internal__n_start_req_69; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_63: + s_n_llhttp__internal__n_start_req_63: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_63; + } + switch (*p) { + case 'N': { + p++; + goto s_n_llhttp__internal__n_start_req_64; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req: + s_n_llhttp__internal__n_start_req: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req; + } + switch (*p) { + case 'A': { + p++; + goto s_n_llhttp__internal__n_start_req_1; + } + case 'B': { + p++; + goto s_n_llhttp__internal__n_start_req_4; + } + case 'C': { + p++; + goto s_n_llhttp__internal__n_start_req_5; + } + case 'D': { + p++; + goto s_n_llhttp__internal__n_start_req_10; + } + case 'F': { + p++; + goto s_n_llhttp__internal__n_start_req_14; + } + case 'G': { + p++; + goto s_n_llhttp__internal__n_start_req_15; + } + case 'H': { + p++; + goto s_n_llhttp__internal__n_start_req_18; + } + case 'L': { + p++; + goto s_n_llhttp__internal__n_start_req_19; + } + case 'M': { + p++; + goto s_n_llhttp__internal__n_start_req_22; + } + case 'N': { + p++; + goto s_n_llhttp__internal__n_start_req_31; + } + case 'O': { + p++; + goto s_n_llhttp__internal__n_start_req_32; + } + case 'P': { + p++; + goto s_n_llhttp__internal__n_start_req_33; + } + case 'R': { + p++; + goto s_n_llhttp__internal__n_start_req_46; + } + case 'S': { + p++; + goto s_n_llhttp__internal__n_start_req_52; + } + case 'T': { + p++; + goto s_n_llhttp__internal__n_start_req_60; + } + case 'U': { + p++; + goto s_n_llhttp__internal__n_start_req_63; + } + default: { + goto s_n_llhttp__internal__n_error_56; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_llhttp__on_status_complete: + s_n_llhttp__internal__n_invoke_llhttp__on_status_complete: { + switch (llhttp__on_status_complete(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_field_start; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_res_line_almost_done: + s_n_llhttp__internal__n_res_line_almost_done: { + if (p == endp) { + return s_n_llhttp__internal__n_res_line_almost_done; + } + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_res_status: + s_n_llhttp__internal__n_res_status: { + if (p == endp) { + return s_n_llhttp__internal__n_res_status; + } + switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_status; + } + case 13: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_status_1; + } + default: { + p++; + goto s_n_llhttp__internal__n_res_status; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_llhttp__on_status: + s_n_llhttp__internal__n_span_start_llhttp__on_status: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_status; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_status; + goto s_n_llhttp__internal__n_res_status; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_res_status_start: + s_n_llhttp__internal__n_res_status_start: { + if (p == endp) { + return s_n_llhttp__internal__n_res_status_start; + } + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; + } + case 13: { + p++; + goto s_n_llhttp__internal__n_res_line_almost_done; + } + default: { + goto s_n_llhttp__internal__n_span_start_llhttp__on_status; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_res_status_code_otherwise: + s_n_llhttp__internal__n_res_status_code_otherwise: { + if (p == endp) { + return s_n_llhttp__internal__n_res_status_code_otherwise; + } + switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_res_status_start; + } + case 13: { + goto s_n_llhttp__internal__n_res_status_start; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_res_status_start; + } + default: { + goto s_n_llhttp__internal__n_error_50; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_res_status_code: + s_n_llhttp__internal__n_res_status_code: { + if (p == endp) { + return s_n_llhttp__internal__n_res_status_code; + } + switch (*p) { + case '0': { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + case '1': { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + case '2': { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + case '3': { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + case '4': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + case '5': { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + case '6': { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + case '7': { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + case '8': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + case '9': { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + default: { + goto s_n_llhttp__internal__n_res_status_code_otherwise; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_res_http_end: + s_n_llhttp__internal__n_res_http_end: { + if (p == endp) { + return s_n_llhttp__internal__n_res_http_end; + } + switch (*p) { + case ' ': { + p++; + goto s_n_llhttp__internal__n_invoke_update_status_code; + } + default: { + goto s_n_llhttp__internal__n_error_51; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_res_http_minor: + s_n_llhttp__internal__n_res_http_minor: { + if (p == endp) { + return s_n_llhttp__internal__n_res_http_minor; + } + switch (*p) { + case '0': { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + case '1': { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + case '2': { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + case '3': { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + case '4': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + case '5': { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + case '6': { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + case '7': { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + case '8': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + case '9': { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + default: { + goto s_n_llhttp__internal__n_error_52; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_res_http_dot: + s_n_llhttp__internal__n_res_http_dot: { + if (p == endp) { + return s_n_llhttp__internal__n_res_http_dot; + } + switch (*p) { + case '.': { + p++; + goto s_n_llhttp__internal__n_res_http_minor; + } + default: { + goto s_n_llhttp__internal__n_error_53; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_res_http_major: + s_n_llhttp__internal__n_res_http_major: { + if (p == endp) { + return s_n_llhttp__internal__n_res_http_major; + } + switch (*p) { + case '0': { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + case '1': { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + case '2': { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + case '3': { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + case '4': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + case '5': { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + case '6': { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + case '7': { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + case '8': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + case '9': { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + default: { + goto s_n_llhttp__internal__n_error_54; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_res: + s_n_llhttp__internal__n_start_res: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_res; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob58, 5); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_res_http_major; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_res; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_57; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_or_res_method_2: + s_n_llhttp__internal__n_req_or_res_method_2: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_req_or_res_method_2; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob59, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_store_method; + } + case kMatchPause: { + return s_n_llhttp__internal__n_req_or_res_method_2; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_55; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_or_res_method_3: + s_n_llhttp__internal__n_req_or_res_method_3: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_req_or_res_method_3; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob60, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_invoke_update_type_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_req_or_res_method_3; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_55; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_or_res_method_1: + s_n_llhttp__internal__n_req_or_res_method_1: { + if (p == endp) { + return s_n_llhttp__internal__n_req_or_res_method_1; + } + switch (*p) { + case 'E': { + p++; + goto s_n_llhttp__internal__n_req_or_res_method_2; + } + case 'T': { + p++; + goto s_n_llhttp__internal__n_req_or_res_method_3; + } + default: { + goto s_n_llhttp__internal__n_error_55; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_or_res_method: + s_n_llhttp__internal__n_req_or_res_method: { + if (p == endp) { + return s_n_llhttp__internal__n_req_or_res_method; + } + switch (*p) { + case 'H': { + p++; + goto s_n_llhttp__internal__n_req_or_res_method_1; + } + default: { + goto s_n_llhttp__internal__n_error_55; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_or_res: + s_n_llhttp__internal__n_start_req_or_res: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_or_res; + } + switch (*p) { + case 'H': { + goto s_n_llhttp__internal__n_req_or_res_method; + } + default: { + goto s_n_llhttp__internal__n_invoke_update_type_2; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_load_type: + s_n_llhttp__internal__n_invoke_load_type: { + switch (llhttp__internal__c_load_type(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_start_req; + case 2: + goto s_n_llhttp__internal__n_start_res; + default: + goto s_n_llhttp__internal__n_start_req_or_res; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start: + s_n_llhttp__internal__n_start: { + if (p == endp) { + return s_n_llhttp__internal__n_start; + } + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_start; + } + case 13: { + p++; + goto s_n_llhttp__internal__n_start; + } + default: { + goto s_n_llhttp__internal__n_invoke_update_finish; + } + } + /* UNREACHABLE */; + abort(); + } + default: + /* UNREACHABLE */ + abort(); + } + s_n_llhttp__internal__n_error_1: { + state->error = 0x7; + state->reason = "Invalid characters in url (strict mode)"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_43: { + state->error = 0x7; + state->reason = "Invalid characters in url"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_finish_2: { + switch (llhttp__internal__c_update_finish_1(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_start; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_4: { + state->error = 0x5; + state->reason = "Data after `Connection: close`"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_finish_1: { + switch (llhttp__internal__c_update_finish_1(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_closed; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_5: { + state->error = 0x15; + state->reason = "on_message_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_is_equal_upgrade; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_13: { + state->error = 0x12; + state->reason = "`on_message_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_7: { + state->error = 0x15; + state->reason = "on_chunk_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_16: { + state->error = 0x14; + state->reason = "`on_chunk_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete_1: { + switch (llhttp__on_chunk_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; + case 21: + goto s_n_llhttp__internal__n_pause_7; + default: + goto s_n_llhttp__internal__n_error_16; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_15: { + state->error = 0x4; + state->reason = "Content-Length can't be present with Transfer-Encoding"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_2: { + state->error = 0x15; + state->reason = "on_message_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_pause_1; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_5: { + state->error = 0x12; + state->reason = "`on_message_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_1: { + switch (llhttp__on_message_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_pause_1; + case 21: + goto s_n_llhttp__internal__n_pause_2; + default: + goto s_n_llhttp__internal__n_error_5; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_11: { + state->error = 0xc; + state->reason = "Chunk size overflow"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_3: { + state->error = 0x15; + state->reason = "on_chunk_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_update_content_length; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_7: { + state->error = 0x14; + state->reason = "`on_chunk_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete: { + switch (llhttp__on_chunk_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_update_content_length; + case 21: + goto s_n_llhttp__internal__n_pause_3; + default: + goto s_n_llhttp__internal__n_error_7; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_8: { + state->error = 0x2; + state->reason = "Expected CRLF after chunk"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_body: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_body(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_chunk_data_almost_done; + return s_error; + } + goto s_n_llhttp__internal__n_chunk_data_almost_done; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags: { + switch (llhttp__internal__c_or_flags(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_field_start; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_4: { + state->error = 0x15; + state->reason = "on_chunk_header pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_is_equal_content_length; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_6: { + state->error = 0x13; + state->reason = "`on_chunk_header` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_header: { + switch (llhttp__on_chunk_header(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_is_equal_content_length; + case 21: + goto s_n_llhttp__internal__n_pause_4; + default: + goto s_n_llhttp__internal__n_error_6; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_9: { + state->error = 0x2; + state->reason = "Expected LF after chunk size"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_10: { + state->error = 0xc; + state->reason = "Invalid character in chunk size"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_mul_add_content_length: { + switch (llhttp__internal__c_mul_add_content_length(state, p, endp, match)) { + case 1: + goto s_n_llhttp__internal__n_error_11; + default: + goto s_n_llhttp__internal__n_chunk_size; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_12: { + state->error = 0xc; + state->reason = "Invalid character in chunk size"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_body_1: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_body(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; + return s_error; + } + goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_finish_3: { + switch (llhttp__internal__c_update_finish_3(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_span_start_llhttp__on_body_2; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_14: { + state->error = 0xf; + state->reason = "Request has invalid `Transfer-Encoding`"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause: { + state->error = 0x15; + state->reason = "on_message_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__after_message_complete; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_3: { + state->error = 0x12; + state->reason = "`on_message_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_message_complete: { + switch (llhttp__on_message_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_llhttp__after_message_complete; + case 21: + goto s_n_llhttp__internal__n_pause; + default: + goto s_n_llhttp__internal__n_error_3; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_1: { + switch (llhttp__internal__c_or_flags_1(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_2: { + switch (llhttp__internal__c_or_flags_1(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_upgrade: { + switch (llhttp__internal__c_update_upgrade(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_or_flags_2; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_6: { + state->error = 0x15; + state->reason = "Paused by on_headers_complete"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_2: { + state->error = 0x11; + state->reason = "User callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_headers_complete: { + switch (llhttp__on_headers_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; + case 1: + goto s_n_llhttp__internal__n_invoke_or_flags_1; + case 2: + goto s_n_llhttp__internal__n_invoke_update_upgrade; + case 21: + goto s_n_llhttp__internal__n_pause_6; + default: + goto s_n_llhttp__internal__n_error_2; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete: { + switch (llhttp__before_headers_complete(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__on_headers_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_test_lenient_flags: { + switch (llhttp__internal__c_test_lenient_flags(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_error_15; + default: + goto s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_test_flags_1: { + switch (llhttp__internal__c_test_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_test_lenient_flags; + default: + goto s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_test_flags: { + switch (llhttp__internal__c_test_flags(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete_1; + default: + goto s_n_llhttp__internal__n_invoke_test_flags_1; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_17: { + state->error = 0x2; + state->reason = "Expected LF after headers"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_18: { + state->error = 0xb; + state->reason = "Empty Content-Length"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_value: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; + return s_error; + } + goto s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state: { + switch (llhttp__internal__c_update_header_state(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_3: { + switch (llhttp__internal__c_or_flags_3(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_4: { + switch (llhttp__internal__c_or_flags_4(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_5: { + switch (llhttp__internal__c_or_flags_5(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_6: { + switch (llhttp__internal__c_or_flags_6(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_load_header_state_1: { + switch (llhttp__internal__c_load_header_state(state, p, endp)) { + case 5: + goto s_n_llhttp__internal__n_invoke_or_flags_3; + case 6: + goto s_n_llhttp__internal__n_invoke_or_flags_4; + case 7: + goto s_n_llhttp__internal__n_invoke_or_flags_5; + case 8: + goto s_n_llhttp__internal__n_invoke_or_flags_6; + default: + goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_load_header_state: { + switch (llhttp__internal__c_load_header_state(state, p, endp)) { + case 2: + goto s_n_llhttp__internal__n_error_18; + default: + goto s_n_llhttp__internal__n_invoke_load_header_state_1; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_19: { + state->error = 0x2; + state->reason = "Expected LF after CR"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_1: { + switch (llhttp__internal__c_update_header_state(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_7: { + switch (llhttp__internal__c_or_flags_3(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state_1; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_8: { + switch (llhttp__internal__c_or_flags_4(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state_1; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_9: { + switch (llhttp__internal__c_or_flags_5(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state_1; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_10: { + switch (llhttp__internal__c_or_flags_6(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_load_header_state_3: { + switch (llhttp__internal__c_load_header_state(state, p, endp)) { + case 5: + goto s_n_llhttp__internal__n_invoke_or_flags_7; + case 6: + goto s_n_llhttp__internal__n_invoke_or_flags_8; + case 7: + goto s_n_llhttp__internal__n_invoke_or_flags_9; + case 8: + goto s_n_llhttp__internal__n_invoke_or_flags_10; + default: + goto s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_20: { + state->error = 0x3; + state->reason = "Missing expected LF after header value"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_1: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_header_value_almost_done; + return s_error; + } + goto s_n_llhttp__internal__n_header_value_almost_done; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_2: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_header_value_almost_done; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_header_value_almost_done; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_3: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_header_value_almost_done; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_header_value_almost_done; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_21: { + state->error = 0xa; + state->reason = "Invalid header value char"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_1: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_header_value_lenient; + default: + goto s_n_llhttp__internal__n_error_21; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_3: { + switch (llhttp__internal__c_update_header_state(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_connection; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_11: { + switch (llhttp__internal__c_or_flags_3(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state_3; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_12: { + switch (llhttp__internal__c_or_flags_4(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state_3; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_13: { + switch (llhttp__internal__c_or_flags_5(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state_3; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_14: { + switch (llhttp__internal__c_or_flags_6(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_connection; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_load_header_state_4: { + switch (llhttp__internal__c_load_header_state(state, p, endp)) { + case 5: + goto s_n_llhttp__internal__n_invoke_or_flags_11; + case 6: + goto s_n_llhttp__internal__n_invoke_or_flags_12; + case 7: + goto s_n_llhttp__internal__n_invoke_or_flags_13; + case 8: + goto s_n_llhttp__internal__n_invoke_or_flags_14; + default: + goto s_n_llhttp__internal__n_header_value_connection; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_4: { + switch (llhttp__internal__c_update_header_state_4(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_connection_token; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_2: { + switch (llhttp__internal__c_update_header_state_2(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_connection_ws; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_5: { + switch (llhttp__internal__c_update_header_state_5(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_connection_ws; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_6: { + switch (llhttp__internal__c_update_header_state_6(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_connection_ws; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_4: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_23; + return s_error; + } + goto s_n_llhttp__internal__n_error_23; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_mul_add_content_length_1: { + switch (llhttp__internal__c_mul_add_content_length_1(state, p, endp, match)) { + case 1: + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_4; + default: + goto s_n_llhttp__internal__n_header_value_content_length; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_15: { + switch (llhttp__internal__c_or_flags_15(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_otherwise; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_5: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_24; + return s_error; + } + goto s_n_llhttp__internal__n_error_24; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_22: { + state->error = 0x4; + state->reason = "Duplicate Content-Length"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_test_flags_2: { + switch (llhttp__internal__c_test_flags_2(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_header_value_content_length; + default: + goto s_n_llhttp__internal__n_error_22; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_7: { + switch (llhttp__internal__c_update_header_state_7(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_otherwise; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_8: { + switch (llhttp__internal__c_update_header_state_4(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_and_flags: { + switch (llhttp__internal__c_and_flags(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_te_chunked; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_16: { + switch (llhttp__internal__c_or_flags_16(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_and_flags; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_17: { + switch (llhttp__internal__c_or_flags_17(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state_8; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_load_header_state_2: { + switch (llhttp__internal__c_load_header_state(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_header_value_connection; + case 2: + goto s_n_llhttp__internal__n_invoke_test_flags_2; + case 3: + goto s_n_llhttp__internal__n_invoke_or_flags_16; + case 4: + goto s_n_llhttp__internal__n_invoke_or_flags_17; + default: + goto s_n_llhttp__internal__n_header_value; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_field: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_field(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_field_1: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_field(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_25: { + state->error = 0xa; + state->reason = "Invalid header token"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_9: { + switch (llhttp__internal__c_update_header_state_4(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_field_general; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_store_header_state: { + switch (llhttp__internal__c_store_header_state(state, p, endp, match)) { + default: + goto s_n_llhttp__internal__n_header_field_colon; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_10: { + switch (llhttp__internal__c_update_header_state_4(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_field_general; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_url_complete: { + switch (llhttp__on_url_complete(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_field_start; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_http_minor: { + switch (llhttp__internal__c_update_http_minor(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__on_url_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_http_major: { + switch (llhttp__internal__c_update_http_major(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_http_minor; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_3: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_26: { + state->error = 0x7; + state->reason = "Expected CRLF"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_4: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_lf_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_lf_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_29: { + state->error = 0x16; + state->reason = "Pause on PRI/Upgrade"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_30: { + state->error = 0x9; + state->reason = "Expected HTTP/2 Connection Preface"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_28: { + state->error = 0x9; + state->reason = "Expected CRLF after version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_load_method_1: { + switch (llhttp__internal__c_load_method(state, p, endp)) { + case 34: + goto s_n_llhttp__internal__n_req_pri_upgrade; + default: + goto s_n_llhttp__internal__n_req_http_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_store_http_minor: { + switch (llhttp__internal__c_store_http_minor(state, p, endp, match)) { + default: + goto s_n_llhttp__internal__n_invoke_load_method_1; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_31: { + state->error = 0x9; + state->reason = "Invalid minor version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_32: { + state->error = 0x9; + state->reason = "Expected dot"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_store_http_major: { + switch (llhttp__internal__c_store_http_major(state, p, endp, match)) { + default: + goto s_n_llhttp__internal__n_req_http_dot; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_33: { + state->error = 0x9; + state->reason = "Invalid major version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_27: { + state->error = 0x8; + state->reason = "Invalid method for HTTP/x.x request"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_load_method: { + switch (llhttp__internal__c_load_method(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_req_http_major; + case 1: + goto s_n_llhttp__internal__n_req_http_major; + case 2: + goto s_n_llhttp__internal__n_req_http_major; + case 3: + goto s_n_llhttp__internal__n_req_http_major; + case 4: + goto s_n_llhttp__internal__n_req_http_major; + case 5: + goto s_n_llhttp__internal__n_req_http_major; + case 6: + goto s_n_llhttp__internal__n_req_http_major; + case 7: + goto s_n_llhttp__internal__n_req_http_major; + case 8: + goto s_n_llhttp__internal__n_req_http_major; + case 9: + goto s_n_llhttp__internal__n_req_http_major; + case 10: + goto s_n_llhttp__internal__n_req_http_major; + case 11: + goto s_n_llhttp__internal__n_req_http_major; + case 12: + goto s_n_llhttp__internal__n_req_http_major; + case 13: + goto s_n_llhttp__internal__n_req_http_major; + case 14: + goto s_n_llhttp__internal__n_req_http_major; + case 15: + goto s_n_llhttp__internal__n_req_http_major; + case 16: + goto s_n_llhttp__internal__n_req_http_major; + case 17: + goto s_n_llhttp__internal__n_req_http_major; + case 18: + goto s_n_llhttp__internal__n_req_http_major; + case 19: + goto s_n_llhttp__internal__n_req_http_major; + case 20: + goto s_n_llhttp__internal__n_req_http_major; + case 21: + goto s_n_llhttp__internal__n_req_http_major; + case 22: + goto s_n_llhttp__internal__n_req_http_major; + case 23: + goto s_n_llhttp__internal__n_req_http_major; + case 24: + goto s_n_llhttp__internal__n_req_http_major; + case 25: + goto s_n_llhttp__internal__n_req_http_major; + case 26: + goto s_n_llhttp__internal__n_req_http_major; + case 27: + goto s_n_llhttp__internal__n_req_http_major; + case 28: + goto s_n_llhttp__internal__n_req_http_major; + case 29: + goto s_n_llhttp__internal__n_req_http_major; + case 30: + goto s_n_llhttp__internal__n_req_http_major; + case 31: + goto s_n_llhttp__internal__n_req_http_major; + case 32: + goto s_n_llhttp__internal__n_req_http_major; + case 33: + goto s_n_llhttp__internal__n_req_http_major; + case 34: + goto s_n_llhttp__internal__n_req_http_major; + default: + goto s_n_llhttp__internal__n_error_27; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_36: { + state->error = 0x8; + state->reason = "Expected HTTP/"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_34: { + state->error = 0x8; + state->reason = "Expected SOURCE method for ICE/x.x request"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_load_method_2: { + switch (llhttp__internal__c_load_method(state, p, endp)) { + case 33: + goto s_n_llhttp__internal__n_req_http_major; + default: + goto s_n_llhttp__internal__n_error_34; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_35: { + state->error = 0x8; + state->reason = "Invalid method for RTSP/x.x request"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_load_method_3: { + switch (llhttp__internal__c_load_method(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_req_http_major; + case 3: + goto s_n_llhttp__internal__n_req_http_major; + case 6: + goto s_n_llhttp__internal__n_req_http_major; + case 35: + goto s_n_llhttp__internal__n_req_http_major; + case 36: + goto s_n_llhttp__internal__n_req_http_major; + case 37: + goto s_n_llhttp__internal__n_req_http_major; + case 38: + goto s_n_llhttp__internal__n_req_http_major; + case 39: + goto s_n_llhttp__internal__n_req_http_major; + case 40: + goto s_n_llhttp__internal__n_req_http_major; + case 41: + goto s_n_llhttp__internal__n_req_http_major; + case 42: + goto s_n_llhttp__internal__n_req_http_major; + case 43: + goto s_n_llhttp__internal__n_req_http_major; + case 44: + goto s_n_llhttp__internal__n_req_http_major; + case 45: + goto s_n_llhttp__internal__n_req_http_major; + default: + goto s_n_llhttp__internal__n_error_35; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_url_complete_1: { + switch (llhttp__on_url_complete(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_req_http_start; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_5: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_6: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_7: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_lf_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_lf_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_8: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_37: { + state->error = 0x7; + state->reason = "Invalid char in url fragment start"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_9: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_10: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_lf_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_lf_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_11: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_38: { + state->error = 0x7; + state->reason = "Invalid char in url query"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_39: { + state->error = 0x7; + state->reason = "Invalid char in url path"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_1: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_lf_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_lf_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_2: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_12: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_13: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_lf_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_lf_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_14: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_40: { + state->error = 0x7; + state->reason = "Double @ in url"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_41: { + state->error = 0x7; + state->reason = "Unexpected char in url server"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_42: { + state->error = 0x7; + state->reason = "Unexpected char in url server"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_44: { + state->error = 0x7; + state->reason = "Unexpected char in url schema"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_45: { + state->error = 0x7; + state->reason = "Unexpected char in url schema"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_46: { + state->error = 0x7; + state->reason = "Unexpected start char in url"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_is_equal_method: { + switch (llhttp__internal__c_is_equal_method(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_url_entry_normal; + default: + goto s_n_llhttp__internal__n_url_entry_connect; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_47: { + state->error = 0x6; + state->reason = "Expected space after method"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_store_method_1: { + switch (llhttp__internal__c_store_method(state, p, endp, match)) { + default: + goto s_n_llhttp__internal__n_req_first_space_before_url; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_56: { + state->error = 0x6; + state->reason = "Invalid method encountered"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_48: { + state->error = 0xd; + state->reason = "Response overflow"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_mul_add_status_code: { + switch (llhttp__internal__c_mul_add_status_code(state, p, endp, match)) { + case 1: + goto s_n_llhttp__internal__n_error_48; + default: + goto s_n_llhttp__internal__n_res_status_code; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_49: { + state->error = 0x2; + state->reason = "Expected LF after CR"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_status: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_status(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_status_1: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_status(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_res_line_almost_done; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_res_line_almost_done; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_50: { + state->error = 0xd; + state->reason = "Invalid response status"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_status_code: { + switch (llhttp__internal__c_update_status_code(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_res_status_code; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_51: { + state->error = 0x9; + state->reason = "Expected space after version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_store_http_minor_1: { + switch (llhttp__internal__c_store_http_minor(state, p, endp, match)) { + default: + goto s_n_llhttp__internal__n_res_http_end; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_52: { + state->error = 0x9; + state->reason = "Invalid minor version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_53: { + state->error = 0x9; + state->reason = "Expected dot"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_store_http_major_1: { + switch (llhttp__internal__c_store_http_major(state, p, endp, match)) { + default: + goto s_n_llhttp__internal__n_res_http_dot; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_54: { + state->error = 0x9; + state->reason = "Invalid major version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_57: { + state->error = 0x8; + state->reason = "Expected HTTP/"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_type: { + switch (llhttp__internal__c_update_type(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_req_first_space_before_url; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_store_method: { + switch (llhttp__internal__c_store_method(state, p, endp, match)) { + default: + goto s_n_llhttp__internal__n_invoke_update_type; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_55: { + state->error = 0x8; + state->reason = "Invalid word encountered"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_type_1: { + switch (llhttp__internal__c_update_type_1(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_res_http_major; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_type_2: { + switch (llhttp__internal__c_update_type(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_start_req; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_8: { + state->error = 0x15; + state->reason = "on_message_begin pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_load_type; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error: { + state->error = 0x10; + state->reason = "`on_message_begin` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_message_begin: { + switch (llhttp__on_message_begin(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_load_type; + case 21: + goto s_n_llhttp__internal__n_pause_8; + default: + goto s_n_llhttp__internal__n_error; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_finish: { + switch (llhttp__internal__c_update_finish(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__on_message_begin; + } + /* UNREACHABLE */; + abort(); + } +} + +int llhttp__internal_execute(llhttp__internal_t* state, const char* p, const char* endp) { + llparse_state_t next; + + /* check lingering errors */ + if (state->error != 0) { + return state->error; + } + + /* restart spans */ + if (state->_span_pos0 != NULL) { + state->_span_pos0 = (void*) p; + } + + next = llhttp__internal__run(state, (const unsigned char*) p, (const unsigned char*) endp); + if (next == s_error) { + return state->error; + } + state->_current = (void*) (intptr_t) next; + + /* execute spans */ + if (state->_span_pos0 != NULL) { + int error; + + error = ((llhttp__internal__span_cb) state->_span_cb0)(state, state->_span_pos0, (const char*) endp); + if (error != 0) { + state->error = error; + state->error_pos = endp; + return error; + } + } + + return 0; +} + +#else /* !LLHTTP_STRICT_MODE */ + +#include +#include +#include + +#ifdef __SSE4_2__ + #ifdef _MSC_VER + #include + #else /* !_MSC_VER */ + #include + #endif /* _MSC_VER */ +#endif /* __SSE4_2__ */ + +#ifdef _MSC_VER + #define ALIGN(n) _declspec(align(n)) +#else /* !_MSC_VER */ + #define ALIGN(n) __attribute__((aligned(n))) +#endif /* _MSC_VER */ + +#include "llhttp.h" + +typedef int (*llhttp__internal__span_cb)( + llhttp__internal_t*, const char*, const char*); + +#ifdef __SSE4_2__ +static const unsigned char ALIGN(16) llparse_blob0[] = { + 0x9, 0x9, 0xc, 0xc, '!', '"', '$', '>', '@', '~', 0x80, + 0xff, 0x0, 0x0, 0x0, 0x0 +}; +#endif /* __SSE4_2__ */ +static const unsigned char llparse_blob1[] = { + 'o', 'n' +}; +static const unsigned char llparse_blob2[] = { + 'e', 'c', 't', 'i', 'o', 'n' +}; +static const unsigned char llparse_blob3[] = { + 'l', 'o', 's', 'e' +}; +static const unsigned char llparse_blob4[] = { + 'e', 'e', 'p', '-', 'a', 'l', 'i', 'v', 'e' +}; +static const unsigned char llparse_blob5[] = { + 'p', 'g', 'r', 'a', 'd', 'e' +}; +static const unsigned char llparse_blob6[] = { + 'c', 'h', 'u', 'n', 'k', 'e', 'd' +}; +#ifdef __SSE4_2__ +static const unsigned char ALIGN(16) llparse_blob7[] = { + 0x9, 0x9, ' ', '~', 0x80, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0 +}; +#endif /* __SSE4_2__ */ +#ifdef __SSE4_2__ +static const unsigned char ALIGN(16) llparse_blob8[] = { + ' ', '!', '#', '\'', '*', '+', '-', '.', '0', '9', 'A', + 'Z', '^', 'z', '|', '|' +}; +#endif /* __SSE4_2__ */ +#ifdef __SSE4_2__ +static const unsigned char ALIGN(16) llparse_blob9[] = { + '~', '~', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0 +}; +#endif /* __SSE4_2__ */ +static const unsigned char llparse_blob10[] = { + 'e', 'n', 't', '-', 'l', 'e', 'n', 'g', 't', 'h' +}; +static const unsigned char llparse_blob11[] = { + 'r', 'o', 'x', 'y', '-', 'c', 'o', 'n', 'n', 'e', 'c', + 't', 'i', 'o', 'n' +}; +static const unsigned char llparse_blob12[] = { + 'r', 'a', 'n', 's', 'f', 'e', 'r', '-', 'e', 'n', 'c', + 'o', 'd', 'i', 'n', 'g' +}; +static const unsigned char llparse_blob13[] = { + 'p', 'g', 'r', 'a', 'd', 'e' +}; +static const unsigned char llparse_blob14[] = { + 0xd, 0xa +}; +static const unsigned char llparse_blob15[] = { + 'T', 'T', 'P', '/' +}; +static const unsigned char llparse_blob16[] = { + 0xd, 0xa, 0xd, 0xa, 'S', 'M', 0xd, 0xa, 0xd, 0xa +}; +static const unsigned char llparse_blob17[] = { + 'C', 'E', '/' +}; +static const unsigned char llparse_blob18[] = { + 'T', 'S', 'P', '/' +}; +static const unsigned char llparse_blob19[] = { + 'N', 'O', 'U', 'N', 'C', 'E' +}; +static const unsigned char llparse_blob20[] = { + 'I', 'N', 'D' +}; +static const unsigned char llparse_blob21[] = { + 'E', 'C', 'K', 'O', 'U', 'T' +}; +static const unsigned char llparse_blob22[] = { + 'N', 'E', 'C', 'T' +}; +static const unsigned char llparse_blob23[] = { + 'E', 'T', 'E' +}; +static const unsigned char llparse_blob24[] = { + 'C', 'R', 'I', 'B', 'E' +}; +static const unsigned char llparse_blob25[] = { + 'L', 'U', 'S', 'H' +}; +static const unsigned char llparse_blob26[] = { + 'E', 'T' +}; +static const unsigned char llparse_blob27[] = { + 'P', 'A', 'R', 'A', 'M', 'E', 'T', 'E', 'R' +}; +static const unsigned char llparse_blob28[] = { + 'E', 'A', 'D' +}; +static const unsigned char llparse_blob29[] = { + 'N', 'K' +}; +static const unsigned char llparse_blob30[] = { + 'C', 'K' +}; +static const unsigned char llparse_blob31[] = { + 'S', 'E', 'A', 'R', 'C', 'H' +}; +static const unsigned char llparse_blob32[] = { + 'R', 'G', 'E' +}; +static const unsigned char llparse_blob33[] = { + 'C', 'T', 'I', 'V', 'I', 'T', 'Y' +}; +static const unsigned char llparse_blob34[] = { + 'L', 'E', 'N', 'D', 'A', 'R' +}; +static const unsigned char llparse_blob35[] = { + 'V', 'E' +}; +static const unsigned char llparse_blob36[] = { + 'O', 'T', 'I', 'F', 'Y' +}; +static const unsigned char llparse_blob37[] = { + 'P', 'T', 'I', 'O', 'N', 'S' +}; +static const unsigned char llparse_blob38[] = { + 'C', 'H' +}; +static const unsigned char llparse_blob39[] = { + 'S', 'E' +}; +static const unsigned char llparse_blob40[] = { + 'A', 'Y' +}; +static const unsigned char llparse_blob41[] = { + 'S', 'T' +}; +static const unsigned char llparse_blob42[] = { + 'I', 'N', 'D' +}; +static const unsigned char llparse_blob43[] = { + 'A', 'T', 'C', 'H' +}; +static const unsigned char llparse_blob44[] = { + 'G', 'E' +}; +static const unsigned char llparse_blob45[] = { + 'I', 'N', 'D' +}; +static const unsigned char llparse_blob46[] = { + 'O', 'R', 'D' +}; +static const unsigned char llparse_blob47[] = { + 'I', 'R', 'E', 'C', 'T' +}; +static const unsigned char llparse_blob48[] = { + 'O', 'R', 'T' +}; +static const unsigned char llparse_blob49[] = { + 'R', 'C', 'H' +}; +static const unsigned char llparse_blob50[] = { + 'P', 'A', 'R', 'A', 'M', 'E', 'T', 'E', 'R' +}; +static const unsigned char llparse_blob51[] = { + 'U', 'R', 'C', 'E' +}; +static const unsigned char llparse_blob52[] = { + 'B', 'S', 'C', 'R', 'I', 'B', 'E' +}; +static const unsigned char llparse_blob53[] = { + 'A', 'R', 'D', 'O', 'W', 'N' +}; +static const unsigned char llparse_blob54[] = { + 'A', 'C', 'E' +}; +static const unsigned char llparse_blob55[] = { + 'I', 'N', 'D' +}; +static const unsigned char llparse_blob56[] = { + 'N', 'K' +}; +static const unsigned char llparse_blob57[] = { + 'C', 'K' +}; +static const unsigned char llparse_blob58[] = { + 'U', 'B', 'S', 'C', 'R', 'I', 'B', 'E' +}; +static const unsigned char llparse_blob59[] = { + 'H', 'T', 'T', 'P', '/' +}; +static const unsigned char llparse_blob60[] = { + 'A', 'D' +}; +static const unsigned char llparse_blob61[] = { + 'T', 'P', '/' +}; + +enum llparse_match_status_e { + kMatchComplete, + kMatchPause, + kMatchMismatch +}; +typedef enum llparse_match_status_e llparse_match_status_t; + +struct llparse_match_s { + llparse_match_status_t status; + const unsigned char* current; +}; +typedef struct llparse_match_s llparse_match_t; + +static llparse_match_t llparse__match_sequence_to_lower( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp, + const unsigned char* seq, uint32_t seq_len) { + uint32_t index; + llparse_match_t res; + + index = s->_index; + for (; p != endp; p++) { + unsigned char current; + + current = ((*p) >= 'A' && (*p) <= 'Z' ? (*p | 0x20) : (*p)); + if (current == seq[index]) { + if (++index == seq_len) { + res.status = kMatchComplete; + goto reset; + } + } else { + res.status = kMatchMismatch; + goto reset; + } + } + s->_index = index; + res.status = kMatchPause; + res.current = p; + return res; +reset: + s->_index = 0; + res.current = p; + return res; +} + +static llparse_match_t llparse__match_sequence_to_lower_unsafe( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp, + const unsigned char* seq, uint32_t seq_len) { + uint32_t index; + llparse_match_t res; + + index = s->_index; + for (; p != endp; p++) { + unsigned char current; + + current = ((*p) | 0x20); + if (current == seq[index]) { + if (++index == seq_len) { + res.status = kMatchComplete; + goto reset; + } + } else { + res.status = kMatchMismatch; + goto reset; + } + } + s->_index = index; + res.status = kMatchPause; + res.current = p; + return res; +reset: + s->_index = 0; + res.current = p; + return res; +} + +static llparse_match_t llparse__match_sequence_id( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp, + const unsigned char* seq, uint32_t seq_len) { + uint32_t index; + llparse_match_t res; + + index = s->_index; + for (; p != endp; p++) { + unsigned char current; + + current = *p; + if (current == seq[index]) { + if (++index == seq_len) { + res.status = kMatchComplete; + goto reset; + } + } else { + res.status = kMatchMismatch; + goto reset; + } + } + s->_index = index; + res.status = kMatchPause; + res.current = p; + return res; +reset: + s->_index = 0; + res.current = p; + return res; +} + +enum llparse_state_e { + s_error, + s_n_llhttp__internal__n_invoke_llhttp__after_message_complete, + s_n_llhttp__internal__n_pause_1, + s_n_llhttp__internal__n_invoke_is_equal_upgrade, + s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2, + s_n_llhttp__internal__n_chunk_data_almost_done_skip, + s_n_llhttp__internal__n_chunk_data_almost_done, + s_n_llhttp__internal__n_consume_content_length, + s_n_llhttp__internal__n_span_start_llhttp__on_body, + s_n_llhttp__internal__n_invoke_is_equal_content_length, + s_n_llhttp__internal__n_chunk_size_almost_done, + s_n_llhttp__internal__n_chunk_parameters, + s_n_llhttp__internal__n_chunk_size_otherwise, + s_n_llhttp__internal__n_chunk_size, + s_n_llhttp__internal__n_chunk_size_digit, + s_n_llhttp__internal__n_invoke_update_content_length, + s_n_llhttp__internal__n_consume_content_length_1, + s_n_llhttp__internal__n_span_start_llhttp__on_body_1, + s_n_llhttp__internal__n_eof, + s_n_llhttp__internal__n_span_start_llhttp__on_body_2, + s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete, + s_n_llhttp__internal__n_headers_almost_done, + s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete, + s_n_llhttp__internal__n_span_start_llhttp__on_header_value, + s_n_llhttp__internal__n_header_value_discard_lws, + s_n_llhttp__internal__n_header_value_discard_ws_almost_done, + s_n_llhttp__internal__n_header_value_lws, + s_n_llhttp__internal__n_header_value_almost_done, + s_n_llhttp__internal__n_header_value_lenient, + s_n_llhttp__internal__n_header_value_otherwise, + s_n_llhttp__internal__n_header_value_connection_token, + s_n_llhttp__internal__n_header_value_connection_ws, + s_n_llhttp__internal__n_header_value_connection_1, + s_n_llhttp__internal__n_header_value_connection_2, + s_n_llhttp__internal__n_header_value_connection_3, + s_n_llhttp__internal__n_header_value_connection, + s_n_llhttp__internal__n_error_17, + s_n_llhttp__internal__n_error_18, + s_n_llhttp__internal__n_header_value_content_length_ws, + s_n_llhttp__internal__n_header_value_content_length, + s_n_llhttp__internal__n_header_value_te_chunked_last, + s_n_llhttp__internal__n_header_value_te_token_ows, + s_n_llhttp__internal__n_header_value, + s_n_llhttp__internal__n_header_value_te_token, + s_n_llhttp__internal__n_header_value_te_chunked, + s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1, + s_n_llhttp__internal__n_header_value_discard_ws, + s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete, + s_n_llhttp__internal__n_header_field_general_otherwise, + s_n_llhttp__internal__n_header_field_general, + s_n_llhttp__internal__n_header_field_colon, + s_n_llhttp__internal__n_header_field_3, + s_n_llhttp__internal__n_header_field_4, + s_n_llhttp__internal__n_header_field_2, + s_n_llhttp__internal__n_header_field_1, + s_n_llhttp__internal__n_header_field_5, + s_n_llhttp__internal__n_header_field_6, + s_n_llhttp__internal__n_header_field_7, + s_n_llhttp__internal__n_header_field, + s_n_llhttp__internal__n_span_start_llhttp__on_header_field, + s_n_llhttp__internal__n_header_field_start, + s_n_llhttp__internal__n_url_skip_to_http09, + s_n_llhttp__internal__n_url_skip_lf_to_http09, + s_n_llhttp__internal__n_req_pri_upgrade, + s_n_llhttp__internal__n_req_http_complete_1, + s_n_llhttp__internal__n_req_http_complete, + s_n_llhttp__internal__n_req_http_minor, + s_n_llhttp__internal__n_req_http_dot, + s_n_llhttp__internal__n_req_http_major, + s_n_llhttp__internal__n_req_http_start_1, + s_n_llhttp__internal__n_req_http_start_2, + s_n_llhttp__internal__n_req_http_start_3, + s_n_llhttp__internal__n_req_http_start, + s_n_llhttp__internal__n_url_skip_to_http, + s_n_llhttp__internal__n_url_fragment, + s_n_llhttp__internal__n_span_end_stub_query_3, + s_n_llhttp__internal__n_url_query, + s_n_llhttp__internal__n_url_query_or_fragment, + s_n_llhttp__internal__n_url_path, + s_n_llhttp__internal__n_span_start_stub_path_2, + s_n_llhttp__internal__n_span_start_stub_path, + s_n_llhttp__internal__n_span_start_stub_path_1, + s_n_llhttp__internal__n_url_server_with_at, + s_n_llhttp__internal__n_url_server, + s_n_llhttp__internal__n_url_schema_delim_1, + s_n_llhttp__internal__n_url_schema_delim, + s_n_llhttp__internal__n_span_end_stub_schema, + s_n_llhttp__internal__n_url_schema, + s_n_llhttp__internal__n_url_start, + s_n_llhttp__internal__n_span_start_llhttp__on_url_1, + s_n_llhttp__internal__n_span_start_llhttp__on_url, + s_n_llhttp__internal__n_req_spaces_before_url, + s_n_llhttp__internal__n_req_first_space_before_url, + s_n_llhttp__internal__n_start_req_2, + s_n_llhttp__internal__n_start_req_3, + s_n_llhttp__internal__n_start_req_1, + s_n_llhttp__internal__n_start_req_4, + s_n_llhttp__internal__n_start_req_6, + s_n_llhttp__internal__n_start_req_8, + s_n_llhttp__internal__n_start_req_9, + s_n_llhttp__internal__n_start_req_7, + s_n_llhttp__internal__n_start_req_5, + s_n_llhttp__internal__n_start_req_12, + s_n_llhttp__internal__n_start_req_13, + s_n_llhttp__internal__n_start_req_11, + s_n_llhttp__internal__n_start_req_10, + s_n_llhttp__internal__n_start_req_14, + s_n_llhttp__internal__n_start_req_17, + s_n_llhttp__internal__n_start_req_16, + s_n_llhttp__internal__n_start_req_15, + s_n_llhttp__internal__n_start_req_18, + s_n_llhttp__internal__n_start_req_20, + s_n_llhttp__internal__n_start_req_21, + s_n_llhttp__internal__n_start_req_19, + s_n_llhttp__internal__n_start_req_23, + s_n_llhttp__internal__n_start_req_24, + s_n_llhttp__internal__n_start_req_26, + s_n_llhttp__internal__n_start_req_28, + s_n_llhttp__internal__n_start_req_29, + s_n_llhttp__internal__n_start_req_27, + s_n_llhttp__internal__n_start_req_25, + s_n_llhttp__internal__n_start_req_30, + s_n_llhttp__internal__n_start_req_22, + s_n_llhttp__internal__n_start_req_31, + s_n_llhttp__internal__n_start_req_32, + s_n_llhttp__internal__n_start_req_35, + s_n_llhttp__internal__n_start_req_36, + s_n_llhttp__internal__n_start_req_34, + s_n_llhttp__internal__n_start_req_37, + s_n_llhttp__internal__n_start_req_38, + s_n_llhttp__internal__n_start_req_42, + s_n_llhttp__internal__n_start_req_43, + s_n_llhttp__internal__n_start_req_41, + s_n_llhttp__internal__n_start_req_40, + s_n_llhttp__internal__n_start_req_39, + s_n_llhttp__internal__n_start_req_45, + s_n_llhttp__internal__n_start_req_44, + s_n_llhttp__internal__n_start_req_33, + s_n_llhttp__internal__n_start_req_48, + s_n_llhttp__internal__n_start_req_49, + s_n_llhttp__internal__n_start_req_50, + s_n_llhttp__internal__n_start_req_51, + s_n_llhttp__internal__n_start_req_47, + s_n_llhttp__internal__n_start_req_46, + s_n_llhttp__internal__n_start_req_54, + s_n_llhttp__internal__n_start_req_56, + s_n_llhttp__internal__n_start_req_57, + s_n_llhttp__internal__n_start_req_55, + s_n_llhttp__internal__n_start_req_53, + s_n_llhttp__internal__n_start_req_58, + s_n_llhttp__internal__n_start_req_59, + s_n_llhttp__internal__n_start_req_52, + s_n_llhttp__internal__n_start_req_61, + s_n_llhttp__internal__n_start_req_62, + s_n_llhttp__internal__n_start_req_60, + s_n_llhttp__internal__n_start_req_65, + s_n_llhttp__internal__n_start_req_67, + s_n_llhttp__internal__n_start_req_68, + s_n_llhttp__internal__n_start_req_66, + s_n_llhttp__internal__n_start_req_69, + s_n_llhttp__internal__n_start_req_64, + s_n_llhttp__internal__n_start_req_63, + s_n_llhttp__internal__n_start_req, + s_n_llhttp__internal__n_invoke_llhttp__on_status_complete, + s_n_llhttp__internal__n_res_line_almost_done, + s_n_llhttp__internal__n_res_status, + s_n_llhttp__internal__n_span_start_llhttp__on_status, + s_n_llhttp__internal__n_res_status_start, + s_n_llhttp__internal__n_res_status_code_otherwise, + s_n_llhttp__internal__n_res_status_code, + s_n_llhttp__internal__n_res_http_end, + s_n_llhttp__internal__n_res_http_minor, + s_n_llhttp__internal__n_res_http_dot, + s_n_llhttp__internal__n_res_http_major, + s_n_llhttp__internal__n_start_res, + s_n_llhttp__internal__n_req_or_res_method_2, + s_n_llhttp__internal__n_req_or_res_method_3, + s_n_llhttp__internal__n_req_or_res_method_1, + s_n_llhttp__internal__n_req_or_res_method, + s_n_llhttp__internal__n_start_req_or_res, + s_n_llhttp__internal__n_invoke_load_type, + s_n_llhttp__internal__n_start, +}; +typedef enum llparse_state_e llparse_state_t; + +int llhttp__on_url( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__on_header_field( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__on_header_value( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__on_body( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__on_status( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_update_finish( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->finish = 2; + return 0; +} + +int llhttp__on_message_begin( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_load_type( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return state->type; +} + +int llhttp__internal__c_store_method( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp, + int match) { + state->method = match; + return 0; +} + +int llhttp__internal__c_is_equal_method( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return state->method == 5; +} + +int llhttp__internal__c_update_http_major( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->http_major = 0; + return 0; +} + +int llhttp__internal__c_update_http_minor( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->http_minor = 9; + return 0; +} + +int llhttp__on_url_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_test_flags( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->flags & 128) == 128; +} + +int llhttp__on_chunk_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__on_message_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_is_equal_upgrade( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return state->upgrade == 1; +} + +int llhttp__after_message_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_update_finish_1( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->finish = 0; + return 0; +} + +int llhttp__internal__c_test_flags_1( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->flags & 544) == 544; +} + +int llhttp__internal__c_test_lenient_flags( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->lenient_flags & 2) == 2; +} + +int llhttp__before_headers_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__on_headers_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__after_headers_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_update_content_length( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->content_length = 0; + return 0; +} + +int llhttp__internal__c_mul_add_content_length( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp, + int match) { + /* Multiplication overflow */ + if (state->content_length > 0xffffffffffffffffULL / 16) { + return 1; + } + + state->content_length *= 16; + + /* Addition overflow */ + if (match >= 0) { + if (state->content_length > 0xffffffffffffffffULL - match) { + return 1; + } + } else { + if (state->content_length < 0ULL - match) { + return 1; + } + } + state->content_length += match; + return 0; +} + +int llhttp__on_chunk_header( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_is_equal_content_length( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return state->content_length == 0; +} + +int llhttp__internal__c_or_flags( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags |= 128; + return 0; +} + +int llhttp__internal__c_update_finish_2( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->finish = 1; + return 0; +} + +int llhttp__internal__c_or_flags_1( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags |= 64; + return 0; +} + +int llhttp__internal__c_update_upgrade( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->upgrade = 1; + return 0; +} + +int llhttp__internal__c_store_header_state( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp, + int match) { + state->header_state = match; + return 0; +} + +int llhttp__on_header_field_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_load_header_state( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return state->header_state; +} + +int llhttp__internal__c_or_flags_3( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags |= 1; + return 0; +} + +int llhttp__internal__c_update_header_state( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->header_state = 1; + return 0; +} + +int llhttp__on_header_value_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_or_flags_4( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags |= 2; + return 0; +} + +int llhttp__internal__c_or_flags_5( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags |= 4; + return 0; +} + +int llhttp__internal__c_or_flags_6( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags |= 8; + return 0; +} + +int llhttp__internal__c_update_header_state_2( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->header_state = 6; + return 0; +} + +int llhttp__internal__c_test_lenient_flags_1( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->lenient_flags & 1) == 1; +} + +int llhttp__internal__c_update_header_state_4( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->header_state = 0; + return 0; +} + +int llhttp__internal__c_update_header_state_5( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->header_state = 5; + return 0; +} + +int llhttp__internal__c_update_header_state_6( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->header_state = 7; + return 0; +} + +int llhttp__internal__c_test_flags_2( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->flags & 32) == 32; +} + +int llhttp__internal__c_mul_add_content_length_1( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp, + int match) { + /* Multiplication overflow */ + if (state->content_length > 0xffffffffffffffffULL / 10) { + return 1; + } + + state->content_length *= 10; + + /* Addition overflow */ + if (match >= 0) { + if (state->content_length > 0xffffffffffffffffULL - match) { + return 1; + } + } else { + if (state->content_length < 0ULL - match) { + return 1; + } + } + state->content_length += match; + return 0; +} + +int llhttp__internal__c_or_flags_15( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags |= 32; + return 0; +} + +int llhttp__internal__c_or_flags_16( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags |= 512; + return 0; +} + +int llhttp__internal__c_and_flags( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags &= -9; + return 0; +} + +int llhttp__internal__c_update_header_state_7( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->header_state = 8; + return 0; +} + +int llhttp__internal__c_or_flags_17( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->flags |= 16; + return 0; +} + +int llhttp__internal__c_load_method( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return state->method; +} + +int llhttp__internal__c_store_http_major( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp, + int match) { + state->http_major = match; + return 0; +} + +int llhttp__internal__c_store_http_minor( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp, + int match) { + state->http_minor = match; + return 0; +} + +int llhttp__internal__c_update_status_code( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->status_code = 0; + return 0; +} + +int llhttp__internal__c_mul_add_status_code( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp, + int match) { + /* Multiplication overflow */ + if (state->status_code > 0xffff / 10) { + return 1; + } + + state->status_code *= 10; + + /* Addition overflow */ + if (match >= 0) { + if (state->status_code > 0xffff - match) { + return 1; + } + } else { + if (state->status_code < 0 - match) { + return 1; + } + } + state->status_code += match; + + /* Enforce maximum */ + if (state->status_code > 999) { + return 1; + } + return 0; +} + +int llhttp__on_status_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_update_type( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->type = 1; + return 0; +} + +int llhttp__internal__c_update_type_1( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->type = 2; + return 0; +} + +int llhttp__internal_init(llhttp__internal_t* state) { + memset(state, 0, sizeof(*state)); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_start; + return 0; +} + +static llparse_state_t llhttp__internal__run( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + int match; + switch ((llparse_state_t) (intptr_t) state->_current) { + case s_n_llhttp__internal__n_invoke_llhttp__after_message_complete: + s_n_llhttp__internal__n_invoke_llhttp__after_message_complete: { + switch (llhttp__after_message_complete(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_finish_1; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_pause_1: + s_n_llhttp__internal__n_pause_1: { + state->error = 0x16; + state->reason = "Pause on CONNECT/Upgrade"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__after_message_complete; + return s_error; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_is_equal_upgrade: + s_n_llhttp__internal__n_invoke_is_equal_upgrade: { + switch (llhttp__internal__c_is_equal_upgrade(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_llhttp__after_message_complete; + default: + goto s_n_llhttp__internal__n_pause_1; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2: + s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2: { + switch (llhttp__on_message_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_is_equal_upgrade; + case 21: + goto s_n_llhttp__internal__n_pause_5; + default: + goto s_n_llhttp__internal__n_error_9; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_chunk_data_almost_done_skip: + s_n_llhttp__internal__n_chunk_data_almost_done_skip: { + if (p == endp) { + return s_n_llhttp__internal__n_chunk_data_almost_done_skip; + } + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_chunk_data_almost_done: + s_n_llhttp__internal__n_chunk_data_almost_done: { + if (p == endp) { + return s_n_llhttp__internal__n_chunk_data_almost_done; + } + p++; + goto s_n_llhttp__internal__n_chunk_data_almost_done_skip; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_consume_content_length: + s_n_llhttp__internal__n_consume_content_length: { + size_t avail; + size_t need; + + avail = endp - p; + need = state->content_length; + if (avail >= need) { + p += need; + state->content_length = 0; + goto s_n_llhttp__internal__n_span_end_llhttp__on_body; + } + + state->content_length -= avail; + return s_n_llhttp__internal__n_consume_content_length; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_llhttp__on_body: + s_n_llhttp__internal__n_span_start_llhttp__on_body: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_body; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_body; + goto s_n_llhttp__internal__n_consume_content_length; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_is_equal_content_length: + s_n_llhttp__internal__n_invoke_is_equal_content_length: { + switch (llhttp__internal__c_is_equal_content_length(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_span_start_llhttp__on_body; + default: + goto s_n_llhttp__internal__n_invoke_or_flags; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_chunk_size_almost_done: + s_n_llhttp__internal__n_chunk_size_almost_done: { + if (p == endp) { + return s_n_llhttp__internal__n_chunk_size_almost_done; + } + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_header; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_chunk_parameters: + s_n_llhttp__internal__n_chunk_parameters: { + if (p == endp) { + return s_n_llhttp__internal__n_chunk_parameters; + } + switch (*p) { + case 13: { + p++; + goto s_n_llhttp__internal__n_chunk_size_almost_done; + } + default: { + p++; + goto s_n_llhttp__internal__n_chunk_parameters; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_chunk_size_otherwise: + s_n_llhttp__internal__n_chunk_size_otherwise: { + if (p == endp) { + return s_n_llhttp__internal__n_chunk_size_otherwise; + } + switch (*p) { + case 13: { + p++; + goto s_n_llhttp__internal__n_chunk_size_almost_done; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_chunk_parameters; + } + case ';': { + p++; + goto s_n_llhttp__internal__n_chunk_parameters; + } + default: { + goto s_n_llhttp__internal__n_error_6; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_chunk_size: + s_n_llhttp__internal__n_chunk_size: { + if (p == endp) { + return s_n_llhttp__internal__n_chunk_size; + } + switch (*p) { + case '0': { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '1': { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '2': { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '3': { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '4': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '5': { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '6': { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '7': { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '8': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '9': { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'A': { + p++; + match = 10; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'B': { + p++; + match = 11; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'C': { + p++; + match = 12; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'D': { + p++; + match = 13; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'E': { + p++; + match = 14; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'F': { + p++; + match = 15; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'a': { + p++; + match = 10; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'b': { + p++; + match = 11; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'c': { + p++; + match = 12; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'd': { + p++; + match = 13; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'e': { + p++; + match = 14; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'f': { + p++; + match = 15; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + default: { + goto s_n_llhttp__internal__n_chunk_size_otherwise; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_chunk_size_digit: + s_n_llhttp__internal__n_chunk_size_digit: { + if (p == endp) { + return s_n_llhttp__internal__n_chunk_size_digit; + } + switch (*p) { + case '0': { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '1': { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '2': { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '3': { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '4': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '5': { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '6': { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '7': { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '8': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case '9': { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'A': { + p++; + match = 10; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'B': { + p++; + match = 11; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'C': { + p++; + match = 12; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'D': { + p++; + match = 13; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'E': { + p++; + match = 14; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'F': { + p++; + match = 15; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'a': { + p++; + match = 10; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'b': { + p++; + match = 11; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'c': { + p++; + match = 12; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'd': { + p++; + match = 13; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'e': { + p++; + match = 14; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + case 'f': { + p++; + match = 15; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length; + } + default: { + goto s_n_llhttp__internal__n_error_8; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_update_content_length: + s_n_llhttp__internal__n_invoke_update_content_length: { + switch (llhttp__internal__c_update_content_length(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_chunk_size_digit; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_consume_content_length_1: + s_n_llhttp__internal__n_consume_content_length_1: { + size_t avail; + size_t need; + + avail = endp - p; + need = state->content_length; + if (avail >= need) { + p += need; + state->content_length = 0; + goto s_n_llhttp__internal__n_span_end_llhttp__on_body_1; + } + + state->content_length -= avail; + return s_n_llhttp__internal__n_consume_content_length_1; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_llhttp__on_body_1: + s_n_llhttp__internal__n_span_start_llhttp__on_body_1: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_body_1; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_body; + goto s_n_llhttp__internal__n_consume_content_length_1; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_eof: + s_n_llhttp__internal__n_eof: { + if (p == endp) { + return s_n_llhttp__internal__n_eof; + } + p++; + goto s_n_llhttp__internal__n_eof; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_llhttp__on_body_2: + s_n_llhttp__internal__n_span_start_llhttp__on_body_2: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_body_2; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_body; + goto s_n_llhttp__internal__n_eof; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete: + s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete: { + switch (llhttp__after_headers_complete(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_1; + case 2: + goto s_n_llhttp__internal__n_invoke_update_content_length; + case 3: + goto s_n_llhttp__internal__n_span_start_llhttp__on_body_1; + case 4: + goto s_n_llhttp__internal__n_invoke_update_finish_2; + case 5: + goto s_n_llhttp__internal__n_error_10; + default: + goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_headers_almost_done: + s_n_llhttp__internal__n_headers_almost_done: { + if (p == endp) { + return s_n_llhttp__internal__n_headers_almost_done; + } + p++; + goto s_n_llhttp__internal__n_invoke_test_flags; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete: + s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete: { + switch (llhttp__on_header_value_complete(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_field_start; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_llhttp__on_header_value: + s_n_llhttp__internal__n_span_start_llhttp__on_header_value: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_header_value; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_header_value; + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_discard_lws: + s_n_llhttp__internal__n_header_value_discard_lws: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_discard_lws; + } + switch (*p) { + case 9: { + p++; + goto s_n_llhttp__internal__n_header_value_discard_ws; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_header_value_discard_ws; + } + default: { + goto s_n_llhttp__internal__n_invoke_load_header_state; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_discard_ws_almost_done: + s_n_llhttp__internal__n_header_value_discard_ws_almost_done: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_discard_ws_almost_done; + } + p++; + goto s_n_llhttp__internal__n_header_value_discard_lws; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_lws: + s_n_llhttp__internal__n_header_value_lws: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_lws; + } + switch (*p) { + case 9: { + goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1; + } + case ' ': { + goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1; + } + default: { + goto s_n_llhttp__internal__n_invoke_load_header_state_3; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_almost_done: + s_n_llhttp__internal__n_header_value_almost_done: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_almost_done; + } + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_header_value_lws; + } + default: { + goto s_n_llhttp__internal__n_error_14; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_lenient: + s_n_llhttp__internal__n_header_value_lenient: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_lenient; + } + switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_1; + } + case 13: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_3; + } + default: { + p++; + goto s_n_llhttp__internal__n_header_value_lenient; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_otherwise: + s_n_llhttp__internal__n_header_value_otherwise: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_otherwise; + } + switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_1; + } + case 13: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_2; + } + default: { + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_1; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_connection_token: + s_n_llhttp__internal__n_header_value_connection_token: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + if (p == endp) { + return s_n_llhttp__internal__n_header_value_connection_token; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_header_value_connection_token; + } + case 2: { + p++; + goto s_n_llhttp__internal__n_header_value_connection; + } + default: { + goto s_n_llhttp__internal__n_header_value_otherwise; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_connection_ws: + s_n_llhttp__internal__n_header_value_connection_ws: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_connection_ws; + } + switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_header_value_otherwise; + } + case 13: { + goto s_n_llhttp__internal__n_header_value_otherwise; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_header_value_connection_ws; + } + case ',': { + p++; + goto s_n_llhttp__internal__n_invoke_load_header_state_4; + } + default: { + goto s_n_llhttp__internal__n_invoke_update_header_state_4; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_connection_1: + s_n_llhttp__internal__n_header_value_connection_1: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_value_connection_1; + } + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob3, 4); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_invoke_update_header_state_2; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_value_connection_1; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_header_value_connection_token; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_connection_2: + s_n_llhttp__internal__n_header_value_connection_2: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_value_connection_2; + } + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob4, 9); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_invoke_update_header_state_5; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_value_connection_2; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_header_value_connection_token; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_connection_3: + s_n_llhttp__internal__n_header_value_connection_3: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_value_connection_3; + } + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob5, 6); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_invoke_update_header_state_6; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_value_connection_3; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_header_value_connection_token; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_connection: + s_n_llhttp__internal__n_header_value_connection: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_connection; + } + switch (((*p) >= 'A' && (*p) <= 'Z' ? (*p | 0x20) : (*p))) { + case 9: { + p++; + goto s_n_llhttp__internal__n_header_value_connection; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_header_value_connection; + } + case 'c': { + p++; + goto s_n_llhttp__internal__n_header_value_connection_1; + } + case 'k': { + p++; + goto s_n_llhttp__internal__n_header_value_connection_2; + } + case 'u': { + p++; + goto s_n_llhttp__internal__n_header_value_connection_3; + } + default: { + goto s_n_llhttp__internal__n_header_value_connection_token; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_error_17: + s_n_llhttp__internal__n_error_17: { + state->error = 0xb; + state->reason = "Content-Length overflow"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_error_18: + s_n_llhttp__internal__n_error_18: { + state->error = 0xb; + state->reason = "Invalid character in Content-Length"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_content_length_ws: + s_n_llhttp__internal__n_header_value_content_length_ws: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_content_length_ws; + } + switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_invoke_or_flags_15; + } + case 13: { + goto s_n_llhttp__internal__n_invoke_or_flags_15; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_header_value_content_length_ws; + } + default: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_5; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_content_length: + s_n_llhttp__internal__n_header_value_content_length: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_content_length; + } + switch (*p) { + case '0': { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + case '1': { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + case '2': { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + case '3': { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + case '4': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + case '5': { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + case '6': { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + case '7': { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + case '8': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + case '9': { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; + } + default: { + goto s_n_llhttp__internal__n_header_value_content_length_ws; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_te_chunked_last: + s_n_llhttp__internal__n_header_value_te_chunked_last: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_te_chunked_last; + } + switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_invoke_update_header_state_7; + } + case 13: { + goto s_n_llhttp__internal__n_invoke_update_header_state_7; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_header_value_te_chunked_last; + } + default: { + goto s_n_llhttp__internal__n_header_value_te_chunked; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_te_token_ows: + s_n_llhttp__internal__n_header_value_te_token_ows: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_te_token_ows; + } + switch (*p) { + case 9: { + p++; + goto s_n_llhttp__internal__n_header_value_te_token_ows; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_header_value_te_token_ows; + } + default: { + goto s_n_llhttp__internal__n_header_value_te_chunked; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value: + s_n_llhttp__internal__n_header_value: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + if (p == endp) { + return s_n_llhttp__internal__n_header_value; + } + #ifdef __SSE4_2__ + if (endp - p >= 16) { + __m128i ranges; + __m128i input; + int avail; + int match_len; + + /* Load input */ + input = _mm_loadu_si128((__m128i const*) p); + ranges = _mm_loadu_si128((__m128i const*) llparse_blob7); + + /* Find first character that does not match `ranges` */ + match_len = _mm_cmpestri(ranges, 6, + input, 16, + _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES | + _SIDD_NEGATIVE_POLARITY); + + if (match_len != 0) { + p += match_len; + goto s_n_llhttp__internal__n_header_value; + } + goto s_n_llhttp__internal__n_header_value_otherwise; + } + #endif /* __SSE4_2__ */ + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_header_value; + } + default: { + goto s_n_llhttp__internal__n_header_value_otherwise; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_te_token: + s_n_llhttp__internal__n_header_value_te_token: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + if (p == endp) { + return s_n_llhttp__internal__n_header_value_te_token; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_header_value_te_token; + } + case 2: { + p++; + goto s_n_llhttp__internal__n_header_value_te_token_ows; + } + default: { + goto s_n_llhttp__internal__n_invoke_update_header_state_8; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_te_chunked: + s_n_llhttp__internal__n_header_value_te_chunked: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_value_te_chunked; + } + match_seq = llparse__match_sequence_to_lower_unsafe(state, p, endp, llparse_blob6, 7); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_header_value_te_chunked_last; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_value_te_chunked; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_header_value_te_token; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1: + s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_header_value; + goto s_n_llhttp__internal__n_invoke_load_header_state_2; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_value_discard_ws: + s_n_llhttp__internal__n_header_value_discard_ws: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_discard_ws; + } + switch (*p) { + case 9: { + p++; + goto s_n_llhttp__internal__n_header_value_discard_ws; + } + case 10: { + p++; + goto s_n_llhttp__internal__n_header_value_discard_lws; + } + case 13: { + p++; + goto s_n_llhttp__internal__n_header_value_discard_ws_almost_done; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_header_value_discard_ws; + } + default: { + goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete: + s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete: { + switch (llhttp__on_header_field_complete(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_discard_ws; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_general_otherwise: + s_n_llhttp__internal__n_header_field_general_otherwise: { + if (p == endp) { + return s_n_llhttp__internal__n_header_field_general_otherwise; + } + switch (*p) { + case ':': { + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_field_1; + } + default: { + goto s_n_llhttp__internal__n_error_19; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_general: + s_n_llhttp__internal__n_header_field_general: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + if (p == endp) { + return s_n_llhttp__internal__n_header_field_general; + } + #ifdef __SSE4_2__ + if (endp - p >= 16) { + __m128i ranges; + __m128i input; + int avail; + int match_len; + + /* Load input */ + input = _mm_loadu_si128((__m128i const*) p); + ranges = _mm_loadu_si128((__m128i const*) llparse_blob8); + + /* Find first character that does not match `ranges` */ + match_len = _mm_cmpestri(ranges, 16, + input, 16, + _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES | + _SIDD_NEGATIVE_POLARITY); + + if (match_len != 0) { + p += match_len; + goto s_n_llhttp__internal__n_header_field_general; + } + ranges = _mm_loadu_si128((__m128i const*) llparse_blob9); + + /* Find first character that does not match `ranges` */ + match_len = _mm_cmpestri(ranges, 2, + input, 16, + _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES | + _SIDD_NEGATIVE_POLARITY); + + if (match_len != 0) { + p += match_len; + goto s_n_llhttp__internal__n_header_field_general; + } + goto s_n_llhttp__internal__n_header_field_general_otherwise; + } + #endif /* __SSE4_2__ */ + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_header_field_general; + } + default: { + goto s_n_llhttp__internal__n_header_field_general_otherwise; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_colon: + s_n_llhttp__internal__n_header_field_colon: { + if (p == endp) { + return s_n_llhttp__internal__n_header_field_colon; + } + switch (*p) { + case ' ': { + p++; + goto s_n_llhttp__internal__n_header_field_colon; + } + case ':': { + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_field; + } + default: { + goto s_n_llhttp__internal__n_invoke_update_header_state_9; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_3: + s_n_llhttp__internal__n_header_field_3: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_field_3; + } + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob2, 6); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_store_header_state; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_field_3; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_invoke_update_header_state_10; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_4: + s_n_llhttp__internal__n_header_field_4: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_field_4; + } + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob10, 10); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_store_header_state; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_field_4; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_invoke_update_header_state_10; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_2: + s_n_llhttp__internal__n_header_field_2: { + if (p == endp) { + return s_n_llhttp__internal__n_header_field_2; + } + switch (((*p) >= 'A' && (*p) <= 'Z' ? (*p | 0x20) : (*p))) { + case 'n': { + p++; + goto s_n_llhttp__internal__n_header_field_3; + } + case 't': { + p++; + goto s_n_llhttp__internal__n_header_field_4; + } + default: { + goto s_n_llhttp__internal__n_invoke_update_header_state_10; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_1: + s_n_llhttp__internal__n_header_field_1: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_field_1; + } + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob1, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_header_field_2; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_field_1; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_invoke_update_header_state_10; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_5: + s_n_llhttp__internal__n_header_field_5: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_field_5; + } + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob11, 15); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_store_header_state; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_field_5; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_invoke_update_header_state_10; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_6: + s_n_llhttp__internal__n_header_field_6: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_field_6; + } + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob12, 16); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_store_header_state; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_field_6; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_invoke_update_header_state_10; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_7: + s_n_llhttp__internal__n_header_field_7: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_header_field_7; + } + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob13, 6); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_store_header_state; + } + case kMatchPause: { + return s_n_llhttp__internal__n_header_field_7; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_invoke_update_header_state_10; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field: + s_n_llhttp__internal__n_header_field: { + if (p == endp) { + return s_n_llhttp__internal__n_header_field; + } + switch (((*p) >= 'A' && (*p) <= 'Z' ? (*p | 0x20) : (*p))) { + case 'c': { + p++; + goto s_n_llhttp__internal__n_header_field_1; + } + case 'p': { + p++; + goto s_n_llhttp__internal__n_header_field_5; + } + case 't': { + p++; + goto s_n_llhttp__internal__n_header_field_6; + } + case 'u': { + p++; + goto s_n_llhttp__internal__n_header_field_7; + } + default: { + goto s_n_llhttp__internal__n_invoke_update_header_state_10; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_llhttp__on_header_field: + s_n_llhttp__internal__n_span_start_llhttp__on_header_field: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_header_field; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_header_field; + goto s_n_llhttp__internal__n_header_field; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_header_field_start: + s_n_llhttp__internal__n_header_field_start: { + if (p == endp) { + return s_n_llhttp__internal__n_header_field_start; + } + switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_headers_almost_done; + } + case 13: { + p++; + goto s_n_llhttp__internal__n_headers_almost_done; + } + default: { + goto s_n_llhttp__internal__n_span_start_llhttp__on_header_field; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_skip_to_http09: + s_n_llhttp__internal__n_url_skip_to_http09: { + if (p == endp) { + return s_n_llhttp__internal__n_url_skip_to_http09; + } + p++; + goto s_n_llhttp__internal__n_invoke_update_http_major; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_skip_lf_to_http09: + s_n_llhttp__internal__n_url_skip_lf_to_http09: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_url_skip_lf_to_http09; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob14, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_invoke_update_http_major; + } + case kMatchPause: { + return s_n_llhttp__internal__n_url_skip_lf_to_http09; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_20; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_pri_upgrade: + s_n_llhttp__internal__n_req_pri_upgrade: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_req_pri_upgrade; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob16, 10); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_error_23; + } + case kMatchPause: { + return s_n_llhttp__internal__n_req_pri_upgrade; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_24; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_http_complete_1: + s_n_llhttp__internal__n_req_http_complete_1: { + if (p == endp) { + return s_n_llhttp__internal__n_req_http_complete_1; + } + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_header_field_start; + } + default: { + goto s_n_llhttp__internal__n_error_22; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_http_complete: + s_n_llhttp__internal__n_req_http_complete: { + if (p == endp) { + return s_n_llhttp__internal__n_req_http_complete; + } + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_header_field_start; + } + case 13: { + p++; + goto s_n_llhttp__internal__n_req_http_complete_1; + } + default: { + goto s_n_llhttp__internal__n_error_22; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_http_minor: + s_n_llhttp__internal__n_req_http_minor: { + if (p == endp) { + return s_n_llhttp__internal__n_req_http_minor; + } + switch (*p) { + case '0': { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + case '1': { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + case '2': { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + case '3': { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + case '4': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + case '5': { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + case '6': { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + case '7': { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + case '8': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + case '9': { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_store_http_minor; + } + default: { + goto s_n_llhttp__internal__n_error_25; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_http_dot: + s_n_llhttp__internal__n_req_http_dot: { + if (p == endp) { + return s_n_llhttp__internal__n_req_http_dot; + } + switch (*p) { + case '.': { + p++; + goto s_n_llhttp__internal__n_req_http_minor; + } + default: { + goto s_n_llhttp__internal__n_error_26; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_http_major: + s_n_llhttp__internal__n_req_http_major: { + if (p == endp) { + return s_n_llhttp__internal__n_req_http_major; + } + switch (*p) { + case '0': { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + case '1': { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + case '2': { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + case '3': { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + case '4': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + case '5': { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + case '6': { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + case '7': { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + case '8': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + case '9': { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_store_http_major; + } + default: { + goto s_n_llhttp__internal__n_error_27; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_http_start_1: + s_n_llhttp__internal__n_req_http_start_1: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_req_http_start_1; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob15, 4); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_invoke_load_method; + } + case kMatchPause: { + return s_n_llhttp__internal__n_req_http_start_1; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_30; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_http_start_2: + s_n_llhttp__internal__n_req_http_start_2: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_req_http_start_2; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob17, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_invoke_load_method_2; + } + case kMatchPause: { + return s_n_llhttp__internal__n_req_http_start_2; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_30; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_http_start_3: + s_n_llhttp__internal__n_req_http_start_3: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_req_http_start_3; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob18, 4); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_invoke_load_method_3; + } + case kMatchPause: { + return s_n_llhttp__internal__n_req_http_start_3; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_30; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_http_start: + s_n_llhttp__internal__n_req_http_start: { + if (p == endp) { + return s_n_llhttp__internal__n_req_http_start; + } + switch (*p) { + case ' ': { + p++; + goto s_n_llhttp__internal__n_req_http_start; + } + case 'H': { + p++; + goto s_n_llhttp__internal__n_req_http_start_1; + } + case 'I': { + p++; + goto s_n_llhttp__internal__n_req_http_start_2; + } + case 'R': { + p++; + goto s_n_llhttp__internal__n_req_http_start_3; + } + default: { + goto s_n_llhttp__internal__n_error_30; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_skip_to_http: + s_n_llhttp__internal__n_url_skip_to_http: { + if (p == endp) { + return s_n_llhttp__internal__n_url_skip_to_http; + } + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_url_complete_1; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_fragment: + s_n_llhttp__internal__n_url_fragment: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 3, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + if (p == endp) { + return s_n_llhttp__internal__n_url_fragment; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_url_fragment; + } + case 2: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_6; + } + case 3: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_7; + } + case 4: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_8; + } + default: { + goto s_n_llhttp__internal__n_error_31; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_end_stub_query_3: + s_n_llhttp__internal__n_span_end_stub_query_3: { + if (p == endp) { + return s_n_llhttp__internal__n_span_end_stub_query_3; + } + p++; + goto s_n_llhttp__internal__n_url_fragment; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_query: + s_n_llhttp__internal__n_url_query: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 3, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 1, 1, 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + if (p == endp) { + return s_n_llhttp__internal__n_url_query; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_url_query; + } + case 2: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_9; + } + case 3: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_10; + } + case 4: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_11; + } + case 5: { + goto s_n_llhttp__internal__n_span_end_stub_query_3; + } + default: { + goto s_n_llhttp__internal__n_error_32; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_query_or_fragment: + s_n_llhttp__internal__n_url_query_or_fragment: { + if (p == endp) { + return s_n_llhttp__internal__n_url_query_or_fragment; + } + switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_3; + } + case 13: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_4; + } + case ' ': { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_5; + } + case '#': { + p++; + goto s_n_llhttp__internal__n_url_fragment; + } + case '?': { + p++; + goto s_n_llhttp__internal__n_url_query; + } + default: { + goto s_n_llhttp__internal__n_error_33; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_path: + s_n_llhttp__internal__n_url_path: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + if (p == endp) { + return s_n_llhttp__internal__n_url_path; + } + #ifdef __SSE4_2__ + if (endp - p >= 16) { + __m128i ranges; + __m128i input; + int avail; + int match_len; + + /* Load input */ + input = _mm_loadu_si128((__m128i const*) p); + ranges = _mm_loadu_si128((__m128i const*) llparse_blob0); + + /* Find first character that does not match `ranges` */ + match_len = _mm_cmpestri(ranges, 12, + input, 16, + _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES | + _SIDD_NEGATIVE_POLARITY); + + if (match_len != 0) { + p += match_len; + goto s_n_llhttp__internal__n_url_path; + } + goto s_n_llhttp__internal__n_url_query_or_fragment; + } + #endif /* __SSE4_2__ */ + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_url_path; + } + default: { + goto s_n_llhttp__internal__n_url_query_or_fragment; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_stub_path_2: + s_n_llhttp__internal__n_span_start_stub_path_2: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_stub_path_2; + } + p++; + goto s_n_llhttp__internal__n_url_path; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_stub_path: + s_n_llhttp__internal__n_span_start_stub_path: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_stub_path; + } + p++; + goto s_n_llhttp__internal__n_url_path; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_stub_path_1: + s_n_llhttp__internal__n_span_start_stub_path_1: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_stub_path_1; + } + p++; + goto s_n_llhttp__internal__n_url_path; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_server_with_at: + s_n_llhttp__internal__n_url_server_with_at: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 3, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 0, 6, + 7, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 0, 4, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + if (p == endp) { + return s_n_llhttp__internal__n_url_server_with_at; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_12; + } + case 2: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_13; + } + case 3: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_14; + } + case 4: { + p++; + goto s_n_llhttp__internal__n_url_server; + } + case 5: { + goto s_n_llhttp__internal__n_span_start_stub_path_1; + } + case 6: { + p++; + goto s_n_llhttp__internal__n_url_query; + } + case 7: { + p++; + goto s_n_llhttp__internal__n_error_34; + } + default: { + goto s_n_llhttp__internal__n_error_35; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_server: + s_n_llhttp__internal__n_url_server: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 3, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 0, 6, + 7, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 0, 4, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + if (p == endp) { + return s_n_llhttp__internal__n_url_server; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url; + } + case 2: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_1; + } + case 3: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_url_2; + } + case 4: { + p++; + goto s_n_llhttp__internal__n_url_server; + } + case 5: { + goto s_n_llhttp__internal__n_span_start_stub_path; + } + case 6: { + p++; + goto s_n_llhttp__internal__n_url_query; + } + case 7: { + p++; + goto s_n_llhttp__internal__n_url_server_with_at; + } + default: { + goto s_n_llhttp__internal__n_error_36; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_schema_delim_1: + s_n_llhttp__internal__n_url_schema_delim_1: { + if (p == endp) { + return s_n_llhttp__internal__n_url_schema_delim_1; + } + switch (*p) { + case '/': { + p++; + goto s_n_llhttp__internal__n_url_server; + } + default: { + goto s_n_llhttp__internal__n_error_38; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_schema_delim: + s_n_llhttp__internal__n_url_schema_delim: { + if (p == endp) { + return s_n_llhttp__internal__n_url_schema_delim; + } + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_error_37; + } + case 13: { + p++; + goto s_n_llhttp__internal__n_error_37; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_error_37; + } + case '/': { + p++; + goto s_n_llhttp__internal__n_url_schema_delim_1; + } + default: { + goto s_n_llhttp__internal__n_error_38; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_end_stub_schema: + s_n_llhttp__internal__n_span_end_stub_schema: { + if (p == endp) { + return s_n_llhttp__internal__n_span_end_stub_schema; + } + p++; + goto s_n_llhttp__internal__n_url_schema_delim; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_schema: + s_n_llhttp__internal__n_url_schema: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, + 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, + 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + if (p == endp) { + return s_n_llhttp__internal__n_url_schema; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_error_37; + } + case 2: { + goto s_n_llhttp__internal__n_span_end_stub_schema; + } + case 3: { + p++; + goto s_n_llhttp__internal__n_url_schema; + } + default: { + goto s_n_llhttp__internal__n_error_39; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_url_start: + s_n_llhttp__internal__n_url_start: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, + 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + if (p == endp) { + return s_n_llhttp__internal__n_url_start; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_error_37; + } + case 2: { + goto s_n_llhttp__internal__n_span_start_stub_path_2; + } + case 3: { + goto s_n_llhttp__internal__n_url_schema; + } + default: { + goto s_n_llhttp__internal__n_error_40; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_llhttp__on_url_1: + s_n_llhttp__internal__n_span_start_llhttp__on_url_1: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_url_1; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_url; + goto s_n_llhttp__internal__n_url_start; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_llhttp__on_url: + s_n_llhttp__internal__n_span_start_llhttp__on_url: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_url; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_url; + goto s_n_llhttp__internal__n_url_server; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_spaces_before_url: + s_n_llhttp__internal__n_req_spaces_before_url: { + if (p == endp) { + return s_n_llhttp__internal__n_req_spaces_before_url; + } + switch (*p) { + case ' ': { + p++; + goto s_n_llhttp__internal__n_req_spaces_before_url; + } + default: { + goto s_n_llhttp__internal__n_invoke_is_equal_method; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_first_space_before_url: + s_n_llhttp__internal__n_req_first_space_before_url: { + if (p == endp) { + return s_n_llhttp__internal__n_req_first_space_before_url; + } + switch (*p) { + case ' ': { + p++; + goto s_n_llhttp__internal__n_req_spaces_before_url; + } + default: { + goto s_n_llhttp__internal__n_error_41; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_2: + s_n_llhttp__internal__n_start_req_2: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_2; + } + switch (*p) { + case 'L': { + p++; + match = 19; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_3: + s_n_llhttp__internal__n_start_req_3: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_3; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob19, 6); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 36; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_3; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_1: + s_n_llhttp__internal__n_start_req_1: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_1; + } + switch (*p) { + case 'C': { + p++; + goto s_n_llhttp__internal__n_start_req_2; + } + case 'N': { + p++; + goto s_n_llhttp__internal__n_start_req_3; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_4: + s_n_llhttp__internal__n_start_req_4: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_4; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob20, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 16; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_4; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_6: + s_n_llhttp__internal__n_start_req_6: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_6; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob21, 6); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 22; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_6; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_8: + s_n_llhttp__internal__n_start_req_8: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_8; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob22, 4); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_8; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_9: + s_n_llhttp__internal__n_start_req_9: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_9; + } + switch (*p) { + case 'Y': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_7: + s_n_llhttp__internal__n_start_req_7: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_7; + } + switch (*p) { + case 'N': { + p++; + goto s_n_llhttp__internal__n_start_req_8; + } + case 'P': { + p++; + goto s_n_llhttp__internal__n_start_req_9; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_5: + s_n_llhttp__internal__n_start_req_5: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_5; + } + switch (*p) { + case 'H': { + p++; + goto s_n_llhttp__internal__n_start_req_6; + } + case 'O': { + p++; + goto s_n_llhttp__internal__n_start_req_7; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_12: + s_n_llhttp__internal__n_start_req_12: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_12; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob23, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_12; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_13: + s_n_llhttp__internal__n_start_req_13: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_13; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob24, 5); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 35; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_13; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_11: + s_n_llhttp__internal__n_start_req_11: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_11; + } + switch (*p) { + case 'L': { + p++; + goto s_n_llhttp__internal__n_start_req_12; + } + case 'S': { + p++; + goto s_n_llhttp__internal__n_start_req_13; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_10: + s_n_llhttp__internal__n_start_req_10: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_10; + } + switch (*p) { + case 'E': { + p++; + goto s_n_llhttp__internal__n_start_req_11; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_14: + s_n_llhttp__internal__n_start_req_14: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_14; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob25, 4); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 45; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_14; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_17: + s_n_llhttp__internal__n_start_req_17: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_17; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob27, 9); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 41; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_17; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_16: + s_n_llhttp__internal__n_start_req_16: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_16; + } + switch (*p) { + case '_': { + p++; + goto s_n_llhttp__internal__n_start_req_17; + } + default: { + match = 1; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_15: + s_n_llhttp__internal__n_start_req_15: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_15; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob26, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_start_req_16; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_15; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_18: + s_n_llhttp__internal__n_start_req_18: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_18; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob28, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_18; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_20: + s_n_llhttp__internal__n_start_req_20: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_20; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob29, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 31; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_20; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_21: + s_n_llhttp__internal__n_start_req_21: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_21; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob30, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_21; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_19: + s_n_llhttp__internal__n_start_req_19: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_19; + } + switch (*p) { + case 'I': { + p++; + goto s_n_llhttp__internal__n_start_req_20; + } + case 'O': { + p++; + goto s_n_llhttp__internal__n_start_req_21; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_23: + s_n_llhttp__internal__n_start_req_23: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_23; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob31, 6); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 24; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_23; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_24: + s_n_llhttp__internal__n_start_req_24: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_24; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob32, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 23; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_24; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_26: + s_n_llhttp__internal__n_start_req_26: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_26; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob33, 7); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 21; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_26; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_28: + s_n_llhttp__internal__n_start_req_28: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_28; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob34, 6); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 30; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_28; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_29: + s_n_llhttp__internal__n_start_req_29: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_29; + } + switch (*p) { + case 'L': { + p++; + match = 10; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_27: + s_n_llhttp__internal__n_start_req_27: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_27; + } + switch (*p) { + case 'A': { + p++; + goto s_n_llhttp__internal__n_start_req_28; + } + case 'O': { + p++; + goto s_n_llhttp__internal__n_start_req_29; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_25: + s_n_llhttp__internal__n_start_req_25: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_25; + } + switch (*p) { + case 'A': { + p++; + goto s_n_llhttp__internal__n_start_req_26; + } + case 'C': { + p++; + goto s_n_llhttp__internal__n_start_req_27; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_30: + s_n_llhttp__internal__n_start_req_30: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_30; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob35, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 11; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_30; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_22: + s_n_llhttp__internal__n_start_req_22: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_22; + } + switch (*p) { + case '-': { + p++; + goto s_n_llhttp__internal__n_start_req_23; + } + case 'E': { + p++; + goto s_n_llhttp__internal__n_start_req_24; + } + case 'K': { + p++; + goto s_n_llhttp__internal__n_start_req_25; + } + case 'O': { + p++; + goto s_n_llhttp__internal__n_start_req_30; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_31: + s_n_llhttp__internal__n_start_req_31: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_31; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob36, 5); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 25; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_31; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_32: + s_n_llhttp__internal__n_start_req_32: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_32; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob37, 6); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_32; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_35: + s_n_llhttp__internal__n_start_req_35: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_35; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob38, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 28; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_35; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_36: + s_n_llhttp__internal__n_start_req_36: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_36; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob39, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 39; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_36; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_34: + s_n_llhttp__internal__n_start_req_34: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_34; + } + switch (*p) { + case 'T': { + p++; + goto s_n_llhttp__internal__n_start_req_35; + } + case 'U': { + p++; + goto s_n_llhttp__internal__n_start_req_36; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_37: + s_n_llhttp__internal__n_start_req_37: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_37; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob40, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 38; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_37; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_38: + s_n_llhttp__internal__n_start_req_38: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_38; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob41, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_38; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_42: + s_n_llhttp__internal__n_start_req_42: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_42; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob42, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 12; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_42; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_43: + s_n_llhttp__internal__n_start_req_43: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_43; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob43, 4); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 13; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_43; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_41: + s_n_llhttp__internal__n_start_req_41: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_41; + } + switch (*p) { + case 'F': { + p++; + goto s_n_llhttp__internal__n_start_req_42; + } + case 'P': { + p++; + goto s_n_llhttp__internal__n_start_req_43; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_40: + s_n_llhttp__internal__n_start_req_40: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_40; + } + switch (*p) { + case 'P': { + p++; + goto s_n_llhttp__internal__n_start_req_41; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_39: + s_n_llhttp__internal__n_start_req_39: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_39; + } + switch (*p) { + case 'I': { + p++; + match = 34; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case 'O': { + p++; + goto s_n_llhttp__internal__n_start_req_40; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_45: + s_n_llhttp__internal__n_start_req_45: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_45; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob44, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 29; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_45; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_44: + s_n_llhttp__internal__n_start_req_44: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_44; + } + switch (*p) { + case 'R': { + p++; + goto s_n_llhttp__internal__n_start_req_45; + } + case 'T': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_33: + s_n_llhttp__internal__n_start_req_33: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_33; + } + switch (*p) { + case 'A': { + p++; + goto s_n_llhttp__internal__n_start_req_34; + } + case 'L': { + p++; + goto s_n_llhttp__internal__n_start_req_37; + } + case 'O': { + p++; + goto s_n_llhttp__internal__n_start_req_38; + } + case 'R': { + p++; + goto s_n_llhttp__internal__n_start_req_39; + } + case 'U': { + p++; + goto s_n_llhttp__internal__n_start_req_44; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_48: + s_n_llhttp__internal__n_start_req_48: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_48; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob45, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 17; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_48; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_49: + s_n_llhttp__internal__n_start_req_49: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_49; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob46, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 44; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_49; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_50: + s_n_llhttp__internal__n_start_req_50: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_50; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob47, 5); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 43; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_50; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_51: + s_n_llhttp__internal__n_start_req_51: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_51; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob48, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 20; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_51; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_47: + s_n_llhttp__internal__n_start_req_47: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_47; + } + switch (*p) { + case 'B': { + p++; + goto s_n_llhttp__internal__n_start_req_48; + } + case 'C': { + p++; + goto s_n_llhttp__internal__n_start_req_49; + } + case 'D': { + p++; + goto s_n_llhttp__internal__n_start_req_50; + } + case 'P': { + p++; + goto s_n_llhttp__internal__n_start_req_51; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_46: + s_n_llhttp__internal__n_start_req_46: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_46; + } + switch (*p) { + case 'E': { + p++; + goto s_n_llhttp__internal__n_start_req_47; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_54: + s_n_llhttp__internal__n_start_req_54: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_54; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob49, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 14; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_54; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_56: + s_n_llhttp__internal__n_start_req_56: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_56; + } + switch (*p) { + case 'P': { + p++; + match = 37; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_57: + s_n_llhttp__internal__n_start_req_57: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_57; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob50, 9); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 42; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_57; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_55: + s_n_llhttp__internal__n_start_req_55: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_55; + } + switch (*p) { + case 'U': { + p++; + goto s_n_llhttp__internal__n_start_req_56; + } + case '_': { + p++; + goto s_n_llhttp__internal__n_start_req_57; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_53: + s_n_llhttp__internal__n_start_req_53: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_53; + } + switch (*p) { + case 'A': { + p++; + goto s_n_llhttp__internal__n_start_req_54; + } + case 'T': { + p++; + goto s_n_llhttp__internal__n_start_req_55; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_58: + s_n_llhttp__internal__n_start_req_58: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_58; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob51, 4); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 33; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_58; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_59: + s_n_llhttp__internal__n_start_req_59: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_59; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob52, 7); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 26; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_59; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_52: + s_n_llhttp__internal__n_start_req_52: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_52; + } + switch (*p) { + case 'E': { + p++; + goto s_n_llhttp__internal__n_start_req_53; + } + case 'O': { + p++; + goto s_n_llhttp__internal__n_start_req_58; + } + case 'U': { + p++; + goto s_n_llhttp__internal__n_start_req_59; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_61: + s_n_llhttp__internal__n_start_req_61: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_61; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob53, 6); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 40; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_61; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_62: + s_n_llhttp__internal__n_start_req_62: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_62; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob54, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_62; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_60: + s_n_llhttp__internal__n_start_req_60: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_60; + } + switch (*p) { + case 'E': { + p++; + goto s_n_llhttp__internal__n_start_req_61; + } + case 'R': { + p++; + goto s_n_llhttp__internal__n_start_req_62; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_65: + s_n_llhttp__internal__n_start_req_65: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_65; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob55, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 18; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_65; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_67: + s_n_llhttp__internal__n_start_req_67: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_67; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob56, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 32; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_67; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_68: + s_n_llhttp__internal__n_start_req_68: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_68; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob57, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 15; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_68; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_66: + s_n_llhttp__internal__n_start_req_66: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_66; + } + switch (*p) { + case 'I': { + p++; + goto s_n_llhttp__internal__n_start_req_67; + } + case 'O': { + p++; + goto s_n_llhttp__internal__n_start_req_68; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_69: + s_n_llhttp__internal__n_start_req_69: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_req_69; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob58, 8); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 27; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_req_69; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_64: + s_n_llhttp__internal__n_start_req_64: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_64; + } + switch (*p) { + case 'B': { + p++; + goto s_n_llhttp__internal__n_start_req_65; + } + case 'L': { + p++; + goto s_n_llhttp__internal__n_start_req_66; + } + case 'S': { + p++; + goto s_n_llhttp__internal__n_start_req_69; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_63: + s_n_llhttp__internal__n_start_req_63: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_63; + } + switch (*p) { + case 'N': { + p++; + goto s_n_llhttp__internal__n_start_req_64; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req: + s_n_llhttp__internal__n_start_req: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req; + } + switch (*p) { + case 'A': { + p++; + goto s_n_llhttp__internal__n_start_req_1; + } + case 'B': { + p++; + goto s_n_llhttp__internal__n_start_req_4; + } + case 'C': { + p++; + goto s_n_llhttp__internal__n_start_req_5; + } + case 'D': { + p++; + goto s_n_llhttp__internal__n_start_req_10; + } + case 'F': { + p++; + goto s_n_llhttp__internal__n_start_req_14; + } + case 'G': { + p++; + goto s_n_llhttp__internal__n_start_req_15; + } + case 'H': { + p++; + goto s_n_llhttp__internal__n_start_req_18; + } + case 'L': { + p++; + goto s_n_llhttp__internal__n_start_req_19; + } + case 'M': { + p++; + goto s_n_llhttp__internal__n_start_req_22; + } + case 'N': { + p++; + goto s_n_llhttp__internal__n_start_req_31; + } + case 'O': { + p++; + goto s_n_llhttp__internal__n_start_req_32; + } + case 'P': { + p++; + goto s_n_llhttp__internal__n_start_req_33; + } + case 'R': { + p++; + goto s_n_llhttp__internal__n_start_req_46; + } + case 'S': { + p++; + goto s_n_llhttp__internal__n_start_req_52; + } + case 'T': { + p++; + goto s_n_llhttp__internal__n_start_req_60; + } + case 'U': { + p++; + goto s_n_llhttp__internal__n_start_req_63; + } + default: { + goto s_n_llhttp__internal__n_error_49; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_llhttp__on_status_complete: + s_n_llhttp__internal__n_invoke_llhttp__on_status_complete: { + switch (llhttp__on_status_complete(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_field_start; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_res_line_almost_done: + s_n_llhttp__internal__n_res_line_almost_done: { + if (p == endp) { + return s_n_llhttp__internal__n_res_line_almost_done; + } + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_res_status: + s_n_llhttp__internal__n_res_status: { + if (p == endp) { + return s_n_llhttp__internal__n_res_status; + } + switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_status; + } + case 13: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_status_1; + } + default: { + p++; + goto s_n_llhttp__internal__n_res_status; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_span_start_llhttp__on_status: + s_n_llhttp__internal__n_span_start_llhttp__on_status: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_status; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_status; + goto s_n_llhttp__internal__n_res_status; + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_res_status_start: + s_n_llhttp__internal__n_res_status_start: { + if (p == endp) { + return s_n_llhttp__internal__n_res_status_start; + } + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; + } + case 13: { + p++; + goto s_n_llhttp__internal__n_res_line_almost_done; + } + default: { + goto s_n_llhttp__internal__n_span_start_llhttp__on_status; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_res_status_code_otherwise: + s_n_llhttp__internal__n_res_status_code_otherwise: { + if (p == endp) { + return s_n_llhttp__internal__n_res_status_code_otherwise; + } + switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_res_status_start; + } + case 13: { + goto s_n_llhttp__internal__n_res_status_start; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_res_status_start; + } + default: { + goto s_n_llhttp__internal__n_error_43; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_res_status_code: + s_n_llhttp__internal__n_res_status_code: { + if (p == endp) { + return s_n_llhttp__internal__n_res_status_code; + } + switch (*p) { + case '0': { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + case '1': { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + case '2': { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + case '3': { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + case '4': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + case '5': { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + case '6': { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + case '7': { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + case '8': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + case '9': { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code; + } + default: { + goto s_n_llhttp__internal__n_res_status_code_otherwise; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_res_http_end: + s_n_llhttp__internal__n_res_http_end: { + if (p == endp) { + return s_n_llhttp__internal__n_res_http_end; + } + switch (*p) { + case ' ': { + p++; + goto s_n_llhttp__internal__n_invoke_update_status_code; + } + default: { + goto s_n_llhttp__internal__n_error_44; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_res_http_minor: + s_n_llhttp__internal__n_res_http_minor: { + if (p == endp) { + return s_n_llhttp__internal__n_res_http_minor; + } + switch (*p) { + case '0': { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + case '1': { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + case '2': { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + case '3': { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + case '4': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + case '5': { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + case '6': { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + case '7': { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + case '8': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + case '9': { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_store_http_minor_1; + } + default: { + goto s_n_llhttp__internal__n_error_45; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_res_http_dot: + s_n_llhttp__internal__n_res_http_dot: { + if (p == endp) { + return s_n_llhttp__internal__n_res_http_dot; + } + switch (*p) { + case '.': { + p++; + goto s_n_llhttp__internal__n_res_http_minor; + } + default: { + goto s_n_llhttp__internal__n_error_46; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_res_http_major: + s_n_llhttp__internal__n_res_http_major: { + if (p == endp) { + return s_n_llhttp__internal__n_res_http_major; + } + switch (*p) { + case '0': { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + case '1': { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + case '2': { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + case '3': { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + case '4': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + case '5': { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + case '6': { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + case '7': { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + case '8': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + case '9': { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_store_http_major_1; + } + default: { + goto s_n_llhttp__internal__n_error_47; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_res: + s_n_llhttp__internal__n_start_res: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_start_res; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob59, 5); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_res_http_major; + } + case kMatchPause: { + return s_n_llhttp__internal__n_start_res; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_50; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_or_res_method_2: + s_n_llhttp__internal__n_req_or_res_method_2: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_req_or_res_method_2; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob60, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_store_method; + } + case kMatchPause: { + return s_n_llhttp__internal__n_req_or_res_method_2; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_48; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_or_res_method_3: + s_n_llhttp__internal__n_req_or_res_method_3: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_req_or_res_method_3; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob61, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_invoke_update_type_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_req_or_res_method_3; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_48; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_or_res_method_1: + s_n_llhttp__internal__n_req_or_res_method_1: { + if (p == endp) { + return s_n_llhttp__internal__n_req_or_res_method_1; + } + switch (*p) { + case 'E': { + p++; + goto s_n_llhttp__internal__n_req_or_res_method_2; + } + case 'T': { + p++; + goto s_n_llhttp__internal__n_req_or_res_method_3; + } + default: { + goto s_n_llhttp__internal__n_error_48; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_req_or_res_method: + s_n_llhttp__internal__n_req_or_res_method: { + if (p == endp) { + return s_n_llhttp__internal__n_req_or_res_method; + } + switch (*p) { + case 'H': { + p++; + goto s_n_llhttp__internal__n_req_or_res_method_1; + } + default: { + goto s_n_llhttp__internal__n_error_48; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start_req_or_res: + s_n_llhttp__internal__n_start_req_or_res: { + if (p == endp) { + return s_n_llhttp__internal__n_start_req_or_res; + } + switch (*p) { + case 'H': { + goto s_n_llhttp__internal__n_req_or_res_method; + } + default: { + goto s_n_llhttp__internal__n_invoke_update_type_2; + } + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_invoke_load_type: + s_n_llhttp__internal__n_invoke_load_type: { + switch (llhttp__internal__c_load_type(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_start_req; + case 2: + goto s_n_llhttp__internal__n_start_res; + default: + goto s_n_llhttp__internal__n_start_req_or_res; + } + /* UNREACHABLE */; + abort(); + } + case s_n_llhttp__internal__n_start: + s_n_llhttp__internal__n_start: { + if (p == endp) { + return s_n_llhttp__internal__n_start; + } + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_start; + } + case 13: { + p++; + goto s_n_llhttp__internal__n_start; + } + default: { + goto s_n_llhttp__internal__n_invoke_update_finish; + } + } + /* UNREACHABLE */; + abort(); + } + default: + /* UNREACHABLE */ + abort(); + } + s_n_llhttp__internal__n_error_37: { + state->error = 0x7; + state->reason = "Invalid characters in url"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_finish_1: { + switch (llhttp__internal__c_update_finish_1(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_start; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_5: { + state->error = 0x15; + state->reason = "on_message_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_is_equal_upgrade; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_9: { + state->error = 0x12; + state->reason = "`on_message_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_7: { + state->error = 0x15; + state->reason = "on_chunk_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_12: { + state->error = 0x14; + state->reason = "`on_chunk_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete_1: { + switch (llhttp__on_chunk_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; + case 21: + goto s_n_llhttp__internal__n_pause_7; + default: + goto s_n_llhttp__internal__n_error_12; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_11: { + state->error = 0x4; + state->reason = "Content-Length can't be present with Transfer-Encoding"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_2: { + state->error = 0x15; + state->reason = "on_message_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_pause_1; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_3: { + state->error = 0x12; + state->reason = "`on_message_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_1: { + switch (llhttp__on_message_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_pause_1; + case 21: + goto s_n_llhttp__internal__n_pause_2; + default: + goto s_n_llhttp__internal__n_error_3; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_7: { + state->error = 0xc; + state->reason = "Chunk size overflow"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_3: { + state->error = 0x15; + state->reason = "on_chunk_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_update_content_length; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_5: { + state->error = 0x14; + state->reason = "`on_chunk_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete: { + switch (llhttp__on_chunk_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_update_content_length; + case 21: + goto s_n_llhttp__internal__n_pause_3; + default: + goto s_n_llhttp__internal__n_error_5; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_body: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_body(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_chunk_data_almost_done; + return s_error; + } + goto s_n_llhttp__internal__n_chunk_data_almost_done; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags: { + switch (llhttp__internal__c_or_flags(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_field_start; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_4: { + state->error = 0x15; + state->reason = "on_chunk_header pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_is_equal_content_length; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_4: { + state->error = 0x13; + state->reason = "`on_chunk_header` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_header: { + switch (llhttp__on_chunk_header(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_is_equal_content_length; + case 21: + goto s_n_llhttp__internal__n_pause_4; + default: + goto s_n_llhttp__internal__n_error_4; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_6: { + state->error = 0xc; + state->reason = "Invalid character in chunk size"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_mul_add_content_length: { + switch (llhttp__internal__c_mul_add_content_length(state, p, endp, match)) { + case 1: + goto s_n_llhttp__internal__n_error_7; + default: + goto s_n_llhttp__internal__n_chunk_size; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_8: { + state->error = 0xc; + state->reason = "Invalid character in chunk size"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_body_1: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_body(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; + return s_error; + } + goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_finish_2: { + switch (llhttp__internal__c_update_finish_2(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_span_start_llhttp__on_body_2; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_10: { + state->error = 0xf; + state->reason = "Request has invalid `Transfer-Encoding`"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause: { + state->error = 0x15; + state->reason = "on_message_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__after_message_complete; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_2: { + state->error = 0x12; + state->reason = "`on_message_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_message_complete: { + switch (llhttp__on_message_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_llhttp__after_message_complete; + case 21: + goto s_n_llhttp__internal__n_pause; + default: + goto s_n_llhttp__internal__n_error_2; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_1: { + switch (llhttp__internal__c_or_flags_1(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_2: { + switch (llhttp__internal__c_or_flags_1(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_upgrade: { + switch (llhttp__internal__c_update_upgrade(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_or_flags_2; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_6: { + state->error = 0x15; + state->reason = "Paused by on_headers_complete"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_1: { + state->error = 0x11; + state->reason = "User callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_headers_complete: { + switch (llhttp__on_headers_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; + case 1: + goto s_n_llhttp__internal__n_invoke_or_flags_1; + case 2: + goto s_n_llhttp__internal__n_invoke_update_upgrade; + case 21: + goto s_n_llhttp__internal__n_pause_6; + default: + goto s_n_llhttp__internal__n_error_1; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete: { + switch (llhttp__before_headers_complete(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__on_headers_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_test_lenient_flags: { + switch (llhttp__internal__c_test_lenient_flags(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_error_11; + default: + goto s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_test_flags_1: { + switch (llhttp__internal__c_test_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_test_lenient_flags; + default: + goto s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_test_flags: { + switch (llhttp__internal__c_test_flags(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete_1; + default: + goto s_n_llhttp__internal__n_invoke_test_flags_1; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_13: { + state->error = 0xb; + state->reason = "Empty Content-Length"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_value: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; + return s_error; + } + goto s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state: { + switch (llhttp__internal__c_update_header_state(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_3: { + switch (llhttp__internal__c_or_flags_3(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_4: { + switch (llhttp__internal__c_or_flags_4(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_5: { + switch (llhttp__internal__c_or_flags_5(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_6: { + switch (llhttp__internal__c_or_flags_6(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_load_header_state_1: { + switch (llhttp__internal__c_load_header_state(state, p, endp)) { + case 5: + goto s_n_llhttp__internal__n_invoke_or_flags_3; + case 6: + goto s_n_llhttp__internal__n_invoke_or_flags_4; + case 7: + goto s_n_llhttp__internal__n_invoke_or_flags_5; + case 8: + goto s_n_llhttp__internal__n_invoke_or_flags_6; + default: + goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_load_header_state: { + switch (llhttp__internal__c_load_header_state(state, p, endp)) { + case 2: + goto s_n_llhttp__internal__n_error_13; + default: + goto s_n_llhttp__internal__n_invoke_load_header_state_1; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_1: { + switch (llhttp__internal__c_update_header_state(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_7: { + switch (llhttp__internal__c_or_flags_3(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state_1; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_8: { + switch (llhttp__internal__c_or_flags_4(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state_1; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_9: { + switch (llhttp__internal__c_or_flags_5(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state_1; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_10: { + switch (llhttp__internal__c_or_flags_6(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_load_header_state_3: { + switch (llhttp__internal__c_load_header_state(state, p, endp)) { + case 5: + goto s_n_llhttp__internal__n_invoke_or_flags_7; + case 6: + goto s_n_llhttp__internal__n_invoke_or_flags_8; + case 7: + goto s_n_llhttp__internal__n_invoke_or_flags_9; + case 8: + goto s_n_llhttp__internal__n_invoke_or_flags_10; + default: + goto s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_14: { + state->error = 0x3; + state->reason = "Missing expected LF after header value"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_1: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_header_value_almost_done; + return s_error; + } + goto s_n_llhttp__internal__n_header_value_almost_done; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_2: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_header_value_almost_done; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_header_value_almost_done; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_3: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_header_value_almost_done; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_header_value_almost_done; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_15: { + state->error = 0xa; + state->reason = "Invalid header value char"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_1: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_header_value_lenient; + default: + goto s_n_llhttp__internal__n_error_15; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_3: { + switch (llhttp__internal__c_update_header_state(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_connection; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_11: { + switch (llhttp__internal__c_or_flags_3(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state_3; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_12: { + switch (llhttp__internal__c_or_flags_4(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state_3; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_13: { + switch (llhttp__internal__c_or_flags_5(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state_3; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_14: { + switch (llhttp__internal__c_or_flags_6(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_connection; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_load_header_state_4: { + switch (llhttp__internal__c_load_header_state(state, p, endp)) { + case 5: + goto s_n_llhttp__internal__n_invoke_or_flags_11; + case 6: + goto s_n_llhttp__internal__n_invoke_or_flags_12; + case 7: + goto s_n_llhttp__internal__n_invoke_or_flags_13; + case 8: + goto s_n_llhttp__internal__n_invoke_or_flags_14; + default: + goto s_n_llhttp__internal__n_header_value_connection; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_4: { + switch (llhttp__internal__c_update_header_state_4(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_connection_token; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_2: { + switch (llhttp__internal__c_update_header_state_2(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_connection_ws; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_5: { + switch (llhttp__internal__c_update_header_state_5(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_connection_ws; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_6: { + switch (llhttp__internal__c_update_header_state_6(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_connection_ws; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_4: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_17; + return s_error; + } + goto s_n_llhttp__internal__n_error_17; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_mul_add_content_length_1: { + switch (llhttp__internal__c_mul_add_content_length_1(state, p, endp, match)) { + case 1: + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_4; + default: + goto s_n_llhttp__internal__n_header_value_content_length; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_15: { + switch (llhttp__internal__c_or_flags_15(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_otherwise; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_5: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_18; + return s_error; + } + goto s_n_llhttp__internal__n_error_18; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_16: { + state->error = 0x4; + state->reason = "Duplicate Content-Length"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_test_flags_2: { + switch (llhttp__internal__c_test_flags_2(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_header_value_content_length; + default: + goto s_n_llhttp__internal__n_error_16; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_7: { + switch (llhttp__internal__c_update_header_state_7(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_otherwise; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_8: { + switch (llhttp__internal__c_update_header_state_4(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_and_flags: { + switch (llhttp__internal__c_and_flags(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_te_chunked; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_16: { + switch (llhttp__internal__c_or_flags_16(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_and_flags; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_or_flags_17: { + switch (llhttp__internal__c_or_flags_17(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state_8; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_load_header_state_2: { + switch (llhttp__internal__c_load_header_state(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_header_value_connection; + case 2: + goto s_n_llhttp__internal__n_invoke_test_flags_2; + case 3: + goto s_n_llhttp__internal__n_invoke_or_flags_16; + case 4: + goto s_n_llhttp__internal__n_invoke_or_flags_17; + default: + goto s_n_llhttp__internal__n_header_value; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_field: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_field(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_field_1: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_field(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_19: { + state->error = 0xa; + state->reason = "Invalid header token"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_9: { + switch (llhttp__internal__c_update_header_state_4(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_field_general; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_store_header_state: { + switch (llhttp__internal__c_store_header_state(state, p, endp, match)) { + default: + goto s_n_llhttp__internal__n_header_field_colon; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_header_state_10: { + switch (llhttp__internal__c_update_header_state_4(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_field_general; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_url_complete: { + switch (llhttp__on_url_complete(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_field_start; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_http_minor: { + switch (llhttp__internal__c_update_http_minor(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__on_url_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_http_major: { + switch (llhttp__internal__c_update_http_major(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_http_minor; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_3: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_20: { + state->error = 0x7; + state->reason = "Expected CRLF"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_4: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_lf_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_lf_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_23: { + state->error = 0x16; + state->reason = "Pause on PRI/Upgrade"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_24: { + state->error = 0x9; + state->reason = "Expected HTTP/2 Connection Preface"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_22: { + state->error = 0x9; + state->reason = "Expected CRLF after version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_load_method_1: { + switch (llhttp__internal__c_load_method(state, p, endp)) { + case 34: + goto s_n_llhttp__internal__n_req_pri_upgrade; + default: + goto s_n_llhttp__internal__n_req_http_complete; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_store_http_minor: { + switch (llhttp__internal__c_store_http_minor(state, p, endp, match)) { + default: + goto s_n_llhttp__internal__n_invoke_load_method_1; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_25: { + state->error = 0x9; + state->reason = "Invalid minor version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_26: { + state->error = 0x9; + state->reason = "Expected dot"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_store_http_major: { + switch (llhttp__internal__c_store_http_major(state, p, endp, match)) { + default: + goto s_n_llhttp__internal__n_req_http_dot; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_27: { + state->error = 0x9; + state->reason = "Invalid major version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_21: { + state->error = 0x8; + state->reason = "Invalid method for HTTP/x.x request"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_load_method: { + switch (llhttp__internal__c_load_method(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_req_http_major; + case 1: + goto s_n_llhttp__internal__n_req_http_major; + case 2: + goto s_n_llhttp__internal__n_req_http_major; + case 3: + goto s_n_llhttp__internal__n_req_http_major; + case 4: + goto s_n_llhttp__internal__n_req_http_major; + case 5: + goto s_n_llhttp__internal__n_req_http_major; + case 6: + goto s_n_llhttp__internal__n_req_http_major; + case 7: + goto s_n_llhttp__internal__n_req_http_major; + case 8: + goto s_n_llhttp__internal__n_req_http_major; + case 9: + goto s_n_llhttp__internal__n_req_http_major; + case 10: + goto s_n_llhttp__internal__n_req_http_major; + case 11: + goto s_n_llhttp__internal__n_req_http_major; + case 12: + goto s_n_llhttp__internal__n_req_http_major; + case 13: + goto s_n_llhttp__internal__n_req_http_major; + case 14: + goto s_n_llhttp__internal__n_req_http_major; + case 15: + goto s_n_llhttp__internal__n_req_http_major; + case 16: + goto s_n_llhttp__internal__n_req_http_major; + case 17: + goto s_n_llhttp__internal__n_req_http_major; + case 18: + goto s_n_llhttp__internal__n_req_http_major; + case 19: + goto s_n_llhttp__internal__n_req_http_major; + case 20: + goto s_n_llhttp__internal__n_req_http_major; + case 21: + goto s_n_llhttp__internal__n_req_http_major; + case 22: + goto s_n_llhttp__internal__n_req_http_major; + case 23: + goto s_n_llhttp__internal__n_req_http_major; + case 24: + goto s_n_llhttp__internal__n_req_http_major; + case 25: + goto s_n_llhttp__internal__n_req_http_major; + case 26: + goto s_n_llhttp__internal__n_req_http_major; + case 27: + goto s_n_llhttp__internal__n_req_http_major; + case 28: + goto s_n_llhttp__internal__n_req_http_major; + case 29: + goto s_n_llhttp__internal__n_req_http_major; + case 30: + goto s_n_llhttp__internal__n_req_http_major; + case 31: + goto s_n_llhttp__internal__n_req_http_major; + case 32: + goto s_n_llhttp__internal__n_req_http_major; + case 33: + goto s_n_llhttp__internal__n_req_http_major; + case 34: + goto s_n_llhttp__internal__n_req_http_major; + default: + goto s_n_llhttp__internal__n_error_21; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_30: { + state->error = 0x8; + state->reason = "Expected HTTP/"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_28: { + state->error = 0x8; + state->reason = "Expected SOURCE method for ICE/x.x request"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_load_method_2: { + switch (llhttp__internal__c_load_method(state, p, endp)) { + case 33: + goto s_n_llhttp__internal__n_req_http_major; + default: + goto s_n_llhttp__internal__n_error_28; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_29: { + state->error = 0x8; + state->reason = "Invalid method for RTSP/x.x request"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_load_method_3: { + switch (llhttp__internal__c_load_method(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_req_http_major; + case 3: + goto s_n_llhttp__internal__n_req_http_major; + case 6: + goto s_n_llhttp__internal__n_req_http_major; + case 35: + goto s_n_llhttp__internal__n_req_http_major; + case 36: + goto s_n_llhttp__internal__n_req_http_major; + case 37: + goto s_n_llhttp__internal__n_req_http_major; + case 38: + goto s_n_llhttp__internal__n_req_http_major; + case 39: + goto s_n_llhttp__internal__n_req_http_major; + case 40: + goto s_n_llhttp__internal__n_req_http_major; + case 41: + goto s_n_llhttp__internal__n_req_http_major; + case 42: + goto s_n_llhttp__internal__n_req_http_major; + case 43: + goto s_n_llhttp__internal__n_req_http_major; + case 44: + goto s_n_llhttp__internal__n_req_http_major; + case 45: + goto s_n_llhttp__internal__n_req_http_major; + default: + goto s_n_llhttp__internal__n_error_29; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_url_complete_1: { + switch (llhttp__on_url_complete(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_req_http_start; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_5: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_6: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_7: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_lf_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_lf_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_8: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_31: { + state->error = 0x7; + state->reason = "Invalid char in url fragment start"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_9: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_10: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_lf_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_lf_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_11: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_32: { + state->error = 0x7; + state->reason = "Invalid char in url query"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_33: { + state->error = 0x7; + state->reason = "Invalid char in url path"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_1: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_lf_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_lf_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_2: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_12: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_13: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_lf_to_http09; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_lf_to_http09; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_url_14: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_url(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http; + return s_error; + } + goto s_n_llhttp__internal__n_url_skip_to_http; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_34: { + state->error = 0x7; + state->reason = "Double @ in url"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_35: { + state->error = 0x7; + state->reason = "Unexpected char in url server"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_36: { + state->error = 0x7; + state->reason = "Unexpected char in url server"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_38: { + state->error = 0x7; + state->reason = "Unexpected char in url schema"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_39: { + state->error = 0x7; + state->reason = "Unexpected char in url schema"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_40: { + state->error = 0x7; + state->reason = "Unexpected start char in url"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_is_equal_method: { + switch (llhttp__internal__c_is_equal_method(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_span_start_llhttp__on_url_1; + default: + goto s_n_llhttp__internal__n_span_start_llhttp__on_url; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_41: { + state->error = 0x6; + state->reason = "Expected space after method"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_store_method_1: { + switch (llhttp__internal__c_store_method(state, p, endp, match)) { + default: + goto s_n_llhttp__internal__n_req_first_space_before_url; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_49: { + state->error = 0x6; + state->reason = "Invalid method encountered"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_42: { + state->error = 0xd; + state->reason = "Response overflow"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_mul_add_status_code: { + switch (llhttp__internal__c_mul_add_status_code(state, p, endp, match)) { + case 1: + goto s_n_llhttp__internal__n_error_42; + default: + goto s_n_llhttp__internal__n_res_status_code; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_status: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_status(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_span_end_llhttp__on_status_1: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_status(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_res_line_almost_done; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_res_line_almost_done; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_43: { + state->error = 0xd; + state->reason = "Invalid response status"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_status_code: { + switch (llhttp__internal__c_update_status_code(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_res_status_code; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_44: { + state->error = 0x9; + state->reason = "Expected space after version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_store_http_minor_1: { + switch (llhttp__internal__c_store_http_minor(state, p, endp, match)) { + default: + goto s_n_llhttp__internal__n_res_http_end; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_45: { + state->error = 0x9; + state->reason = "Invalid minor version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_46: { + state->error = 0x9; + state->reason = "Expected dot"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_store_http_major_1: { + switch (llhttp__internal__c_store_http_major(state, p, endp, match)) { + default: + goto s_n_llhttp__internal__n_res_http_dot; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_47: { + state->error = 0x9; + state->reason = "Invalid major version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_50: { + state->error = 0x8; + state->reason = "Expected HTTP/"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_type: { + switch (llhttp__internal__c_update_type(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_req_first_space_before_url; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_store_method: { + switch (llhttp__internal__c_store_method(state, p, endp, match)) { + default: + goto s_n_llhttp__internal__n_invoke_update_type; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error_48: { + state->error = 0x8; + state->reason = "Invalid word encountered"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_type_1: { + switch (llhttp__internal__c_update_type_1(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_res_http_major; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_type_2: { + switch (llhttp__internal__c_update_type(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_start_req; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_pause_8: { + state->error = 0x15; + state->reason = "on_message_begin pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_load_type; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_error: { + state->error = 0x10; + state->reason = "`on_message_begin` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_llhttp__on_message_begin: { + switch (llhttp__on_message_begin(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_load_type; + case 21: + goto s_n_llhttp__internal__n_pause_8; + default: + goto s_n_llhttp__internal__n_error; + } + /* UNREACHABLE */; + abort(); + } + s_n_llhttp__internal__n_invoke_update_finish: { + switch (llhttp__internal__c_update_finish(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__on_message_begin; + } + /* UNREACHABLE */; + abort(); + } +} + +int llhttp__internal_execute(llhttp__internal_t* state, const char* p, const char* endp) { + llparse_state_t next; + + /* check lingering errors */ + if (state->error != 0) { + return state->error; + } + + /* restart spans */ + if (state->_span_pos0 != NULL) { + state->_span_pos0 = (void*) p; + } + + next = llhttp__internal__run(state, (const unsigned char*) p, (const unsigned char*) endp); + if (next == s_error) { + return state->error; + } + state->_current = (void*) (intptr_t) next; + + /* execute spans */ + if (state->_span_pos0 != NULL) { + int error; + + error = ((llhttp__internal__span_cb) state->_span_cb0)(state, state->_span_pos0, (const char*) endp); + if (error != 0) { + state->error = error; + state->error_pos = endp; + return error; + } + } + + return 0; +} + +#endif /* LLHTTP_STRICT_MODE */ diff --git a/src/3rdparty/llhttp/llhttp.h b/src/3rdparty/llhttp/llhttp.h new file mode 100644 index 00000000..a7164103 --- /dev/null +++ b/src/3rdparty/llhttp/llhttp.h @@ -0,0 +1,439 @@ +#ifndef INCLUDE_LLHTTP_H_ +#define INCLUDE_LLHTTP_H_ + +#define LLHTTP_VERSION_MAJOR 4 +#define LLHTTP_VERSION_MINOR 0 +#define LLHTTP_VERSION_PATCH 0 + +#ifndef LLHTTP_STRICT_MODE +# define LLHTTP_STRICT_MODE 0 +#endif + +#ifndef INCLUDE_LLHTTP_ITSELF_H_ +#define INCLUDE_LLHTTP_ITSELF_H_ +#ifdef __cplusplus +extern "C" { +#endif + +#include + +typedef struct llhttp__internal_s llhttp__internal_t; +struct llhttp__internal_s { + int32_t _index; + void* _span_pos0; + void* _span_cb0; + int32_t error; + const char* reason; + const char* error_pos; + void* data; + void* _current; + uint64_t content_length; + uint8_t type; + uint8_t method; + uint8_t http_major; + uint8_t http_minor; + uint8_t header_state; + uint8_t lenient_flags; + uint8_t upgrade; + uint8_t finish; + uint16_t flags; + uint16_t status_code; + void* settings; +}; + +int llhttp__internal_init(llhttp__internal_t* s); +int llhttp__internal_execute(llhttp__internal_t* s, const char* p, const char* endp); + +#ifdef __cplusplus +} /* extern "C" */ +#endif +#endif /* INCLUDE_LLHTTP_ITSELF_H_ */ + +#ifndef LLLLHTTP_C_HEADERS_ +#define LLLLHTTP_C_HEADERS_ +#ifdef __cplusplus +extern "C" { +#endif + +enum llhttp_errno { + HPE_OK = 0, + HPE_INTERNAL = 1, + HPE_STRICT = 2, + HPE_LF_EXPECTED = 3, + HPE_UNEXPECTED_CONTENT_LENGTH = 4, + HPE_CLOSED_CONNECTION = 5, + HPE_INVALID_METHOD = 6, + HPE_INVALID_URL = 7, + HPE_INVALID_CONSTANT = 8, + HPE_INVALID_VERSION = 9, + HPE_INVALID_HEADER_TOKEN = 10, + HPE_INVALID_CONTENT_LENGTH = 11, + HPE_INVALID_CHUNK_SIZE = 12, + HPE_INVALID_STATUS = 13, + HPE_INVALID_EOF_STATE = 14, + HPE_INVALID_TRANSFER_ENCODING = 15, + HPE_CB_MESSAGE_BEGIN = 16, + HPE_CB_HEADERS_COMPLETE = 17, + HPE_CB_MESSAGE_COMPLETE = 18, + HPE_CB_CHUNK_HEADER = 19, + HPE_CB_CHUNK_COMPLETE = 20, + HPE_PAUSED = 21, + HPE_PAUSED_UPGRADE = 22, + HPE_USER = 23 +}; +typedef enum llhttp_errno llhttp_errno_t; + +enum llhttp_flags { + F_CONNECTION_KEEP_ALIVE = 0x1, + F_CONNECTION_CLOSE = 0x2, + F_CONNECTION_UPGRADE = 0x4, + F_CHUNKED = 0x8, + F_UPGRADE = 0x10, + F_CONTENT_LENGTH = 0x20, + F_SKIPBODY = 0x40, + F_TRAILING = 0x80, + F_TRANSFER_ENCODING = 0x200 +}; +typedef enum llhttp_flags llhttp_flags_t; + +enum llhttp_lenient_flags { + LENIENT_HEADERS = 0x1, + LENIENT_CHUNKED_LENGTH = 0x2 +}; +typedef enum llhttp_lenient_flags llhttp_lenient_flags_t; + +enum llhttp_type { + HTTP_BOTH = 0, + HTTP_REQUEST = 1, + HTTP_RESPONSE = 2 +}; +typedef enum llhttp_type llhttp_type_t; + +enum llhttp_finish { + HTTP_FINISH_SAFE = 0, + HTTP_FINISH_SAFE_WITH_CB = 1, + HTTP_FINISH_UNSAFE = 2 +}; +typedef enum llhttp_finish llhttp_finish_t; + +enum llhttp_method { + HTTP_DELETE = 0, + HTTP_GET = 1, + HTTP_HEAD = 2, + HTTP_POST = 3, + HTTP_PUT = 4, + HTTP_CONNECT = 5, + HTTP_OPTIONS = 6, + HTTP_TRACE = 7, + HTTP_COPY = 8, + HTTP_LOCK = 9, + HTTP_MKCOL = 10, + HTTP_MOVE = 11, + HTTP_PROPFIND = 12, + HTTP_PROPPATCH = 13, + HTTP_SEARCH = 14, + HTTP_UNLOCK = 15, + HTTP_BIND = 16, + HTTP_REBIND = 17, + HTTP_UNBIND = 18, + HTTP_ACL = 19, + HTTP_REPORT = 20, + HTTP_MKACTIVITY = 21, + HTTP_CHECKOUT = 22, + HTTP_MERGE = 23, + HTTP_MSEARCH = 24, + HTTP_NOTIFY = 25, + HTTP_SUBSCRIBE = 26, + HTTP_UNSUBSCRIBE = 27, + HTTP_PATCH = 28, + HTTP_PURGE = 29, + HTTP_MKCALENDAR = 30, + HTTP_LINK = 31, + HTTP_UNLINK = 32, + HTTP_SOURCE = 33, + HTTP_PRI = 34, + HTTP_DESCRIBE = 35, + HTTP_ANNOUNCE = 36, + HTTP_SETUP = 37, + HTTP_PLAY = 38, + HTTP_PAUSE = 39, + HTTP_TEARDOWN = 40, + HTTP_GET_PARAMETER = 41, + HTTP_SET_PARAMETER = 42, + HTTP_REDIRECT = 43, + HTTP_RECORD = 44, + HTTP_FLUSH = 45 +}; +typedef enum llhttp_method llhttp_method_t; + +#define HTTP_ERRNO_MAP(XX) \ + XX(0, OK, OK) \ + XX(1, INTERNAL, INTERNAL) \ + XX(2, STRICT, STRICT) \ + XX(3, LF_EXPECTED, LF_EXPECTED) \ + XX(4, UNEXPECTED_CONTENT_LENGTH, UNEXPECTED_CONTENT_LENGTH) \ + XX(5, CLOSED_CONNECTION, CLOSED_CONNECTION) \ + XX(6, INVALID_METHOD, INVALID_METHOD) \ + XX(7, INVALID_URL, INVALID_URL) \ + XX(8, INVALID_CONSTANT, INVALID_CONSTANT) \ + XX(9, INVALID_VERSION, INVALID_VERSION) \ + XX(10, INVALID_HEADER_TOKEN, INVALID_HEADER_TOKEN) \ + XX(11, INVALID_CONTENT_LENGTH, INVALID_CONTENT_LENGTH) \ + XX(12, INVALID_CHUNK_SIZE, INVALID_CHUNK_SIZE) \ + XX(13, INVALID_STATUS, INVALID_STATUS) \ + XX(14, INVALID_EOF_STATE, INVALID_EOF_STATE) \ + XX(15, INVALID_TRANSFER_ENCODING, INVALID_TRANSFER_ENCODING) \ + XX(16, CB_MESSAGE_BEGIN, CB_MESSAGE_BEGIN) \ + XX(17, CB_HEADERS_COMPLETE, CB_HEADERS_COMPLETE) \ + XX(18, CB_MESSAGE_COMPLETE, CB_MESSAGE_COMPLETE) \ + XX(19, CB_CHUNK_HEADER, CB_CHUNK_HEADER) \ + XX(20, CB_CHUNK_COMPLETE, CB_CHUNK_COMPLETE) \ + XX(21, PAUSED, PAUSED) \ + XX(22, PAUSED_UPGRADE, PAUSED_UPGRADE) \ + XX(23, USER, USER) \ + + +#define HTTP_METHOD_MAP(XX) \ + XX(0, DELETE, DELETE) \ + XX(1, GET, GET) \ + XX(2, HEAD, HEAD) \ + XX(3, POST, POST) \ + XX(4, PUT, PUT) \ + XX(5, CONNECT, CONNECT) \ + XX(6, OPTIONS, OPTIONS) \ + XX(7, TRACE, TRACE) \ + XX(8, COPY, COPY) \ + XX(9, LOCK, LOCK) \ + XX(10, MKCOL, MKCOL) \ + XX(11, MOVE, MOVE) \ + XX(12, PROPFIND, PROPFIND) \ + XX(13, PROPPATCH, PROPPATCH) \ + XX(14, SEARCH, SEARCH) \ + XX(15, UNLOCK, UNLOCK) \ + XX(16, BIND, BIND) \ + XX(17, REBIND, REBIND) \ + XX(18, UNBIND, UNBIND) \ + XX(19, ACL, ACL) \ + XX(20, REPORT, REPORT) \ + XX(21, MKACTIVITY, MKACTIVITY) \ + XX(22, CHECKOUT, CHECKOUT) \ + XX(23, MERGE, MERGE) \ + XX(24, MSEARCH, M-SEARCH) \ + XX(25, NOTIFY, NOTIFY) \ + XX(26, SUBSCRIBE, SUBSCRIBE) \ + XX(27, UNSUBSCRIBE, UNSUBSCRIBE) \ + XX(28, PATCH, PATCH) \ + XX(29, PURGE, PURGE) \ + XX(30, MKCALENDAR, MKCALENDAR) \ + XX(31, LINK, LINK) \ + XX(32, UNLINK, UNLINK) \ + XX(33, SOURCE, SOURCE) \ + XX(34, PRI, PRI) \ + XX(35, DESCRIBE, DESCRIBE) \ + XX(36, ANNOUNCE, ANNOUNCE) \ + XX(37, SETUP, SETUP) \ + XX(38, PLAY, PLAY) \ + XX(39, PAUSE, PAUSE) \ + XX(40, TEARDOWN, TEARDOWN) \ + XX(41, GET_PARAMETER, GET_PARAMETER) \ + XX(42, SET_PARAMETER, SET_PARAMETER) \ + XX(43, REDIRECT, REDIRECT) \ + XX(44, RECORD, RECORD) \ + XX(45, FLUSH, FLUSH) \ + + + +#ifdef __cplusplus +} /* extern "C" */ +#endif +#endif /* LLLLHTTP_C_HEADERS_ */ + +#ifndef INCLUDE_LLHTTP_API_H_ +#define INCLUDE_LLHTTP_API_H_ +#ifdef __cplusplus +extern "C" { +#endif +#include + +typedef llhttp__internal_t llhttp_t; +typedef struct llhttp_settings_s llhttp_settings_t; + +typedef int (*llhttp_data_cb)(llhttp_t*, const char *at, size_t length); +typedef int (*llhttp_cb)(llhttp_t*); + +struct llhttp_settings_s { + /* Possible return values 0, -1, `HPE_PAUSED` */ + llhttp_cb on_message_begin; + + llhttp_data_cb on_url; + llhttp_data_cb on_status; + llhttp_data_cb on_header_field; + llhttp_data_cb on_header_value; + + /* Possible return values: + * 0 - Proceed normally + * 1 - Assume that request/response has no body, and proceed to parsing the + * next message + * 2 - Assume absence of body (as above) and make `llhttp_execute()` return + * `HPE_PAUSED_UPGRADE` + * -1 - Error + * `HPE_PAUSED` + */ + llhttp_cb on_headers_complete; + + llhttp_data_cb on_body; + + /* Possible return values 0, -1, `HPE_PAUSED` */ + llhttp_cb on_message_complete; + + /* When on_chunk_header is called, the current chunk length is stored + * in parser->content_length. + * Possible return values 0, -1, `HPE_PAUSED` + */ + llhttp_cb on_chunk_header; + llhttp_cb on_chunk_complete; + + llhttp_cb on_url_complete; + llhttp_cb on_status_complete; + llhttp_cb on_header_field_complete; + llhttp_cb on_header_value_complete; +}; + +/* Initialize the parser with specific type and user settings. + * + * NOTE: lifetime of `settings` has to be at least the same as the lifetime of + * the `parser` here. In practice, `settings` has to be either a static + * variable or be allocated with `malloc`, `new`, etc. + */ +void llhttp_init(llhttp_t* parser, llhttp_type_t type, + const llhttp_settings_t* settings); + +/* Reset an already initialized parser back to the start state, preserving the + * existing parser type, callback settings, user data, and lenient flags. + */ +void llhttp_reset(llhttp_t* parser); + +/* Initialize the settings object */ +void llhttp_settings_init(llhttp_settings_t* settings); + +/* Parse full or partial request/response, invoking user callbacks along the + * way. + * + * If any of `llhttp_data_cb` returns errno not equal to `HPE_OK` - the parsing + * interrupts, and such errno is returned from `llhttp_execute()`. If + * `HPE_PAUSED` was used as a errno, the execution can be resumed with + * `llhttp_resume()` call. + * + * In a special case of CONNECT/Upgrade request/response `HPE_PAUSED_UPGRADE` + * is returned after fully parsing the request/response. If the user wishes to + * continue parsing, they need to invoke `llhttp_resume_after_upgrade()`. + * + * NOTE: if this function ever returns a non-pause type error, it will continue + * to return the same error upon each successive call up until `llhttp_init()` + * is called. + */ +llhttp_errno_t llhttp_execute(llhttp_t* parser, const char* data, size_t len); + +/* This method should be called when the other side has no further bytes to + * send (e.g. shutdown of readable side of the TCP connection.) + * + * Requests without `Content-Length` and other messages might require treating + * all incoming bytes as the part of the body, up to the last byte of the + * connection. This method will invoke `on_message_complete()` callback if the + * request was terminated safely. Otherwise a error code would be returned. + */ +llhttp_errno_t llhttp_finish(llhttp_t* parser); + +/* Returns `1` if the incoming message is parsed until the last byte, and has + * to be completed by calling `llhttp_finish()` on EOF + */ +int llhttp_message_needs_eof(const llhttp_t* parser); + +/* Returns `1` if there might be any other messages following the last that was + * successfully parsed. + */ +int llhttp_should_keep_alive(const llhttp_t* parser); + +/* Make further calls of `llhttp_execute()` return `HPE_PAUSED` and set + * appropriate error reason. + * + * Important: do not call this from user callbacks! User callbacks must return + * `HPE_PAUSED` if pausing is required. + */ +void llhttp_pause(llhttp_t* parser); + +/* Might be called to resume the execution after the pause in user's callback. + * See `llhttp_execute()` above for details. + * + * Call this only if `llhttp_execute()` returns `HPE_PAUSED`. + */ +void llhttp_resume(llhttp_t* parser); + +/* Might be called to resume the execution after the pause in user's callback. + * See `llhttp_execute()` above for details. + * + * Call this only if `llhttp_execute()` returns `HPE_PAUSED_UPGRADE` + */ +void llhttp_resume_after_upgrade(llhttp_t* parser); + +/* Returns the latest return error */ +llhttp_errno_t llhttp_get_errno(const llhttp_t* parser); + +/* Returns the verbal explanation of the latest returned error. + * + * Note: User callback should set error reason when returning the error. See + * `llhttp_set_error_reason()` for details. + */ +const char* llhttp_get_error_reason(const llhttp_t* parser); + +/* Assign verbal description to the returned error. Must be called in user + * callbacks right before returning the errno. + * + * Note: `HPE_USER` error code might be useful in user callbacks. + */ +void llhttp_set_error_reason(llhttp_t* parser, const char* reason); + +/* Returns the pointer to the last parsed byte before the returned error. The + * pointer is relative to the `data` argument of `llhttp_execute()`. + * + * Note: this method might be useful for counting the number of parsed bytes. + */ +const char* llhttp_get_error_pos(const llhttp_t* parser); + +/* Returns textual name of error code */ +const char* llhttp_errno_name(llhttp_errno_t err); + +/* Returns textual name of HTTP method */ +const char* llhttp_method_name(llhttp_method_t method); + + +/* Enables/disables lenient header value parsing (disabled by default). + * + * Lenient parsing disables header value token checks, extending llhttp's + * protocol support to highly non-compliant clients/server. No + * `HPE_INVALID_HEADER_TOKEN` will be raised for incorrect header values when + * lenient parsing is "on". + * + * **(USE AT YOUR OWN RISK)** + */ +void llhttp_set_lenient_headers(llhttp_t* parser, int enabled); + + +/* Enables/disables lenient handling of conflicting `Transfer-Encoding` and + * `Content-Length` headers (disabled by default). + * + * Normally `llhttp` would error when `Transfer-Encoding` is present in + * conjunction with `Content-Length`. This error is important to prevent HTTP + * request smuggling, but may be less desirable for small number of cases + * involving legacy servers. + * + * **(USE AT YOUR OWN RISK)** + */ +void llhttp_set_lenient_chunked_length(llhttp_t* parser, int enabled); + +#ifdef __cplusplus +} /* extern "C" */ +#endif +#endif /* INCLUDE_LLHTTP_API_H_ */ + +#endif /* INCLUDE_LLHTTP_H_ */ diff --git a/src/base/api/Api.cpp b/src/base/api/Api.cpp index 3a71fd9c..53e0ebb1 100644 --- a/src/base/api/Api.cpp +++ b/src/base/api/Api.cpp @@ -1,6 +1,6 @@ /* XMRig - * Copyright 2018-2021 SChernykh - * Copyright 2016-2021 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -21,7 +21,6 @@ #include "base/api/Api.h" -#include "3rdparty/http-parser/http_parser.h" #include "base/api/interfaces/IApiListener.h" #include "base/api/requests/HttpApiRequest.h" #include "base/crypto/keccak.h" @@ -186,7 +185,7 @@ void xmrig::Api::exec(IApiRequest &request) } } - request.done(request.isNew() ? HTTP_STATUS_NOT_FOUND : HTTP_STATUS_OK); + request.done(request.isNew() ? 404 : 200); } diff --git a/src/base/api/Api.h b/src/base/api/Api.h index f412a317..c56e29ee 100644 --- a/src/base/api/Api.h +++ b/src/base/api/Api.h @@ -1,6 +1,6 @@ /* XMRig - * Copyright 2018-2021 SChernykh - * Copyright 2016-2021 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/base/api/Httpd.cpp b/src/base/api/Httpd.cpp index 1aba8796..817da920 100644 --- a/src/base/api/Httpd.cpp +++ b/src/base/api/Httpd.cpp @@ -1,12 +1,6 @@ /* XMRig - * Copyright 2010 Jeff Garzik - * Copyright 2012-2014 pooler - * Copyright 2014 Lucas Jones - * Copyright 2014-2016 Wolf9466 - * Copyright 2016 Jay D Dee - * Copyright 2017-2018 XMR-Stak , - * Copyright 2018-2020 SChernykh - * Copyright 2016-2020 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -22,9 +16,8 @@ * along with this program. If not, see . */ - #include "base/api/Httpd.h" -#include "3rdparty/http-parser/http_parser.h" +#include "3rdparty/llhttp/llhttp.h" #include "base/api/Api.h" #include "base/io/log/Log.h" #include "base/net/http/HttpApiResponse.h" @@ -155,25 +148,25 @@ void xmrig::Httpd::onHttpData(const HttpData &data) } # endif - return HttpResponse(data.id(), HTTP_STATUS_NOT_FOUND).end(); + return HttpResponse(data.id(), 404 /* NOT_FOUND */).end(); } if (data.method > 4) { - return HttpApiResponse(data.id(), HTTP_STATUS_METHOD_NOT_ALLOWED).end(); + return HttpApiResponse(data.id(), 405 /* METHOD_NOT_ALLOWED */).end(); } const int status = auth(data); - if (status != HTTP_STATUS_OK) { + if (status != 200) { return HttpApiResponse(data.id(), status).end(); } if (data.method != HTTP_GET) { if (m_base->config()->http().isRestricted()) { - return HttpApiResponse(data.id(), HTTP_STATUS_FORBIDDEN).end(); + return HttpApiResponse(data.id(), 403 /* FORBIDDEN */).end(); } if (!data.headers.count(HttpData::kContentTypeL) || data.headers.at(HttpData::kContentTypeL) != HttpData::kApplicationJson) { - return HttpApiResponse(data.id(), HTTP_STATUS_UNSUPPORTED_MEDIA_TYPE).end(); + return HttpApiResponse(data.id(), 415 /* UNSUPPORTED_MEDIA_TYPE */).end(); } } @@ -186,19 +179,19 @@ int xmrig::Httpd::auth(const HttpData &req) const const Http &config = m_base->config()->http(); if (!req.headers.count(kAuthorization)) { - return config.isAuthRequired() ? HTTP_STATUS_UNAUTHORIZED : HTTP_STATUS_OK; + return config.isAuthRequired() ? 401 /* UNAUTHORIZED */ : 200; } if (config.token().isNull()) { - return HTTP_STATUS_UNAUTHORIZED; + return 401 /* UNAUTHORIZED */; } const std::string &token = req.headers.at(kAuthorization); const size_t size = token.size(); if (token.size() < 8 || config.token().size() != size - 7 || memcmp("Bearer ", token.c_str(), 7) != 0) { - return HTTP_STATUS_FORBIDDEN; + return 403 /* FORBIDDEN */; } - return strncmp(config.token().data(), token.c_str() + 7, config.token().size()) == 0 ? HTTP_STATUS_OK : HTTP_STATUS_FORBIDDEN; + return strncmp(config.token().data(), token.c_str() + 7, config.token().size()) == 0 ? 200 : 403 /* FORBIDDEN */; } diff --git a/src/base/api/Httpd.h b/src/base/api/Httpd.h index 462a0671..188646f5 100644 --- a/src/base/api/Httpd.h +++ b/src/base/api/Httpd.h @@ -1,12 +1,6 @@ /* XMRig - * Copyright 2010 Jeff Garzik - * Copyright 2012-2014 pooler - * Copyright 2014 Lucas Jones - * Copyright 2014-2016 Wolf9466 - * Copyright 2016 Jay D Dee - * Copyright 2017-2018 XMR-Stak , - * Copyright 2018-2020 SChernykh - * Copyright 2016-2020 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/base/api/requests/HttpApiRequest.cpp b/src/base/api/requests/HttpApiRequest.cpp index 70fa2523..60d42b64 100644 --- a/src/base/api/requests/HttpApiRequest.cpp +++ b/src/base/api/requests/HttpApiRequest.cpp @@ -1,12 +1,6 @@ /* XMRig - * Copyright 2010 Jeff Garzik - * Copyright 2012-2014 pooler - * Copyright 2014 Lucas Jones - * Copyright 2014-2016 Wolf9466 - * Copyright 2016 Jay D Dee - * Copyright 2017-2018 XMR-Stak , - * Copyright 2018-2020 SChernykh - * Copyright 2016-2020 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -24,7 +18,7 @@ #include "base/api/requests/HttpApiRequest.h" -#include "3rdparty/http-parser/http_parser.h" +#include "3rdparty/llhttp/llhttp.h" #include "3rdparty/rapidjson/error/en.h" #include "base/io/json/Json.h" #include "base/net/http/HttpData.h" @@ -53,8 +47,8 @@ static inline const char *rpcError(int code) { return "Invalid params"; } - if (code >= HTTP_STATUS_BAD_REQUEST && code <= HTTP_STATUS_NETWORK_AUTHENTICATION_REQUIRED) { - return http_status_str(static_cast(code)); + if (code >= 400 && code < 600) { + return HttpData::statusName(code); } return "Internal error"; @@ -155,10 +149,10 @@ void xmrig::HttpApiRequest::done(int status) using namespace rapidjson; auto &allocator = doc().GetAllocator(); - m_res.setStatus(HTTP_STATUS_OK); + m_res.setStatus(200); - if (status != HTTP_STATUS_OK) { - setRpcError(status == HTTP_STATUS_NOT_FOUND ? RPC_METHOD_NOT_FOUND : status); + if (status != 200) { + setRpcError(status == 404 /* NOT_FOUND */ ? RPC_METHOD_NOT_FOUND : status); } else if (!reply().HasMember(kResult)) { Value result(kObjectType); @@ -205,6 +199,6 @@ void xmrig::HttpApiRequest::rpcDone(const char *key, rapidjson::Value &value) reply().AddMember("jsonrpc", "2.0", allocator); reply().AddMember(StringRef(kId), Value().CopyFrom(Json::getValue(m_body, kId), allocator), allocator); - m_res.setStatus(HTTP_STATUS_OK); + m_res.setStatus(200); m_res.end(); } diff --git a/src/base/api/requests/HttpApiRequest.h b/src/base/api/requests/HttpApiRequest.h index 2a355582..cb159757 100644 --- a/src/base/api/requests/HttpApiRequest.h +++ b/src/base/api/requests/HttpApiRequest.h @@ -1,12 +1,6 @@ /* XMRig - * Copyright 2010 Jeff Garzik - * Copyright 2012-2014 pooler - * Copyright 2014 Lucas Jones - * Copyright 2014-2016 Wolf9466 - * Copyright 2016 Jay D Dee - * Copyright 2017-2018 XMR-Stak , - * Copyright 2018-2020 SChernykh - * Copyright 2016-2020 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/base/base.cmake b/src/base/base.cmake index cd6ee88b..5949c050 100644 --- a/src/base/base.cmake +++ b/src/base/base.cmake @@ -160,7 +160,7 @@ endif() if (WITH_HTTP) set(HEADERS_BASE_HTTP - src/3rdparty/http-parser/http_parser.h + src/3rdparty/llhttp/llhttp.h src/base/api/Api.h src/base/api/Httpd.h src/base/api/interfaces/IApiRequest.h @@ -181,7 +181,9 @@ if (WITH_HTTP) ) set(SOURCES_BASE_HTTP - src/3rdparty/http-parser/http_parser.c + src/3rdparty/llhttp/llhttp.c + src/3rdparty/llhttp/api.c + src/3rdparty/llhttp/http.c src/base/api/Api.cpp src/base/api/Httpd.cpp src/base/api/requests/ApiRequest.cpp diff --git a/src/base/net/http/Fetch.cpp b/src/base/net/http/Fetch.cpp index 84ff715c..f0be4dfd 100644 --- a/src/base/net/http/Fetch.cpp +++ b/src/base/net/http/Fetch.cpp @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2020 SChernykh - * Copyright (c) 2016-2020 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -30,7 +30,7 @@ #endif -xmrig::FetchRequest::FetchRequest(http_method method, const String &host, uint16_t port, const String &path, bool tls, bool quiet, const char *data, size_t size, const char *contentType) : +xmrig::FetchRequest::FetchRequest(llhttp_method method, const String &host, uint16_t port, const String &path, bool tls, bool quiet, const char *data, size_t size, const char *contentType) : quiet(quiet), tls(tls), method(method), @@ -44,7 +44,7 @@ xmrig::FetchRequest::FetchRequest(http_method method, const String &host, uint16 } -xmrig::FetchRequest::FetchRequest(http_method method, const String &host, uint16_t port, const String &path, const rapidjson::Value &value, bool tls, bool quiet) : +xmrig::FetchRequest::FetchRequest(llhttp_method method, const String &host, uint16_t port, const String &path, const rapidjson::Value &value, bool tls, bool quiet) : quiet(quiet), tls(tls), method(method), @@ -99,7 +99,7 @@ void xmrig::fetch(const char *tag, FetchRequest &&req, const std::weak_ptr(req.body.size()), req.body.c_str()); diff --git a/src/base/net/http/Fetch.h b/src/base/net/http/Fetch.h index bdab46ba..6c02e348 100644 --- a/src/base/net/http/Fetch.h +++ b/src/base/net/http/Fetch.h @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2020 SChernykh - * Copyright (c) 2016-2020 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -21,7 +21,7 @@ #define XMRIG_FETCH_H -#include "3rdparty/http-parser/http_parser.h" +#include "3rdparty/llhttp/llhttp.h" #include "3rdparty/rapidjson/fwd.h" #include "base/tools/String.h" @@ -41,24 +41,24 @@ class FetchRequest { public: FetchRequest() = default; - FetchRequest(http_method method, const String &host, uint16_t port, const String &path, bool tls = false, bool quiet = false, const char *data = nullptr, size_t size = 0, const char *contentType = nullptr); - FetchRequest(http_method method, const String &host, uint16_t port, const String &path, const rapidjson::Value &value, bool tls = false, bool quiet = false); + FetchRequest(llhttp_method method, const String &host, uint16_t port, const String &path, bool tls = false, bool quiet = false, const char *data = nullptr, size_t size = 0, const char *contentType = nullptr); + FetchRequest(llhttp_method method, const String &host, uint16_t port, const String &path, const rapidjson::Value &value, bool tls = false, bool quiet = false); void setBody(const char *data, size_t size, const char *contentType = nullptr); void setBody(const rapidjson::Value &value); inline bool hasBody() const { return method != HTTP_GET && method != HTTP_HEAD && !body.empty(); } - bool quiet = false; - bool tls = false; - http_method method = HTTP_GET; + bool quiet = false; + bool tls = false; + llhttp_method method = HTTP_GET; std::map headers; std::string body; String fingerprint; String host; String path; - uint16_t port = 0; - uint64_t timeout = 0; + uint16_t port = 0; + uint64_t timeout = 0; }; diff --git a/src/base/net/http/HttpApiResponse.cpp b/src/base/net/http/HttpApiResponse.cpp index fed3df53..e6758df9 100644 --- a/src/base/net/http/HttpApiResponse.cpp +++ b/src/base/net/http/HttpApiResponse.cpp @@ -1,13 +1,7 @@ /* XMRig - * Copyright 2010 Jeff Garzik - * Copyright 2012-2014 pooler - * Copyright 2014 Lucas Jones - * Copyright 2014-2016 Wolf9466 - * Copyright 2016 Jay D Dee - * Copyright 2017-2018 XMR-Stak , - * Copyright 2014-2019 heapwolf - * Copyright 2018-2020 SChernykh - * Copyright 2016-2020 XMRig , + * Copyright (c) 2014-2019 heapwolf + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -25,7 +19,6 @@ #include "base/net/http/HttpApiResponse.h" -#include "3rdparty/http-parser/http_parser.h" #include "3rdparty/rapidjson/prettywriter.h" #include "3rdparty/rapidjson/stringbuffer.h" #include "base/net/http/HttpData.h" @@ -68,7 +61,7 @@ void xmrig::HttpApiResponse::end() } if (!m_doc.HasMember(kError)) { - m_doc.AddMember(StringRef(kError), StringRef(http_status_str(static_cast(statusCode()))), m_doc.GetAllocator()); + m_doc.AddMember(StringRef(kError), StringRef(HttpData::statusName(statusCode())), m_doc.GetAllocator()); } } diff --git a/src/base/net/http/HttpApiResponse.h b/src/base/net/http/HttpApiResponse.h index 8a782510..ac71c5e0 100644 --- a/src/base/net/http/HttpApiResponse.h +++ b/src/base/net/http/HttpApiResponse.h @@ -1,13 +1,7 @@ /* XMRig - * Copyright 2010 Jeff Garzik - * Copyright 2012-2014 pooler - * Copyright 2014 Lucas Jones - * Copyright 2014-2016 Wolf9466 - * Copyright 2016 Jay D Dee - * Copyright 2017-2018 XMR-Stak , - * Copyright 2014-2019 heapwolf - * Copyright 2018-2020 SChernykh - * Copyright 2016-2020 XMRig , + * Copyright (c) 2014-2019 heapwolf + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -23,7 +17,6 @@ * along with this program. If not, see . */ - #ifndef XMRIG_HTTPAPIRESPONSE_H #define XMRIG_HTTPAPIRESPONSE_H diff --git a/src/base/net/http/HttpClient.cpp b/src/base/net/http/HttpClient.cpp index 320f8ee4..ea0d68c8 100644 --- a/src/base/net/http/HttpClient.cpp +++ b/src/base/net/http/HttpClient.cpp @@ -1,7 +1,7 @@ /* XMRig * Copyright (c) 2014-2019 heapwolf - * Copyright (c) 2018-2020 SChernykh - * Copyright (c) 2016-2020 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,7 +19,7 @@ #include "base/net/http/HttpClient.h" -#include "3rdparty/http-parser/http_parser.h" +#include "3rdparty/llhttp/llhttp.h" #include "base/io/log/Log.h" #include "base/kernel/Platform.h" #include "base/net/dns/Dns.h" @@ -102,7 +102,7 @@ void xmrig::HttpClient::handshake() } std::stringstream ss; - ss << http_method_str(static_cast(method)) << " " << url << " HTTP/1.1" << kCRLF; + ss << llhttp_method_name(static_cast(method)) << " " << url << " HTTP/1.1" << kCRLF; for (auto &header : headers) { ss << header.first << ": " << header.second << kCRLF; diff --git a/src/base/net/http/HttpClient.h b/src/base/net/http/HttpClient.h index 2b9a314d..d012d26f 100644 --- a/src/base/net/http/HttpClient.h +++ b/src/base/net/http/HttpClient.h @@ -1,7 +1,7 @@ /* XMRig * Copyright (c) 2014-2019 heapwolf - * Copyright (c) 2018-2020 SChernykh - * Copyright (c) 2016-2020 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/base/net/http/HttpContext.cpp b/src/base/net/http/HttpContext.cpp index d56db0c8..92c54c68 100644 --- a/src/base/net/http/HttpContext.cpp +++ b/src/base/net/http/HttpContext.cpp @@ -19,7 +19,7 @@ #include "base/net/http/HttpContext.h" -#include "3rdparty/http-parser/http_parser.h" +#include "3rdparty/llhttp/llhttp.h" #include "base/kernel/interfaces/IHttpListener.h" #include "base/tools/Baton.h" #include "base/tools/Chrono.h" @@ -32,7 +32,7 @@ namespace xmrig { -static http_parser_settings http_settings; +static llhttp_settings_t http_settings; static std::map storage; static uint64_t SEQUENCE = 0; @@ -78,13 +78,13 @@ xmrig::HttpContext::HttpContext(int parser_type, const std::weak_ptr(parser_type)); + llhttp_init(m_parser, static_cast(parser_type), &http_settings); m_parser->data = m_tcp->data = this; @@ -124,7 +124,7 @@ size_t xmrig::HttpContext::parse(const char *data, size_t size) return size; } - return http_parser_execute(m_parser, &http_settings, data, size); + return llhttp_execute(m_parser, data, size); } @@ -191,7 +191,7 @@ void xmrig::HttpContext::closeAll() } -int xmrig::HttpContext::onHeaderField(http_parser *parser, const char *at, size_t length) +int xmrig::HttpContext::onHeaderField(llhttp_t *parser, const char *at, size_t length) { auto ctx = static_cast(parser->data); @@ -210,7 +210,7 @@ int xmrig::HttpContext::onHeaderField(http_parser *parser, const char *at, size_ } -int xmrig::HttpContext::onHeaderValue(http_parser *parser, const char *at, size_t length) +int xmrig::HttpContext::onHeaderValue(llhttp_t *parser, const char *at, size_t length) { auto ctx = static_cast(parser->data); @@ -225,14 +225,14 @@ int xmrig::HttpContext::onHeaderValue(http_parser *parser, const char *at, size_ } -void xmrig::HttpContext::attach(http_parser_settings *settings) +void xmrig::HttpContext::attach(llhttp_settings_t *settings) { settings->on_message_begin = nullptr; settings->on_status = nullptr; settings->on_chunk_header = nullptr; settings->on_chunk_complete = nullptr; - settings->on_url = [](http_parser *parser, const char *at, size_t length) -> int + settings->on_url = [](llhttp_t *parser, const char *at, size_t length) -> int { static_cast(parser->data)->url = std::string(at, length); return 0; @@ -241,7 +241,7 @@ void xmrig::HttpContext::attach(http_parser_settings *settings) settings->on_header_field = onHeaderField; settings->on_header_value = onHeaderValue; - settings->on_headers_complete = [](http_parser* parser) -> int { + settings->on_headers_complete = [](llhttp_t *parser) -> int { auto ctx = static_cast(parser->data); ctx->status = parser->status_code; @@ -256,14 +256,14 @@ void xmrig::HttpContext::attach(http_parser_settings *settings) return 0; }; - settings->on_body = [](http_parser *parser, const char *at, size_t len) -> int + settings->on_body = [](llhttp_t *parser, const char *at, size_t len) -> int { static_cast(parser->data)->body.append(at, len); return 0; }; - settings->on_message_complete = [](http_parser *parser) -> int + settings->on_message_complete = [](llhttp_t *parser) -> int { auto ctx = static_cast(parser->data); auto listener = ctx->httpListener(); diff --git a/src/base/net/http/HttpContext.h b/src/base/net/http/HttpContext.h index 9c719333..81248f28 100644 --- a/src/base/net/http/HttpContext.h +++ b/src/base/net/http/HttpContext.h @@ -22,8 +22,8 @@ #define XMRIG_HTTPCONTEXT_H -using http_parser = struct http_parser; -using http_parser_settings = struct http_parser_settings; +using llhttp_settings_t = struct llhttp_settings_s; +using llhttp_t = struct llhttp__internal_s; using uv_connect_t = struct uv_connect_s; using uv_handle_t = struct uv_handle_s; using uv_stream_t = struct uv_stream_s; @@ -76,15 +76,15 @@ protected: private: inline IHttpListener *httpListener() const { return m_listener.expired() ? nullptr : m_listener.lock().get(); } - static int onHeaderField(http_parser *parser, const char *at, size_t length); - static int onHeaderValue(http_parser *parser, const char *at, size_t length); - static void attach(http_parser_settings *settings); + static int onHeaderField(llhttp_t *parser, const char *at, size_t length); + static int onHeaderValue(llhttp_t *parser, const char *at, size_t length); + static void attach(llhttp_settings_t *settings); void setHeader(); bool m_wasHeaderValue = false; const uint64_t m_timestamp; - http_parser *m_parser; + llhttp_t *m_parser; std::string m_lastHeaderField; std::string m_lastHeaderValue; std::weak_ptr m_listener; diff --git a/src/base/net/http/HttpData.cpp b/src/base/net/http/HttpData.cpp index 10eace46..b243118c 100644 --- a/src/base/net/http/HttpData.cpp +++ b/src/base/net/http/HttpData.cpp @@ -1,7 +1,7 @@ /* XMRig * Copyright (c) 2014-2019 heapwolf - * Copyright (c) 2018-2020 SChernykh - * Copyright (c) 2016-2020 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,7 +19,7 @@ #include "base/net/http/HttpData.h" -#include "3rdparty/http-parser/http_parser.h" +#include "3rdparty/llhttp/llhttp.h" #include "3rdparty/rapidjson/document.h" #include "3rdparty/rapidjson/error/en.h" #include "base/io/json/Json.h" @@ -29,6 +29,76 @@ #include +/* Status Codes */ +#define HTTP_STATUS_MAP(XX) \ + XX(100, CONTINUE, Continue) \ + XX(101, SWITCHING_PROTOCOLS, Switching Protocols) \ + XX(102, PROCESSING, Processing) \ + XX(200, OK, OK) \ + XX(201, CREATED, Created) \ + XX(202, ACCEPTED, Accepted) \ + XX(203, NON_AUTHORITATIVE_INFORMATION, Non-Authoritative Information) \ + XX(204, NO_CONTENT, No Content) \ + XX(205, RESET_CONTENT, Reset Content) \ + XX(206, PARTIAL_CONTENT, Partial Content) \ + XX(207, MULTI_STATUS, Multi-Status) \ + XX(208, ALREADY_REPORTED, Already Reported) \ + XX(226, IM_USED, IM Used) \ + XX(300, MULTIPLE_CHOICES, Multiple Choices) \ + XX(301, MOVED_PERMANENTLY, Moved Permanently) \ + XX(302, FOUND, Found) \ + XX(303, SEE_OTHER, See Other) \ + XX(304, NOT_MODIFIED, Not Modified) \ + XX(305, USE_PROXY, Use Proxy) \ + XX(307, TEMPORARY_REDIRECT, Temporary Redirect) \ + XX(308, PERMANENT_REDIRECT, Permanent Redirect) \ + XX(400, BAD_REQUEST, Bad Request) \ + XX(401, UNAUTHORIZED, Unauthorized) \ + XX(402, PAYMENT_REQUIRED, Payment Required) \ + XX(403, FORBIDDEN, Forbidden) \ + XX(404, NOT_FOUND, Not Found) \ + XX(405, METHOD_NOT_ALLOWED, Method Not Allowed) \ + XX(406, NOT_ACCEPTABLE, Not Acceptable) \ + XX(407, PROXY_AUTHENTICATION_REQUIRED, Proxy Authentication Required) \ + XX(408, REQUEST_TIMEOUT, Request Timeout) \ + XX(409, CONFLICT, Conflict) \ + XX(410, GONE, Gone) \ + XX(411, LENGTH_REQUIRED, Length Required) \ + XX(412, PRECONDITION_FAILED, Precondition Failed) \ + XX(413, PAYLOAD_TOO_LARGE, Payload Too Large) \ + XX(414, URI_TOO_LONG, URI Too Long) \ + XX(415, UNSUPPORTED_MEDIA_TYPE, Unsupported Media Type) \ + XX(416, RANGE_NOT_SATISFIABLE, Range Not Satisfiable) \ + XX(417, EXPECTATION_FAILED, Expectation Failed) \ + XX(421, MISDIRECTED_REQUEST, Misdirected Request) \ + XX(422, UNPROCESSABLE_ENTITY, Unprocessable Entity) \ + XX(423, LOCKED, Locked) \ + XX(424, FAILED_DEPENDENCY, Failed Dependency) \ + XX(426, UPGRADE_REQUIRED, Upgrade Required) \ + XX(428, PRECONDITION_REQUIRED, Precondition Required) \ + XX(429, TOO_MANY_REQUESTS, Too Many Requests) \ + XX(431, REQUEST_HEADER_FIELDS_TOO_LARGE, Request Header Fields Too Large) \ + XX(451, UNAVAILABLE_FOR_LEGAL_REASONS, Unavailable For Legal Reasons) \ + XX(500, INTERNAL_SERVER_ERROR, Internal Server Error) \ + XX(501, NOT_IMPLEMENTED, Not Implemented) \ + XX(502, BAD_GATEWAY, Bad Gateway) \ + XX(503, SERVICE_UNAVAILABLE, Service Unavailable) \ + XX(504, GATEWAY_TIMEOUT, Gateway Timeout) \ + XX(505, HTTP_VERSION_NOT_SUPPORTED, HTTP Version Not Supported) \ + XX(506, VARIANT_ALSO_NEGOTIATES, Variant Also Negotiates) \ + XX(507, INSUFFICIENT_STORAGE, Insufficient Storage) \ + XX(508, LOOP_DETECTED, Loop Detected) \ + XX(510, NOT_EXTENDED, Not Extended) \ + XX(511, NETWORK_AUTHENTICATION_REQUIRED, Network Authentication Required) \ + +enum http_status +{ +# define XX(num, name, string) HTTP_STATUS_##name = num, + HTTP_STATUS_MAP(XX) +# undef XX +}; + + namespace xmrig { @@ -38,6 +108,17 @@ const std::string HttpData::kContentTypeL = "content-type"; const std::string HttpData::kTextPlain = "text/plain"; +static const char *http_status_str(enum http_status s) +{ + switch (s) { +# define XX(num, name, string) case HTTP_STATUS_##name: return #string; + HTTP_STATUS_MAP(XX) +# undef XX + default: return ""; + } +} + + } // namespace xmrig @@ -55,17 +136,7 @@ bool xmrig::HttpData::isJSON() const const char *xmrig::HttpData::methodName() const { - return http_method_str(static_cast(method)); -} - - -const char *xmrig::HttpData::statusName() const -{ - if (status < 0) { - return uv_strerror(status); - } - - return http_status_str(static_cast(status)); + return llhttp_method_name(static_cast(method)); } @@ -94,3 +165,13 @@ rapidjson::Document xmrig::HttpData::json() const return doc; } + + +const char *xmrig::HttpData::statusName(int status) +{ + if (status < 0) { + return uv_strerror(status); + } + + return http_status_str(static_cast(status)); +} diff --git a/src/base/net/http/HttpData.h b/src/base/net/http/HttpData.h index 392c8bc8..3337999a 100644 --- a/src/base/net/http/HttpData.h +++ b/src/base/net/http/HttpData.h @@ -1,7 +1,7 @@ /* XMRig * Copyright (c) 2014-2019 heapwolf - * Copyright (c) 2018-2020 SChernykh - * Copyright (c) 2016-2020 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -47,7 +47,8 @@ public: inline HttpData(uint64_t id) : m_id(id) {} virtual ~HttpData() = default; - inline uint64_t id() const { return m_id; } + inline const char *statusName() const { return statusName(status); } + inline uint64_t id() const { return m_id; } virtual bool isRequest() const = 0; virtual const char *host() const = 0; @@ -59,9 +60,10 @@ public: bool isJSON() const; const char *methodName() const; - const char *statusName() const; rapidjson::Document json() const; + static const char *statusName(int status); + int method = 0; int status = 0; int userType = 0; diff --git a/src/base/net/http/HttpListener.cpp b/src/base/net/http/HttpListener.cpp index ba9130ed..695e0f82 100644 --- a/src/base/net/http/HttpListener.cpp +++ b/src/base/net/http/HttpListener.cpp @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2020 SChernykh - * Copyright (c) 2016-2020 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,7 +18,7 @@ #include "base/net/http/HttpListener.h" -#include "3rdparty/http-parser/http_parser.h" +#include "3rdparty/llhttp/llhttp.h" #include "base/io/log/Log.h" #include "base/net/http/HttpData.h" @@ -28,7 +28,7 @@ void xmrig::HttpListener::onHttpData(const HttpData &data) # ifdef APP_DEBUG if (!data.isRequest()) { LOG_DEBUG("%s " CYAN_BOLD("http%s://%s:%u ") MAGENTA_BOLD("\"%s %s\" ") CSI "1;%dm%d" CLEAR BLACK_BOLD(" received: ") CYAN_BOLD("%zu") BLACK_BOLD(" bytes"), - m_tag, data.tlsVersion() ? "s" : "", data.host(), data.port(), http_method_str(static_cast(data.method)), data.url.data(), + m_tag, data.tlsVersion() ? "s" : "", data.host(), data.port(), llhttp_method_name(static_cast(data.method)), data.url.data(), (data.status >= 400 || data.status < 0) ? 31 : 32, data.status, data.body.size()); if (data.body.size() < (Log::kMaxBufferSize - 1024) && data.isJSON()) { diff --git a/src/base/net/http/HttpListener.h b/src/base/net/http/HttpListener.h index 8f86d2ba..3dcc9a0e 100644 --- a/src/base/net/http/HttpListener.h +++ b/src/base/net/http/HttpListener.h @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2020 SChernykh - * Copyright (c) 2016-2020 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/base/net/http/HttpResponse.cpp b/src/base/net/http/HttpResponse.cpp index 89e22431..216e3e51 100644 --- a/src/base/net/http/HttpResponse.cpp +++ b/src/base/net/http/HttpResponse.cpp @@ -1,13 +1,7 @@ /* XMRig - * Copyright 2010 Jeff Garzik - * Copyright 2012-2014 pooler - * Copyright 2014 Lucas Jones - * Copyright 2014-2016 Wolf9466 - * Copyright 2016 Jay D Dee - * Copyright 2017-2018 XMR-Stak , - * Copyright 2014-2019 heapwolf - * Copyright 2018-2020 SChernykh - * Copyright 2016-2020 XMRig , + * Copyright (c) 2014-2019 heapwolf + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -25,7 +19,7 @@ #include "base/net/http/HttpResponse.h" -#include "3rdparty/http-parser/http_parser.h" +#include "3rdparty/llhttp/llhttp.h" #include "base/io/log/Log.h" #include "base/net/http/HttpContext.h" @@ -78,7 +72,7 @@ void xmrig::HttpResponse::end(const char *data, size_t size) setHeader("Connection", "close"); std::stringstream ss; - ss << "HTTP/1.1 " << statusCode() << " " << http_status_str(static_cast(statusCode())) << kCRLF; + ss << "HTTP/1.1 " << statusCode() << " " << HttpData::statusName(statusCode()) << kCRLF; for (auto &header : m_headers) { ss << header.first << ": " << header.second << kCRLF; @@ -97,7 +91,7 @@ void xmrig::HttpResponse::end(const char *data, size_t size) Log::print(err ? Log::ERR : Log::INFO, CYAN("%s ") CLEAR MAGENTA_BOLD("%s") WHITE_BOLD(" %s ") CSI "1;%dm%d " CLEAR WHITE_BOLD("%zu ") CYAN_BOLD("%" PRIu64 "ms ") BLACK_BOLD("\"%s\""), ctx->ip().c_str(), - http_method_str(static_cast(ctx->method)), + llhttp_method_name(static_cast(ctx->method)), ctx->url.c_str(), err ? 31 : 32, statusCode(), diff --git a/src/base/net/http/HttpResponse.h b/src/base/net/http/HttpResponse.h index 1891ad7e..a6b44997 100644 --- a/src/base/net/http/HttpResponse.h +++ b/src/base/net/http/HttpResponse.h @@ -1,13 +1,7 @@ /* XMRig - * Copyright 2010 Jeff Garzik - * Copyright 2012-2014 pooler - * Copyright 2014 Lucas Jones - * Copyright 2014-2016 Wolf9466 - * Copyright 2016 Jay D Dee - * Copyright 2017-2018 XMR-Stak , - * Copyright 2014-2019 heapwolf - * Copyright 2018-2020 SChernykh - * Copyright 2016-2020 XMRig , + * Copyright (c) 2014-2019 heapwolf + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -23,7 +17,6 @@ * along with this program. If not, see . */ - #ifndef XMRIG_HTTPRESPONSE_H #define XMRIG_HTTPRESPONSE_H diff --git a/src/base/net/https/HttpsContext.cpp b/src/base/net/https/HttpsContext.cpp index 8db31812..ba38e18d 100644 --- a/src/base/net/https/HttpsContext.cpp +++ b/src/base/net/https/HttpsContext.cpp @@ -1,6 +1,6 @@ /* XMRig - * Copyright 2018-2020 SChernykh - * Copyright 2016-2020 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,7 +18,7 @@ #include "base/net/https/HttpsContext.h" -#include "3rdparty/http-parser/http_parser.h" +#include "3rdparty/llhttp/llhttp.h" #include "base/net/tls/TlsContext.h" diff --git a/src/base/net/https/HttpsContext.h b/src/base/net/https/HttpsContext.h index af57b6b4..ff060f02 100644 --- a/src/base/net/https/HttpsContext.h +++ b/src/base/net/https/HttpsContext.h @@ -1,6 +1,6 @@ /* XMRig - * Copyright 2018-2020 SChernykh - * Copyright 2016-2020 XMRig , + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/base/net/stratum/DaemonClient.cpp b/src/base/net/stratum/DaemonClient.cpp index 07cc7360..087fd0f7 100644 --- a/src/base/net/stratum/DaemonClient.cpp +++ b/src/base/net/stratum/DaemonClient.cpp @@ -25,7 +25,6 @@ #include "base/net/stratum/DaemonClient.h" -#include "3rdparty/http-parser/http_parser.h" #include "3rdparty/rapidjson/document.h" #include "3rdparty/rapidjson/error/en.h" #include "base/io/json/Json.h" @@ -151,7 +150,7 @@ void xmrig::DaemonClient::connect(const Pool &pool) void xmrig::DaemonClient::onHttpData(const HttpData &data) { - if (data.status != HTTP_STATUS_OK) { + if (data.status != 200) { return retry(); } diff --git a/src/base/net/stratum/SelfSelectClient.cpp b/src/base/net/stratum/SelfSelectClient.cpp index 2dca057a..7f7ee544 100644 --- a/src/base/net/stratum/SelfSelectClient.cpp +++ b/src/base/net/stratum/SelfSelectClient.cpp @@ -20,7 +20,6 @@ #include "base/net/stratum/SelfSelectClient.h" -#include "3rdparty/http-parser/http_parser.h" #include "3rdparty/rapidjson/document.h" #include "3rdparty/rapidjson/error/en.h" #include "base/io/json/Json.h" @@ -288,7 +287,7 @@ void xmrig::SelfSelectClient::submitOriginDaemon(const JobResult& result) void xmrig::SelfSelectClient::onHttpData(const HttpData &data) { - if (data.status != HTTP_STATUS_OK) { + if (data.status != 200) { return retry(); } From 856c8e6bcdee1c9402bc7c495b8a7be3d5198693 Mon Sep 17 00:00:00 2001 From: XMRig Date: Sat, 6 Mar 2021 02:07:10 +0700 Subject: [PATCH 14/16] Fixed build without TLS support. --- src/base/net/http/HttpServer.cpp | 14 ++++---------- src/base/net/http/HttpServer.h | 17 +++-------------- 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/src/base/net/http/HttpServer.cpp b/src/base/net/http/HttpServer.cpp index 80377f77..3b7e6ef1 100644 --- a/src/base/net/http/HttpServer.cpp +++ b/src/base/net/http/HttpServer.cpp @@ -1,13 +1,7 @@ /* XMRig - * Copyright 2010 Jeff Garzik - * Copyright 2012-2014 pooler - * Copyright 2014 Lucas Jones - * Copyright 2014-2016 Wolf9466 - * Copyright 2016 Jay D Dee - * Copyright 2017-2018 XMR-Stak , - * Copyright 2014-2019 heapwolf - * Copyright 2018-2020 SChernykh - * Copyright 2016-2020 XMRig , + * Copyright (c) 2014-2019 heapwolf + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -29,7 +23,7 @@ #include "base/net/http/HttpServer.h" -#include "3rdparty/http-parser/http_parser.h" +#include "3rdparty/llhttp/llhttp.h" #include "base/net/http/HttpContext.h" #include "base/net/tools/NetBuffer.h" diff --git a/src/base/net/http/HttpServer.h b/src/base/net/http/HttpServer.h index 2ced767f..e0da35dd 100644 --- a/src/base/net/http/HttpServer.h +++ b/src/base/net/http/HttpServer.h @@ -1,13 +1,7 @@ /* XMRig - * Copyright 2010 Jeff Garzik - * Copyright 2012-2014 pooler - * Copyright 2014 Lucas Jones - * Copyright 2014-2016 Wolf9466 - * Copyright 2016 Jay D Dee - * Copyright 2017-2018 XMR-Stak , - * Copyright 2014-2019 heapwolf - * Copyright 2018-2020 SChernykh - * Copyright 2016-2020 XMRig , + * Copyright (c) 2014-2019 heapwolf + * Copyright (c) 2018-2021 SChernykh + * Copyright (c) 2016-2021 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -23,15 +17,10 @@ * along with this program. If not, see . */ - #ifndef XMRIG_HTTPSERVER_H #define XMRIG_HTTPSERVER_H -using http_parser = struct http_parser; -using http_parser_settings = struct http_parser_settings; - - #include "base/kernel/interfaces/ITcpServerListener.h" #include "base/tools/Object.h" From f5db50c9d715508b5864c37740f563b638d07a22 Mon Sep 17 00:00:00 2001 From: XMRig Date: Sat, 6 Mar 2021 05:32:54 +0700 Subject: [PATCH 15/16] Sync with the proxy. --- src/base/net/stratum/Job.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/base/net/stratum/Job.cpp b/src/base/net/stratum/Job.cpp index acd5a0d6..134a5064 100644 --- a/src/base/net/stratum/Job.cpp +++ b/src/base/net/stratum/Job.cpp @@ -119,8 +119,10 @@ bool xmrig::Job::setTarget(const char *target) } # ifdef XMRIG_PROXY_PROJECT + assert(sizeof(m_rawTarget) > (size * 2)); + memset(m_rawTarget, 0, sizeof(m_rawTarget)); - memcpy(m_rawTarget, target, len); + memcpy(m_rawTarget, target, std::min(size * 2, sizeof(m_rawTarget))); # endif m_diff = toDiff(m_target); @@ -134,8 +136,7 @@ void xmrig::Job::setDiff(uint64_t diff) m_target = toDiff(diff); # ifdef XMRIG_PROXY_PROJECT - Buffer::toHex(reinterpret_cast(&m_target), 8, m_rawTarget); - m_rawTarget[16] = '\0'; + Cvt::toHex(m_rawTarget, sizeof(m_rawTarget), reinterpret_cast(&m_target), sizeof(m_target)); # endif } From b8f9a326aaec8ff7789d8f9cbe2af8f994149e8a Mon Sep 17 00:00:00 2001 From: XMRig Date: Sun, 7 Mar 2021 01:44:38 +0700 Subject: [PATCH 16/16] 6.10.0-dev --- CHANGELOG.md | 10 ++++++++++ src/version.h | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a137cf9..989d193a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +# v6.10.0 +- [#2122](https://github.com/xmrig/xmrig/pull/2122) Fixed pause logic when both pause on battery and user activity are enabled. +- [#2123](https://github.com/xmrig/xmrig/issues/2123) Fixed compatibility with gcc 4.8. +- [#2147](https://github.com/xmrig/xmrig/pull/2147) Fixed many `new job` messages when solo mining. +- [#2150](https://github.com/xmrig/xmrig/pull/2150) Updated `sse2neon.h` to the latest master, fixes build on ARMv7. +- [#2157](https://github.com/xmrig/xmrig/pull/2157) Fixed crash in `cn-heavy` on Zen3 with manual thread count. +- Fixed possible out of order write to log file. +- [http-parser](https://github.com/nodejs/http-parser) replaced to [llhttp](https://github.com/nodejs/llhttp). +- For official builds: libuv, hwloc and OpenSSL updated to latest versions. + # v6.9.0 - [#2104](https://github.com/xmrig/xmrig/pull/2104) Added [pause-on-active](https://xmrig.com/docs/miner/config/misc#pause-on-active) config option and `--pause-on-active=N` command line option. - [#2112](https://github.com/xmrig/xmrig/pull/2112) Added support for [Tari merge mining](https://github.com/tari-project/tari/blob/development/README.md#tari-merge-mining). diff --git a/src/version.h b/src/version.h index 2eb246d1..703079c2 100644 --- a/src/version.h +++ b/src/version.h @@ -28,15 +28,15 @@ #define APP_ID "xmrig" #define APP_NAME "XMRig" #define APP_DESC "XMRig miner" -#define APP_VERSION "6.9.1-dev" +#define APP_VERSION "6.10.0-dev" #define APP_DOMAIN "xmrig.com" #define APP_SITE "www.xmrig.com" #define APP_COPYRIGHT "Copyright (C) 2016-2021 xmrig.com" #define APP_KIND "miner" #define APP_VER_MAJOR 6 -#define APP_VER_MINOR 9 -#define APP_VER_PATCH 1 +#define APP_VER_MINOR 10 +#define APP_VER_PATCH 0 #ifdef _MSC_VER # if (_MSC_VER >= 1920)