diff options
84 files changed, 32146 insertions, 0 deletions
diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..29cdc70 --- /dev/null +++ b/.clang-format @@ -0,0 +1,129 @@ +AccessModifierOffset: -4 +AlignAfterOpenBracket: AlwaysBreak +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: DontAlign +AlignOperands: true +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Inline +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterReturnType: TopLevel +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: true +BinPackArguments: false +BinPackParameters: false +BraceWrapping: + AfterClass: true + AfterControlStatement: true + AfterEnum: true + AfterFunction: true + AfterNamespace: false + AfterStruct: true + AfterUnion: true + AfterExternBlock: false + BeforeCatch: true + BeforeElse: true +BreakBeforeBraces: Custom +BreakBeforeBinaryOperators: None +BreakBeforeTernaryOperators: true +BreakConstructorInitializers: AfterColon +BreakStringLiterals: false +ColumnLimit: 120 +CommentPragmas: '^begin_wpp|^end_wpp|^FUNC |^USESUFFIX |^USESUFFIX ' +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: false +DerivePointerAlignment: false +ExperimentalAutoDetectBinPacking: false +IndentCaseLabels: false +IndentPPDirectives: AfterHash +IndentWidth: 4 +KeepEmptyLinesAtTheStartOfBlocks: false +Language: Cpp +MacroBlockBegin: '^BEGIN_MODULE$|^BEGIN_TEST_CLASS$|^BEGIN_TEST_METHOD$' +MacroBlockEnd: '^END_MODULE$|^END_TEST_CLASS$|^END_TEST_METHOD$' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +PointerAlignment: Right +ReflowComments: true +SortIncludes: false +SpaceAfterCStyleCast: false +SpaceBeforeAssignmentOperators: true +SpaceBeforeCtorInitializerColon: true +SpaceBeforeCtorInitializerColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyParentheses: false +SpacesInAngles: false +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Cpp11 +StatementMacros: [ + '__drv_allocatesMem', + '__drv_freesMem', + '__drv_preferredFunction', + '__drv_setsIRQL', + '_Acquires_exclusive_lock_', + '_Acquires_lock_', + '_Acquires_shared_lock_', + '_Acquires_rcu_', + '_At_', + '_At_buffer_', + '_Check_return_', + '_Dispatch_type_', + '_Field_size_bytes_', + '_Function_class_', + '_Guarded_by_', + '_IRQL_lowers_', + '_IRQL_raises_', + '_IRQL_requires_', + '_IRQL_requires_max_', + '_IRQL_requires_min_', + '_IRQL_requires_same_', + '_IRQL_restores_', + '_IRQL_restores_global_', + '_IRQL_saves_', + '_IRQL_saves_global_', + '_Must_inspect_result_', + '_Out_writes_bytes_all_', + '_Post_equals_last_error_', + '_Post_maybenull_', + '_Post_notnull_', + '_Post_writable_byte_size_', + '_Releases_exclusive_lock_', + '_Releases_lock_', + '_Releases_shared_lock_', + '_Releases_rcu_', + '_Requires_shared_lock_held_', + '_Requires_exclusive_lock_held_', + '_Requires_lock_held_', + '_Requires_lock_not_held_', + '_Requires_rcu_held_', + '_Ret_bytecount_', + '_Ret_maybenull_', + '_Ret_range_', + '_Ret_writes_bytes_', + '_Return_type_success_', + '_Success_', + '_Use_decl_annotations_', + '_When_', + 'EXTERN_C', + 'INITCODE', + 'NONPAGED', + 'PAGED', + 'PAGEDX', + 'PNPCODE' + ] +TabWidth: '4' +UseTab: Never +ForEachMacros: + - 'LIST_FOR_EACH_ENTRY' + - 'LIST_FOR_EACH_ENTRY_SAFE' + - 'HLIST_FOR_EACH_ENTRY' + - 'HLIST_FOR_EACH_ENTRY_SAFE' + - 'HLIST_FOR_EACH_ENTRY_RCU' diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..d5f42f1 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,7 @@ +[*.{c,h,inf,rc}] +indent_style = space +indent_size = 4 + +[*.{proj,props,vcxproj,filters}] +indent_style = space +indent_size = 2 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..febf51b --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +# Visual Studio Local Files +.vs/ +*.user + +# Build Output +/dist +/Debug +/Release + +# Static Driver Verifier Output +/driver/sdv +/driver/smvbuild.log +/driver/smvstats.txt + +# Driver Verification Log +/driver/wireguard.DVL.XML + +# Temporary files +*~ @@ -0,0 +1,341 @@ +This program is Copyright (C) 2018-2021 WireGuard LLC. +------------------------------------------------------------------------------ + + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License version 2 + as published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..312c157 --- /dev/null +++ b/README.md @@ -0,0 +1,79 @@ +# [WireGuard](https://www.wireguard.com/) for the NT Kernel +### High performance in-kernel WireGuard implementation for Windows + +WireGuardNT is an implementation of WireGuard, for the NT Kernel as used in Windows 7, 8, 8.1, and 10, supporting AMD64, x86, ARM64, and ARM processors. + +#### Not the droids you're looking for + +**If you've come here looking to run [WireGuard on Windows](https://git.zx2c4.com/wireguard-windows/about/), you're in the wrong place. Instead, head on over to the [WireGuard Download Page](https://www.wireguard.com/install/) to download the WireGuard application.** Alternatively, if you've come here looking to embed WireGuard into your Windows program, **you are still in the wrong place**. Instead, head on over to the [embeddable DLL service project](https://git.zx2c4.com/wireguard-windows/about/embeddable-dll-service/README.md), to get everything you need to bake WireGuard into your Windows programs. These projects use (or will eventually use) WireGuardNT inside. + +#### Experimental, unfinished, work in progress + +This repository is under active development, is not yet complete, and is not *yet* properly optimized for high performance. Do not use it! In addition to the usual concerns about it *not yet being secure*, it's also not even *finished*. So if you do attempt to drive this off the lot, realize that a wheel or two are likely missing, in addition to, perhaps, the entire crankshaft, and the radio dials haven't yet been painted and the seatbelts are made of Fruit by the Foot. + +## Usage + +#### Download + +WireGuardNT is deployed as a platform-specific `wireguard.dll` file. Install the `wireguard.dll` file side-by-side with your application. Download the dll from [the wireguard-nt download server](https://download.wireguard.com/wireguard-nt/), alongside the header file for your application described below. + +#### API + +Include the [`wireguard.h` file](https://git.zx2c4.com/wireguard-nt/tree/api/wireguard.h) in your project simply by copying it there and dynamically load the `wireguard.dll` using [`LoadLibraryEx()`](https://docs.microsoft.com/en-us/windows/win32/api/libloaderapi/nf-libloaderapi-loadlibraryexa) and [`GetProcAddress()`](https://docs.microsoft.com/en-us/windows/win32/api/libloaderapi/nf-libloaderapi-getprocaddress) to resolve each function, using the typedefs provided in the header file. The [`InitializeWireGuardNT` function in the example.c code](https://git.zx2c4.com/wireguard-nt/tree/example/example.c) provides this in a function that you can simply copy and paste. + +With the library setup, WireGuardNT can then be used by first creating an adapter, configuring it, and then setting its status to "up". Adapters have names (e.g. "OfficeNet"), and each one belongs to a _pool_ (e.g. "WireGuard"). So, for example, the WireGuard application app creates multiple tunnels all inside of its "WireGuard" _pool_: + +```C +WIREGUARD_ADAPTER_HANDLE Adapter1 = WireGuardCreateAdapter(L"WireGuard", L"OfficeNet", &SomeFixedGUID1, NULL); +WIREGUARD_ADAPTER_HANDLE Adapter2 = WireGuardCreateAdapter(L"WireGuard", L"HomeNet", &SomeFixedGUID2, NULL); +WIREGUARD_ADAPTER_HANDLE Adapter3 = WireGuardCreateAdapter(L"WireGuard", L"Data Center", &SomeFixedGUID3, NULL); +``` + +After creating an adapter, we can use it by setting a configuration and setting its status to "up": + +```C +struct +{ + WIREGUARD_INTERFACE Interface; + WIREGUARD_PEER FirstPeer; + WIREGUARD_ALLOWED_IP FirstPeerAllowedIP1; + WIREGUARD_ALLOWED_IP FirstPeerAllowedIP2; + WIREGUARD_PEER SecondPeer; + WIREGUARD_ALLOWED_IP SecondtPeerAllowedIP1; +} Config = { + .Interface = { + .Flags = WIREGUARD_INTERFACE_HAS_PRIVATE_KEY, + .PrivateKey = ..., + .PeersCount = 2 + }, + .FirstPeer = { + .Flags = WIREGUARD_PEER_HAS_PUBLIC_KEY | WIREGUARD_PEER_HAS_ENDPOINT, + .PublicKey = ..., + .Endpoint = ..., + .AllowedIPsCount = 2 + }, + .FirstPeerAllowedIP1 = { ... }, + ... +}; +WireGuardSetConfiguration(Adapter1, &Config.Interface, sizeof(Config)); +WireGuardSetAdapterState(Adapter1, WIREGUARD_ADAPTER_STATE_UP); +``` + +You are *highly encouraged* to read the [**example.c short example**](https://git.zx2c4.com/wireguard-nt/tree/example/example.c) to see how to put together a simple network tunnel. The example one connects to the [demo server](https://demo.wireguard.com/). + +The various functions and definitions are [documented in `wireguard.h`](https://git.zx2c4.com/wireguard-nt/tree/api/wireguard.h). + +## Building + +**Do not distribute drivers or files named "WireGuard" or "wireguard" or similar, as they will most certainly clash with official deployments. Instead distribute [`wireguard.dll` as downloaded from the wireguard-nt download server](https://download.wireguard.com/wireguard-nt/).** + +General requirements: + +- [Visual Studio 2019](https://visualstudio.microsoft.com/downloads/) with Windows SDK +- [Windows Driver Kit](https://docs.microsoft.com/en-us/windows-hardware/drivers/download-the-wdk) + +`wireguard-nt.sln` may be opened in Visual Studio for development and building. Be sure to run `bcdedit /set testsigning on` and then reboot before to enable unsigned driver loading. The default run sequence (F5) in Visual Studio will build the example project and its dependencies. + +## License + +The entire contents of [this repository](https://git.zx2c4.com/wireguard-nt/), including all documentation and example code, is "Copyright © 2018-2021 WireGuard LLC. All Rights Reserved." Source code is licensed under the [GPLv2](COPYING). Prebuilt binaries from [the wireguard-nt download server](https://download.wireguard.com/wireguard-nt/) are released under a more permissive license suitable for more forms of software contained inside of the .zip files distributed there. @@ -0,0 +1,42 @@ +# TODO List + +## Driver + +### Performance + +### Rearrange and regroup `WG_DEVICE` +- Most commonly used members should be at the top +- Members used together should be next to each other +- Holes should be minimized +- Don't stretch cachelines + +### SAL Annotations +- Figure out `__rcu` best practice with SAL +- For the MuAcquireReleaseSharedExclusive functions, make SAL detect lock + imbalance. e.g. a AcquireExclusive followed by a ReleaseShared. + +### Remove `_NO_CRT_STDIO_INLINE` once WDK is fixed + +### Make SDV work with full settings + +### Figure out how to make wg-loop routing work safely + +### Implement ICMP reply on error path +- Not totally essential, but nice to have +- One hard part is that we're post-NAT, and we can't dip down into the ct tuple + like we do on linux, which could make the general concept impossible + +### Implement socket route cache + +### Automate CodeQL +Reference: https://docs.microsoft.com/sl-si/windows-hardware/drivers/devtest/static-tools-and-codeql +- Download CodeQL and unzip => .deps +- git clone --recurse https://github.com/microsoft/Windows-Driver-Developer-Supplemental-Tools.git => .deps +- rd /s codeqldb +- codeql.cmd database create -l=cpp -s=driver -c "msbuild driver.vcxproj /t:Rebuild" codeqldb -j 0 +- codeql.cmd database analyze codeqldb windows_driver_recommended.qls --search-path=..\Windows-Driver-Developer-Supplemental-Tools --format=sarifv2.1.0 --output=driver\wireguard.sarif -j 0 + +### WHQL + +#### DVL and Static Tools Logo Test +- Recent (E)WDK DVL always includes CodeQL test results. Even if "NORUN". WHQL 1809 does not support CodeQL test results in DVL and fails Static Tools Logo Test. Those two are in conflict. Either downgrade (E)WDK, or upgrade WHQL rig. diff --git a/api/adapter.c b/api/adapter.c new file mode 100644 index 0000000..3f07319 --- /dev/null +++ b/api/adapter.c @@ -0,0 +1,2000 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#include <WinSock2.h> +#include <Windows.h> +#include <winternl.h> +#include <wincrypt.h> +#include <cfgmgr32.h> +#include <devguid.h> +#include <ws2tcpip.h> +#include <iphlpapi.h> +#include <ndisguid.h> +#include <SetupAPI.h> +#include <Shlwapi.h> +#include <shellapi.h> +#include <wchar.h> +#include <initguid.h> /* Keep these two at bottom in this order, so that we only generate extra GUIDs for devpkey. The other keys we'll get from uuid.lib like usual. */ +#include <devpkey.h> + +#include "../driver/ioctl.h" +#include "adapter.h" +#include "logger.h" +#include "main.h" +#include "namespace.h" +#include "nci.h" +#include "ntdll.h" +#include "registry.h" +#include "resource.h" +#include "rundll32.h" +#include "wireguard-inf.h" + +#pragma warning(disable : 4221) /* nonstandard: address of automatic in initializer */ + +#define WAIT_FOR_REGISTRY_TIMEOUT 10000 /* ms */ +#define MAX_POOL_DEVICE_TYPE (WIREGUARD_MAX_POOL + 8) /* Should accommodate a pool name with " Tunnel" appended */ + +static const DEVPROPKEY DEVPKEY_WireGuard_Pool = { + { 0x65726957, 0x7547, 0x7261, { 0x64, 0x50, 0x6f, 0x6f, 0x6c, 0x4b, 0x65, 0x79 } }, + DEVPROPID_FIRST_USABLE + 0 +}; + +static const DEVPROPKEY DEVPKEY_WireGuard_Name = { + { 0x65726957, 0x7547, 0x7261, { 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x4b, 0x65, 0x79 } }, + DEVPROPID_FIRST_USABLE + 1 +}; + +typedef struct _SP_DEVINFO_DATA_LIST +{ + SP_DEVINFO_DATA Data; + VOID *Configuration; + DWORD ConfigurationBytes; + WIREGUARD_ADAPTER_STATE AdapterState; + struct _SP_DEVINFO_DATA_LIST *Next; +} SP_DEVINFO_DATA_LIST; + +_Must_inspect_result_ +static _Return_type_success_(return != NULL) +_Post_maybenull_ +SP_DRVINFO_DETAIL_DATA_W * +GetAdapterDrvInfoDetail( + _In_ HDEVINFO DevInfo, + _In_opt_ SP_DEVINFO_DATA *DevInfoData, + _In_ SP_DRVINFO_DATA_W *DrvInfoData) +{ + DWORD Size = sizeof(SP_DRVINFO_DETAIL_DATA_W) + 0x100; + for (;;) + { + SP_DRVINFO_DETAIL_DATA_W *DrvInfoDetailData = Alloc(Size); + if (!DrvInfoDetailData) + return NULL; + DrvInfoDetailData->cbSize = sizeof(SP_DRVINFO_DETAIL_DATA_W); + if (SetupDiGetDriverInfoDetailW(DevInfo, DevInfoData, DrvInfoData, DrvInfoDetailData, Size, &Size)) + return DrvInfoDetailData; + DWORD LastError = GetLastError(); + Free(DrvInfoDetailData); + if (LastError != ERROR_INSUFFICIENT_BUFFER) + { + if (DevInfoData) + LOG_ERROR(LastError, L"Failed for adapter %u", DevInfoData->DevInst); + else + LOG_ERROR(LastError, L"Failed"); + SetLastError(LastError); + return NULL; + } + } +} + +_Must_inspect_result_ +static _Return_type_success_(return != NULL) +_Post_maybenull_ +_Post_writable_byte_size_(*BufLen) +VOID * +GetDeviceRegistryProperty( + _In_ HDEVINFO DevInfo, + _In_ SP_DEVINFO_DATA *DevInfoData, + _In_ DWORD Property, + _Out_opt_ DWORD *ValueType, + _Inout_ DWORD *BufLen) +{ + for (;;) + { + BYTE *Data = Alloc(*BufLen); + if (!Data) + return NULL; + if (SetupDiGetDeviceRegistryPropertyW(DevInfo, DevInfoData, Property, ValueType, Data, *BufLen, BufLen)) + return Data; + DWORD LastError = GetLastError(); + Free(Data); + if (LastError != ERROR_INSUFFICIENT_BUFFER) + { + SetLastError( + LOG_ERROR(LastError, L"Failed to query adapter %u property 0x%x", DevInfoData->DevInst, Property)); + return NULL; + } + } +} + +_Must_inspect_result_ +static _Return_type_success_(return != NULL) +_Post_maybenull_ +LPWSTR +GetDeviceRegistryString(_In_ HDEVINFO DevInfo, _In_ SP_DEVINFO_DATA *DevInfoData, _In_ DWORD Property) +{ + DWORD LastError, ValueType, Size = 256 * sizeof(WCHAR); + LPWSTR Buf = GetDeviceRegistryProperty(DevInfo, DevInfoData, Property, &ValueType, &Size); + if (!Buf) + return NULL; + switch (ValueType) + { + case REG_SZ: + case REG_EXPAND_SZ: + case REG_MULTI_SZ: + if (RegistryGetString(&Buf, Size / sizeof(*Buf), ValueType)) + return Buf; + LastError = GetLastError(); + break; + default: + LOG(WIREGUARD_LOG_ERR, + L"Adapter %u property 0x%x is not a string (type: %u)", + DevInfoData->DevInst, + Property, + ValueType); + LastError = ERROR_INVALID_DATATYPE; + } + Free(Buf); + SetLastError(LastError); + return NULL; +} + +_Must_inspect_result_ +static _Return_type_success_(return != NULL) +_Post_maybenull_ +PZZWSTR +GetDeviceRegistryMultiString(_In_ HDEVINFO DevInfo, _In_ SP_DEVINFO_DATA *DevInfoData, _In_ DWORD Property) +{ + DWORD LastError, ValueType, Size = 256 * sizeof(WCHAR); + PZZWSTR Buf = GetDeviceRegistryProperty(DevInfo, DevInfoData, Property, &ValueType, &Size); + if (!Buf) + return NULL; + switch (ValueType) + { + case REG_SZ: + case REG_EXPAND_SZ: + case REG_MULTI_SZ: + if (RegistryGetMultiString(&Buf, Size / sizeof(*Buf), ValueType)) + return Buf; + LastError = GetLastError(); + break; + default: + LOG(WIREGUARD_LOG_ERR, + L"Adapter %u property 0x%x is not a string (type: %u)", + DevInfoData->DevInst, + Property, + ValueType); + LastError = ERROR_INVALID_DATATYPE; + } + Free(Buf); + SetLastError(LastError); + return NULL; +} + +static BOOL +IsOurHardwareID(_In_z_ PCZZWSTR Hwids) +{ + for (; Hwids[0]; Hwids += wcslen(Hwids) + 1) + if (!_wcsicmp(Hwids, WIREGUARD_HWID)) + return TRUE; + return FALSE; +} + +static BOOL +IsOurAdapter(_In_ HDEVINFO DevInfo, _In_ SP_DEVINFO_DATA *DevInfoData) +{ + PZZWSTR Hwids = GetDeviceRegistryMultiString(DevInfo, DevInfoData, SPDRP_HARDWAREID); + if (!Hwids) + { + LOG_LAST_ERROR(L"Failed to get adapter %u hardware ID", DevInfoData->DevInst); + return FALSE; + } + BOOL IsOurs = IsOurHardwareID(Hwids); + Free(Hwids); + return IsOurs; +} + +_Must_inspect_result_ +static _Return_type_success_(return != NULL) +_Post_maybenull_ +LPWSTR +GetDeviceObjectFileName(_In_z_ LPCWSTR InstanceId) +{ + ULONG InterfacesLen; + DWORD LastError = CM_MapCrToWin32Err( + CM_Get_Device_Interface_List_SizeW( + &InterfacesLen, + (GUID *)&GUID_DEVINTERFACE_NET, + (DEVINSTID_W)InstanceId, + CM_GET_DEVICE_INTERFACE_LIST_PRESENT), + ERROR_GEN_FAILURE); + if (LastError != ERROR_SUCCESS) + { + SetLastError(LOG_ERROR(LastError, L"Failed to query adapter %s associated instances size", InstanceId)); + return NULL; + } + LPWSTR Interfaces = AllocArray(InterfacesLen, sizeof(*Interfaces)); + if (!Interfaces) + return NULL; + LastError = CM_MapCrToWin32Err( + CM_Get_Device_Interface_ListW( + (GUID *)&GUID_DEVINTERFACE_NET, + (DEVINSTID_W)InstanceId, + Interfaces, + InterfacesLen, + CM_GET_DEVICE_INTERFACE_LIST_PRESENT), + ERROR_GEN_FAILURE); + if (LastError != ERROR_SUCCESS) + { + LOG_ERROR(LastError, L"Failed to get adapter %s associated instances", InstanceId); + Free(Interfaces); + SetLastError(LastError); + return NULL; + } + if (!Interfaces[0]) + { + Free(Interfaces); + SetLastError(ERROR_DEVICE_NOT_AVAILABLE); + return NULL; + } + return Interfaces; +} + +_Must_inspect_result_ +static _Return_type_success_(return != INVALID_HANDLE_VALUE) +HANDLE +OpenDeviceObject(_In_z_ LPCWSTR InstanceId) +{ + LPWSTR Filename = GetDeviceObjectFileName(InstanceId); + if (!Filename) + return INVALID_HANDLE_VALUE; + HANDLE Handle = CreateFileW( + Filename, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, + OPEN_EXISTING, + 0, + NULL); + if (Handle == INVALID_HANDLE_VALUE) + LOG_LAST_ERROR(L"Failed to connect to adapter %s associated instance %s", InstanceId, Filename); + Free(Filename); + return Handle; +} + +static BOOL +EnsureDeviceObject(_In_z_ LPCWSTR InstanceId) +{ + LPWSTR Filename = GetDeviceObjectFileName(InstanceId); + if (!Filename) + { + LOG_LAST_ERROR(L"Failed to determine adapter %s device object", InstanceId); + return FALSE; + } + BOOL Exists = TRUE; + const int Attempts = 100; + for (int i = 0; i < Attempts; ++i) + { + HANDLE Handle = CreateFileW(Filename, 0, 0, NULL, OPEN_EXISTING, 0, NULL); + if (Handle != INVALID_HANDLE_VALUE) + { + CloseHandle(Handle); + goto out; + } + if (i != Attempts - 1) + Sleep(50); + } + Exists = FALSE; + LOG_LAST_ERROR(L"Failed to connect to adapter %s associated instance %s", InstanceId, Filename); +out: + Free(Filename); + return Exists; +} + +static _Return_type_success_(return != FALSE) +BOOL +SnapshotConfigurationAndState( + _In_ HDEVINFO DevInfo, + _In_ SP_DEVINFO_DATA *DevInfoData, + _Out_ VOID **Configuration, + _Out_ DWORD *ConfigurationBytes, + _Out_ WIREGUARD_ADAPTER_STATE *State) +{ + DWORD LastError = ERROR_SUCCESS; + DWORD RequiredBytes; + if (SetupDiGetDeviceInstanceIdW(DevInfo, DevInfoData, NULL, 0, &RequiredBytes) || + (LastError = GetLastError()) != ERROR_INSUFFICIENT_BUFFER) + { + LOG_ERROR(LastError, L"Failed to query adapter %u instance ID size", DevInfoData->DevInst); + return FALSE; + } + LastError = ERROR_SUCCESS; + LPWSTR InstanceId = ZallocArray(RequiredBytes, sizeof(*InstanceId)); + if (!InstanceId) + return FALSE; + if (!SetupDiGetDeviceInstanceIdW(DevInfo, DevInfoData, InstanceId, RequiredBytes, &RequiredBytes)) + { + LastError = LOG_LAST_ERROR(L"Failed to get adapter %u instance ID", DevInfoData->DevInst); + goto cleanupInstanceId; + } + HANDLE NdisHandle = OpenDeviceObject(InstanceId); + if (NdisHandle == INVALID_HANDLE_VALUE) + { + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to get adapter %u object", DevInfoData->DevInst); + goto cleanupInstanceId; + } + WG_IOCTL_ADAPTER_STATE Op = WG_IOCTL_ADAPTER_STATE_QUERY; + if (!DeviceIoControl( + NdisHandle, WG_IOCTL_SET_ADAPTER_STATE, &Op, sizeof(Op), State, sizeof(*State), &RequiredBytes, NULL)) + { + LastError = LOG_LAST_ERROR(L"Failed to query adapter state on adapter %u", DevInfoData->DevInst); + goto cleanupHandle; + } + for (RequiredBytes = 512;; RequiredBytes = *ConfigurationBytes) + { + *Configuration = Alloc(RequiredBytes); + if (!*Configuration) + { + LastError = LOG_LAST_ERROR( + L"Failed to allocate %u bytes for configuration on adapter %u", RequiredBytes, DevInfoData->DevInst); + goto cleanupHandle; + } + if (DeviceIoControl(NdisHandle, WG_IOCTL_GET, NULL, 0, *Configuration, RequiredBytes, ConfigurationBytes, NULL)) + break; + Free(*Configuration); + *Configuration = NULL; + if (GetLastError() != ERROR_MORE_DATA) + { + LastError = LOG_LAST_ERROR(L"Failed to query configuration on adapter %u", DevInfoData->DevInst); + goto cleanupHandle; + } + } +cleanupHandle: + CloseHandle(NdisHandle); +cleanupInstanceId: + Free(InstanceId); + return RET_ERROR(TRUE, LastError); +} + +static _Return_type_success_(return != FALSE) +BOOL +RestoreConfigurationAndState( + _In_ HDEVINFO DevInfo, + _In_ SP_DEVINFO_DATA *DevInfoData, + _In_ VOID *Configuration, + _In_ DWORD ConfigurationBytes, + _In_ WIREGUARD_ADAPTER_STATE State) +{ + DWORD LastError = ERROR_SUCCESS; + DWORD RequiredBytes; + if (SetupDiGetDeviceInstanceIdW(DevInfo, DevInfoData, NULL, 0, &RequiredBytes) || + (LastError = GetLastError()) != ERROR_INSUFFICIENT_BUFFER) + { + LOG_ERROR(LastError, L"Failed to query adapter %u instance ID size", DevInfoData->DevInst); + return FALSE; + } + LastError = ERROR_SUCCESS; + LPWSTR InstanceId = ZallocArray(RequiredBytes, sizeof(*InstanceId)); + if (!InstanceId) + return FALSE; + if (!SetupDiGetDeviceInstanceIdW(DevInfo, DevInfoData, InstanceId, RequiredBytes, &RequiredBytes)) + { + LastError = LOG_LAST_ERROR(L"Failed to get adapter %u instance ID", DevInfoData->DevInst); + goto cleanupInstanceId; + } + HANDLE NdisHandle = OpenDeviceObject(InstanceId); + if (NdisHandle == INVALID_HANDLE_VALUE) + { + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to get adapter %u object", DevInfoData->DevInst); + goto cleanupInstanceId; + } + if (!DeviceIoControl(NdisHandle, WG_IOCTL_SET, NULL, 0, Configuration, ConfigurationBytes, &RequiredBytes, NULL)) + { + LastError = LOG_LAST_ERROR(L"Failed to set configuration on adapter %u", DevInfoData->DevInst); + goto cleanupHandle; + } + if (!DeviceIoControl(NdisHandle, WG_IOCTL_SET_ADAPTER_STATE, &State, sizeof(State), NULL, 0, &RequiredBytes, NULL)) + { + LastError = LOG_LAST_ERROR(L"Failed to set adapter state on adapter %u", DevInfoData->DevInst); + goto cleanupHandle; + } +cleanupHandle: + CloseHandle(NdisHandle); +cleanupInstanceId: + Free(InstanceId); + return RET_ERROR(TRUE, LastError); +} + +static _Return_type_success_(return != FALSE) +BOOL +DisableAllOurAdapters(_In_ HDEVINFO DevInfo, _Inout_ SP_DEVINFO_DATA_LIST **DisabledAdapters) +{ + SP_PROPCHANGE_PARAMS Params = { .ClassInstallHeader = { .cbSize = sizeof(SP_CLASSINSTALL_HEADER), + .InstallFunction = DIF_PROPERTYCHANGE }, + .StateChange = DICS_DISABLE, + .Scope = DICS_FLAG_GLOBAL }; + DWORD LastError = ERROR_SUCCESS; + for (DWORD EnumIndex = 0;; ++EnumIndex) + { + SP_DEVINFO_DATA_LIST *DeviceNode = Zalloc(sizeof(*DeviceNode)); + if (!DeviceNode) + return FALSE; + DeviceNode->Data.cbSize = sizeof(SP_DEVINFO_DATA); + if (!SetupDiEnumDeviceInfo(DevInfo, EnumIndex, &DeviceNode->Data)) + { + if (GetLastError() == ERROR_NO_MORE_ITEMS) + { + Free(DeviceNode); + break; + } + goto cleanupDeviceNode; + } + if (!IsOurAdapter(DevInfo, &DeviceNode->Data)) + goto cleanupDeviceNode; + + ULONG Status, ProblemCode; + if (CM_Get_DevNode_Status(&Status, &ProblemCode, DeviceNode->Data.DevInst, 0) != CR_SUCCESS || + ((Status & DN_HAS_PROBLEM) && ProblemCode == CM_PROB_DISABLED)) + goto cleanupDeviceNode; + + LOG(WIREGUARD_LOG_INFO, L"Snapshotting configuration of adapter %u", DeviceNode->Data.DevInst); + if (!SnapshotConfigurationAndState( + DevInfo, + &DeviceNode->Data, + &DeviceNode->Configuration, + &DeviceNode->ConfigurationBytes, + &DeviceNode->AdapterState)) + LOG(WIREGUARD_LOG_WARN, L"Failed to snapshot configuration of adapter %u", DeviceNode->Data.DevInst); + + LOG(WIREGUARD_LOG_INFO, L"Disabling adapter %u", DeviceNode->Data.DevInst); + if (!SetupDiSetClassInstallParamsW(DevInfo, &DeviceNode->Data, &Params.ClassInstallHeader, sizeof(Params)) || + !SetupDiCallClassInstaller(DIF_PROPERTYCHANGE, DevInfo, &DeviceNode->Data)) + { + LOG_LAST_ERROR(L"Failed to disable adapter %u", DeviceNode->Data.DevInst); + LastError = LastError != ERROR_SUCCESS ? LastError : GetLastError(); + goto cleanupDeviceNode; + } + + DeviceNode->Next = *DisabledAdapters; + *DisabledAdapters = DeviceNode; + continue; + + cleanupDeviceNode: + Free(DeviceNode); + } + return RET_ERROR(TRUE, LastError); +} + +static _Return_type_success_(return != FALSE) +BOOL +EnableAllOurAdapters(_In_ HDEVINFO DevInfo, _In_ SP_DEVINFO_DATA_LIST *AdaptersToEnable) +{ + SP_PROPCHANGE_PARAMS Params = { .ClassInstallHeader = { .cbSize = sizeof(SP_CLASSINSTALL_HEADER), + .InstallFunction = DIF_PROPERTYCHANGE }, + .StateChange = DICS_ENABLE, + .Scope = DICS_FLAG_GLOBAL }; + DWORD LastError = ERROR_SUCCESS; + for (SP_DEVINFO_DATA_LIST *DeviceNode = AdaptersToEnable; DeviceNode; DeviceNode = DeviceNode->Next) + { + LOG(WIREGUARD_LOG_INFO, L"Enabling adapter %u", DeviceNode->Data.DevInst); + if (!SetupDiSetClassInstallParamsW(DevInfo, &DeviceNode->Data, &Params.ClassInstallHeader, sizeof(Params)) || + !SetupDiCallClassInstaller(DIF_PROPERTYCHANGE, DevInfo, &DeviceNode->Data)) + { + LOG_LAST_ERROR(L"Failed to enable adapter %u", DeviceNode->Data.DevInst); + LastError = LastError != ERROR_SUCCESS ? LastError : GetLastError(); + } + LOG(WIREGUARD_LOG_INFO, L"Restoring configuration of adapter %u", DeviceNode->Data.DevInst); + if (!RestoreConfigurationAndState( + DevInfo, + &DeviceNode->Data, + DeviceNode->Configuration, + DeviceNode->ConfigurationBytes, + DeviceNode->AdapterState)) + LOG(WIREGUARD_LOG_WARN, L"Failed to restore configuration of adapter %u", DeviceNode->Data.DevInst); + } + return RET_ERROR(TRUE, LastError); +} + +static BOOL +CheckReboot(_In_ HDEVINFO DevInfo, _In_ SP_DEVINFO_DATA *DevInfoData) +{ + SP_DEVINSTALL_PARAMS_W DevInstallParams = { .cbSize = sizeof(SP_DEVINSTALL_PARAMS_W) }; + if (!SetupDiGetDeviceInstallParamsW(DevInfo, DevInfoData, &DevInstallParams)) + { + LOG_LAST_ERROR(L"Failed to retrieve adapter %u device installation parameters", DevInfoData->DevInst); + return FALSE; + } + SetLastError(ERROR_SUCCESS); + return (DevInstallParams.Flags & (DI_NEEDREBOOT | DI_NEEDRESTART)) != 0; +} + +_Must_inspect_result_ +static _Return_type_success_(return != FALSE) +BOOL +GetPoolDeviceTypeName(_In_z_ LPCWSTR Pool, _Out_writes_z_(MAX_POOL_DEVICE_TYPE) LPWSTR Name) +{ + if (_snwprintf_s(Name, MAX_POOL_DEVICE_TYPE, _TRUNCATE, L"%s Tunnel", Pool) == -1) + { + LOG(WIREGUARD_LOG_ERR, L"Pool name too long: %s", Pool); + SetLastError(ERROR_INVALID_PARAMETER); + return FALSE; + } + return TRUE; +} + +static BOOL +IsPoolMember(_In_z_ LPCWSTR Pool, _In_ HDEVINFO DevInfo, _In_ SP_DEVINFO_DATA *DevInfoData) +{ + WCHAR PoolProp[MAX_POOL_DEVICE_TYPE]; + DEVPROPTYPE PropType; + if (!SetupDiGetDevicePropertyW( + DevInfo, DevInfoData, &DEVPKEY_WireGuard_Pool, &PropType, (PBYTE)PoolProp, sizeof(PoolProp), NULL, 0)) + return FALSE; + if (PropType != DEVPROP_TYPE_STRING) + { + SetLastError(ERROR_BAD_DEVICE); + return FALSE; + } + SetLastError(ERROR_SUCCESS); + return !_wcsicmp(PoolProp, Pool); +} + +_Must_inspect_result_ +static _Return_type_success_(return != FALSE) +BOOL +PopulateAdapterData(_Inout_ WIREGUARD_ADAPTER *Adapter, _In_z_ LPCWSTR Pool) +{ + DWORD LastError = ERROR_SUCCESS; + + /* Open HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Class\<class>\<id> registry key. */ + HKEY Key = + SetupDiOpenDevRegKey(Adapter->DevInfo, &Adapter->DevInfoData, DICS_FLAG_GLOBAL, 0, DIREG_DRV, KEY_QUERY_VALUE); + if (Key == INVALID_HANDLE_VALUE) + { + LOG_LAST_ERROR(L"Failed to open adapter %u device registry key", Adapter->DevInfoData.DevInst); + return FALSE; + } + + LPWSTR ValueStr = RegistryQueryString(Key, L"NetCfgInstanceId", TRUE); + if (!ValueStr) + { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(Key, RegPath); + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to get %.*s\\NetCfgInstanceId", MAX_REG_PATH, RegPath); + goto cleanupKey; + } + if (FAILED(CLSIDFromString(ValueStr, &Adapter->CfgInstanceID))) + { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(Key, RegPath); + LastError = + LOG(WIREGUARD_LOG_ERR, L"%.*s\\NetCfgInstanceId is not a GUID: %s", MAX_REG_PATH, RegPath, ValueStr); + Free(ValueStr); + goto cleanupKey; + } + Free(ValueStr); + + if (!RegistryQueryDWORD(Key, L"NetLuidIndex", &Adapter->LuidIndex, TRUE)) + { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(Key, RegPath); + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to get %.*s\\NetLuidIndex", MAX_REG_PATH, RegPath); + goto cleanupKey; + } + + if (!RegistryQueryDWORD(Key, L"*IfType", &Adapter->IfType, TRUE)) + { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(Key, RegPath); + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to get %.*s\\*IfType", MAX_REG_PATH, RegPath); + goto cleanupKey; + } + + DWORD Size; + if (!SetupDiGetDeviceInstanceIdW( + Adapter->DevInfo, &Adapter->DevInfoData, Adapter->DevInstanceID, _countof(Adapter->DevInstanceID), &Size)) + { + LastError = LOG_LAST_ERROR(L"Failed to get adapter %u instance ID", Adapter->DevInfoData.DevInst); + goto cleanupKey; + } + + if (wcsncpy_s(Adapter->Pool, _countof(Adapter->Pool), Pool, _TRUNCATE) == STRUNCATE) + { + LOG(WIREGUARD_LOG_ERR, L"Pool name too long: %s", Pool); + LastError = ERROR_INVALID_PARAMETER; + goto cleanupKey; + } + +cleanupKey: + RegCloseKey(Key); + return RET_ERROR(TRUE, LastError); +} + +_Use_decl_annotations_ +VOID WINAPI +WireGuardFreeAdapter(WIREGUARD_ADAPTER *Adapter) +{ + if (!Adapter) + return; + WireGuardSetAdapterLogging(Adapter, WIREGUARD_ADAPTER_LOG_OFF); + if (Adapter->DevInfo) + SetupDiDestroyDeviceInfoList(Adapter->DevInfo); + Free(Adapter); +} + +_Use_decl_annotations_ +BOOL WINAPI +WireGuardGetAdapterName(WIREGUARD_ADAPTER *Adapter, LPWSTR Name) +{ + DEVPROPTYPE PropType; + if (!SetupDiGetDevicePropertyW( + Adapter->DevInfo, + &Adapter->DevInfoData, + &DEVPKEY_WireGuard_Name, + &PropType, + (PBYTE)Name, + MAX_ADAPTER_NAME * sizeof(*Name), + NULL, + 0)) + return FALSE; + if (PropType != DEVPROP_TYPE_STRING || !*Name) + { + SetLastError(ERROR_BAD_DEVICE); + return FALSE; + } + return TRUE; +} + +_Must_inspect_result_ +static _Return_type_success_(return != FALSE) +BOOL +ConvertInterfaceAliasToGuid(_In_z_ LPCWSTR Name, _Out_ GUID *Guid) +{ + NET_LUID Luid; + DWORD LastError = ConvertInterfaceAliasToLuid(Name, &Luid); + if (LastError != NO_ERROR) + { + SetLastError(LOG_ERROR(LastError, L"Failed convert interface %s name to the locally unique identifier", Name)); + return FALSE; + } + LastError = ConvertInterfaceLuidToGuid(&Luid, Guid); + if (LastError != NO_ERROR) + { + SetLastError(LOG_ERROR(LastError, L"Failed to convert interface %s LUID (%I64u) to GUID", Name, Luid.Value)); + return FALSE; + } + return TRUE; +} + +_Use_decl_annotations_ +BOOL WINAPI +WireGuardSetAdapterName(WIREGUARD_ADAPTER *Adapter, LPCWSTR Name) +{ + const int MaxSuffix = 1000; + WCHAR AvailableName[MAX_ADAPTER_NAME]; + if (wcsncpy_s(AvailableName, _countof(AvailableName), Name, _TRUNCATE) == STRUNCATE) + { + LOG(WIREGUARD_LOG_ERR, L"Adapter name too long: %s", Name); + SetLastError(ERROR_INVALID_PARAMETER); + return FALSE; + } + + if (!SetupDiSetDevicePropertyW( + Adapter->DevInfo, + &Adapter->DevInfoData, + &DEVPKEY_WireGuard_Name, + DEVPROP_TYPE_STRING, +#pragma warning(suppress : 4090) + (const BYTE *)Name, + (DWORD)((wcslen(Name) + 1) * sizeof(*Name)), + 0)) + { + LOG_LAST_ERROR(L"Failed to set adapter %u name", Adapter->DevInfoData.DevInst); + return FALSE; + } + + for (int i = 0;; ++i) + { + DWORD LastError = NciSetConnectionName(&Adapter->CfgInstanceID, AvailableName); + if (LastError == ERROR_DUP_NAME) + { + GUID Guid2; + if (ConvertInterfaceAliasToGuid(AvailableName, &Guid2)) + { + for (int j = 0; j < MaxSuffix; ++j) + { + WCHAR Proposal[MAX_ADAPTER_NAME]; + if (_snwprintf_s(Proposal, _countof(Proposal), _TRUNCATE, L"%s %d", Name, j + 1) == -1) + { + LOG(WIREGUARD_LOG_ERR, L"Adapter name too long: %s %d", Name, j + 1); + SetLastError(ERROR_INVALID_PARAMETER); + return FALSE; + } + if (_wcsnicmp(Proposal, AvailableName, MAX_ADAPTER_NAME) == 0) + continue; + DWORD LastError2 = NciSetConnectionName(&Guid2, Proposal); + if (LastError2 == ERROR_DUP_NAME) + continue; + if (LastError2 == ERROR_SUCCESS) + { + LastError = NciSetConnectionName(&Adapter->CfgInstanceID, AvailableName); + if (LastError == ERROR_SUCCESS) + break; + } + break; + } + } + } + if (LastError == ERROR_SUCCESS) + break; + if (i >= MaxSuffix || LastError != ERROR_DUP_NAME) + { + SetLastError(LOG_ERROR(LastError, L"Failed to set adapter name")); + return FALSE; + } + if (_snwprintf_s(AvailableName, _countof(AvailableName), _TRUNCATE, L"%s %d", Name, i + 1) == -1) + { + LOG(WIREGUARD_LOG_ERR, L"Adapter name too long: %s %d", Name, i + 1); + SetLastError(ERROR_INVALID_PARAMETER); + return FALSE; + } + } + + if (!SetupDiSetDevicePropertyW( + Adapter->DevInfo, + &Adapter->DevInfoData, + &DEVPKEY_WireGuard_Pool, + DEVPROP_TYPE_STRING, +#pragma warning(suppress : 4090) + (const BYTE *)Adapter->Pool, + (DWORD)((wcslen(Adapter->Pool) + 1) * sizeof(*Adapter->Pool)), + 0)) + { + LOG_LAST_ERROR(L"Failed to set adapter %u pool", Adapter->DevInfoData.DevInst); + return FALSE; + } + + WCHAR PoolDeviceTypeName[MAX_POOL_DEVICE_TYPE]; + if (!GetPoolDeviceTypeName(Adapter->Pool, PoolDeviceTypeName)) + return FALSE; + if (!SetupDiSetDeviceRegistryPropertyW( + Adapter->DevInfo, + &Adapter->DevInfoData, + SPDRP_FRIENDLYNAME, + (const BYTE *)PoolDeviceTypeName, + (DWORD)((wcslen(PoolDeviceTypeName) + 1) * sizeof(*PoolDeviceTypeName)))) + { + LOG_LAST_ERROR(L"Failed to set adapter %u friendly name", Adapter->DevInfoData.DevInst); + return FALSE; + } + + return TRUE; +} + +_Use_decl_annotations_ +WIREGUARD_ADAPTER_HANDLE WINAPI +WireGuardOpenAdapter(LPCWSTR Pool, LPCWSTR Name) +{ + WIREGUARD_ADAPTER *Adapter = Zalloc(sizeof(*Adapter)); + if (!Adapter) + return FALSE; + + DWORD LastError = ERROR_SUCCESS; + HANDLE Mutex = NamespaceTakePoolMutex(Pool); + if (!Mutex) + { + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to take %s pool mutex", Pool); + goto cleanup; + } + + Adapter->DevInfo = SetupDiGetClassDevsExW(&GUID_DEVCLASS_NET, NULL, NULL, DIGCF_PRESENT, NULL, NULL, NULL); + if (Adapter->DevInfo == INVALID_HANDLE_VALUE) + { + LastError = LOG_LAST_ERROR(L"Failed to get present adapters"); + goto cleanupMutex; + } + + Adapter->DevInfoData.cbSize = sizeof(Adapter->DevInfoData); + for (DWORD EnumIndex = 0;; ++EnumIndex) + { + if (!SetupDiEnumDeviceInfo(Adapter->DevInfo, EnumIndex, &Adapter->DevInfoData)) + { + if (GetLastError() == ERROR_NO_MORE_ITEMS) + break; + continue; + } + + WCHAR Name2[MAX_ADAPTER_NAME]; + if (!WireGuardGetAdapterName(Adapter, Name2)) + continue; + if (_wcsicmp(Name, Name2)) + continue; + + /* Check the Hardware ID to make sure it's a real WireGuard device. */ + if (!IsOurAdapter(Adapter->DevInfo, &Adapter->DevInfoData)) + { + LOG(WIREGUARD_LOG_ERR, L"Foreign adapter %u named %s exists", Adapter->DevInfoData.DevInst, Name); + LastError = ERROR_ALREADY_EXISTS; + goto cleanupMutex; + } + + if (!IsPoolMember(Pool, Adapter->DevInfo, &Adapter->DevInfoData)) + { + if ((LastError = GetLastError()) == ERROR_SUCCESS) + { + LOG(WIREGUARD_LOG_ERR, + L"Adapter %u named %s is not a member of %s pool", + Adapter->DevInfoData.DevInst, + Name, + Pool); + LastError = ERROR_ALREADY_EXISTS; + goto cleanupMutex; + } + else + { + LOG(WIREGUARD_LOG_ERR, L"Failed to get adapter %u pool membership", Adapter->DevInfoData.DevInst); + goto cleanupMutex; + } + } + + if (!PopulateAdapterData(Adapter, Pool)) + { + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to populate adapter %u data", Adapter->DevInfoData.DevInst); + goto cleanupMutex; + } + + if (!EnsureDeviceObject(Adapter->DevInstanceID)) + { + LastError = GetLastError(); + goto cleanupMutex; + } + + /* Our comparison was case-insensitive, and we also might want to reenforce the NCI connection. */ + WireGuardSetAdapterName(Adapter, Name); + + LastError = ERROR_SUCCESS; + goto cleanupMutex; + } + LastError = ERROR_FILE_NOT_FOUND; +cleanupMutex: + NamespaceReleaseMutex(Mutex); +cleanup: + if (LastError != ERROR_SUCCESS) + WireGuardFreeAdapter(Adapter); + return RET_ERROR(Adapter, LastError); +} + +_Use_decl_annotations_ +VOID WINAPI +WireGuardGetAdapterLUID(WIREGUARD_ADAPTER *Adapter, NET_LUID *Luid) +{ + Luid->Info.Reserved = 0; + Luid->Info.NetLuidIndex = Adapter->LuidIndex; + Luid->Info.IfType = Adapter->IfType; +} + +_Use_decl_annotations_ +HANDLE WINAPI +AdapterOpenDeviceObject(const WIREGUARD_ADAPTER *Adapter) +{ + return OpenDeviceObject(Adapter->DevInstanceID); +} + +static BOOL HaveWHQL(VOID) +{ +#if defined(HAVE_WHQL) + return IsWindows10; +#else + return FALSE; +#endif +} + +static _Return_type_success_(return != FALSE) +BOOL +InstallCertificate(_In_z_ LPCWSTR SignedResource) +{ + LOG(WIREGUARD_LOG_INFO, L"Trusting code signing certificate"); + DWORD SizeResource; + const VOID *LockedResource = ResourceGetAddress(SignedResource, &SizeResource); + if (!LockedResource) + { + LOG(WIREGUARD_LOG_ERR, L"Failed to locate resource %s", SignedResource); + return FALSE; + } + const CERT_BLOB CertBlob = { .cbData = SizeResource, .pbData = (BYTE *)LockedResource }; + HCERTSTORE QueriedStore; + if (!CryptQueryObject( + CERT_QUERY_OBJECT_BLOB, + &CertBlob, + CERT_QUERY_CONTENT_FLAG_PKCS7_SIGNED_EMBED, + CERT_QUERY_FORMAT_FLAG_ALL, + 0, + 0, + 0, + 0, + &QueriedStore, + 0, + NULL)) + { + LOG_LAST_ERROR(L"Failed to find certificate"); + return FALSE; + } + DWORD LastError = ERROR_SUCCESS; + HCERTSTORE TrustedStore = + CertOpenStore(CERT_STORE_PROV_SYSTEM, 0, 0, CERT_SYSTEM_STORE_LOCAL_MACHINE, L"TrustedPublisher"); + if (!TrustedStore) + { + LastError = LOG_LAST_ERROR(L"Failed to open store"); + goto cleanupQueriedStore; + } + LPSTR CodeSigningOid[] = { szOID_PKIX_KP_CODE_SIGNING }; + CERT_ENHKEY_USAGE EnhancedUsage = { .cUsageIdentifier = 1, .rgpszUsageIdentifier = CodeSigningOid }; + for (const CERT_CONTEXT *CertContext = NULL; (CertContext = CertFindCertificateInStore( + QueriedStore, + X509_ASN_ENCODING | PKCS_7_ASN_ENCODING, + CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG, + CERT_FIND_ENHKEY_USAGE, + &EnhancedUsage, + CertContext)) != NULL;) + { + CERT_EXTENSION *Ext = CertFindExtension( + szOID_BASIC_CONSTRAINTS2, CertContext->pCertInfo->cExtension, CertContext->pCertInfo->rgExtension); + CERT_BASIC_CONSTRAINTS2_INFO Constraints; + DWORD Size = sizeof(Constraints); + if (Ext && + CryptDecodeObjectEx( + X509_ASN_ENCODING | PKCS_7_ASN_ENCODING, + szOID_BASIC_CONSTRAINTS2, + Ext->Value.pbData, + Ext->Value.cbData, + 0, + NULL, + &Constraints, + &Size) && + !Constraints.fCA) + if (!CertAddCertificateContextToStore(TrustedStore, CertContext, CERT_STORE_ADD_REPLACE_EXISTING, NULL)) + { + LOG_LAST_ERROR(L"Failed to add certificate to store"); + LastError = LastError != ERROR_SUCCESS ? LastError : GetLastError(); + } + } + CertCloseStore(TrustedStore, 0); +cleanupQueriedStore: + CertCloseStore(QueriedStore, 0); + return RET_ERROR(TRUE, LastError); +} + +static BOOL +IsOurDrvInfoDetail(_In_ const SP_DRVINFO_DETAIL_DATA_W *DrvInfoDetailData) +{ + if (DrvInfoDetailData->CompatIDsOffset > 1 && !_wcsicmp(DrvInfoDetailData->HardwareID, WIREGUARD_HWID)) + return TRUE; + if (DrvInfoDetailData->CompatIDsLength && + IsOurHardwareID(DrvInfoDetailData->HardwareID + DrvInfoDetailData->CompatIDsOffset)) + return TRUE; + return FALSE; +} + +static BOOL +IsNewer( + _In_ const FILETIME *DriverDate1, + _In_ DWORDLONG DriverVersion1, + _In_ const FILETIME *DriverDate2, + _In_ DWORDLONG DriverVersion2) +{ + if (DriverDate1->dwHighDateTime > DriverDate2->dwHighDateTime) + return TRUE; + if (DriverDate1->dwHighDateTime < DriverDate2->dwHighDateTime) + return FALSE; + + if (DriverDate1->dwLowDateTime > DriverDate2->dwLowDateTime) + return TRUE; + if (DriverDate1->dwLowDateTime < DriverDate2->dwLowDateTime) + return FALSE; + + if (DriverVersion1 > DriverVersion2) + return TRUE; + if (DriverVersion1 < DriverVersion2) + return FALSE; + + return FALSE; +} + +_Must_inspect_result_ +static _Return_type_success_(return != FALSE) +BOOL +GetTcpipAdapterRegPath(_In_ const WIREGUARD_ADAPTER *Adapter, _Out_writes_z_(MAX_REG_PATH) LPWSTR Path) +{ + WCHAR Guid[MAX_GUID_STRING_LEN]; + if (_snwprintf_s( + Path, + MAX_REG_PATH, + _TRUNCATE, + L"SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters\\Adapters\\%.*s", + StringFromGUID2(&Adapter->CfgInstanceID, Guid, _countof(Guid)), + Guid) == -1) + { + LOG(WIREGUARD_LOG_ERR, L"Registry path too long"); + SetLastError(ERROR_INVALID_PARAMETER); + return FALSE; + } + return TRUE; +} + +_Must_inspect_result_ +static _Return_type_success_(return != FALSE) +BOOL +GetTcpipInterfaceRegPath(_In_ const WIREGUARD_ADAPTER *Adapter, _Out_writes_z_(MAX_REG_PATH) LPWSTR Path) +{ + HKEY TcpipAdapterRegKey; + WCHAR TcpipAdapterRegPath[MAX_REG_PATH]; + if (!GetTcpipAdapterRegPath(Adapter, TcpipAdapterRegPath)) + return FALSE; + DWORD LastError = RegOpenKeyExW(HKEY_LOCAL_MACHINE, TcpipAdapterRegPath, 0, KEY_QUERY_VALUE, &TcpipAdapterRegKey); + if (LastError != ERROR_SUCCESS) + { + SetLastError(LOG_ERROR(LastError, L"Failed to open registry key %s", TcpipAdapterRegPath)); + return FALSE; + } + LPWSTR Paths = RegistryQueryString(TcpipAdapterRegKey, L"IpConfig", TRUE); + if (!Paths) + { + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to get %s\\IpConfig", TcpipAdapterRegPath); + goto cleanupTcpipAdapterRegKey; + } + if (!Paths[0]) + { + LOG(WIREGUARD_LOG_ERR, L"%s\\IpConfig is empty", TcpipAdapterRegPath); + LastError = ERROR_INVALID_DATA; + goto cleanupPaths; + } + if (_snwprintf_s(Path, MAX_REG_PATH, _TRUNCATE, L"SYSTEM\\CurrentControlSet\\Services\\%s", Paths) == -1) + { + LOG(WIREGUARD_LOG_ERR, L"Registry path too long: %s", Paths); + LastError = ERROR_INVALID_PARAMETER; + goto cleanupPaths; + } +cleanupPaths: + Free(Paths); +cleanupTcpipAdapterRegKey: + RegCloseKey(TcpipAdapterRegKey); + return RET_ERROR(TRUE, LastError); +} + +static _Return_type_success_(return != 0) +DWORD +VersionOfFile(_In_z_ LPCWSTR Filename) +{ + DWORD Zero; + DWORD Len = GetFileVersionInfoSizeW(Filename, &Zero); + if (!Len) + { + LOG_LAST_ERROR(L"Failed to query %s version info size", Filename); + return 0; + } + VOID *VersionInfo = Alloc(Len); + if (!VersionInfo) + return 0; + DWORD LastError = ERROR_SUCCESS, Version = 0; + VS_FIXEDFILEINFO *FixedInfo; + UINT FixedInfoLen = sizeof(*FixedInfo); + if (!GetFileVersionInfoW(Filename, 0, Len, VersionInfo)) + { + LastError = LOG_LAST_ERROR(L"Failed to get %s version info", Filename); + goto out; + } + if (!VerQueryValueW(VersionInfo, L"\\", &FixedInfo, &FixedInfoLen)) + { + LastError = LOG_LAST_ERROR(L"Failed to get %s version info root", Filename); + goto out; + } + Version = FixedInfo->dwFileVersionMS; + if (!Version) + { + LOG(WIREGUARD_LOG_WARN, L"Determined version of %s, but was v0.0, so returning failure", Filename); + LastError = ERROR_VERSION_PARSE_ERROR; + } +out: + Free(VersionInfo); + return RET_ERROR(Version, LastError); +} + +static DWORD WINAPI +MaybeGetRunningDriverVersion(BOOL ReturnOneIfRunningInsteadOfVersion) +{ + PRTL_PROCESS_MODULES Modules; + ULONG BufferSize = 128 * 1024; + for (;;) + { + Modules = Alloc(BufferSize); + if (!Modules) + return 0; + NTSTATUS Status = NtQuerySystemInformation(SystemModuleInformation, Modules, BufferSize, &BufferSize); + if (NT_SUCCESS(Status)) + break; + Free(Modules); + if (Status == STATUS_INFO_LENGTH_MISMATCH) + continue; + LOG(WIREGUARD_LOG_ERR, L"Failed to enumerate drivers (status: 0x%x)", Status); + SetLastError(RtlNtStatusToDosError(Status)); + return 0; + } + DWORD LastError = ERROR_SUCCESS, Version = 0; + for (ULONG i = Modules->NumberOfModules; i-- > 0;) + { + LPCSTR NtPath = (LPCSTR)Modules->Modules[i].FullPathName; + if (!_stricmp(&NtPath[Modules->Modules[i].OffsetToFileName], "wireguard.sys")) + { + if (ReturnOneIfRunningInsteadOfVersion) + { + Version = 1; + goto cleanupModules; + } + WCHAR FilePath[MAX_PATH * 3 + 15]; + if (_snwprintf_s(FilePath, _countof(FilePath), _TRUNCATE, L"\\\\?\\GLOBALROOT%S", NtPath) == -1) + continue; + Version = VersionOfFile(FilePath); + if (!Version) + LastError = GetLastError(); + goto cleanupModules; + } + } + LastError = ERROR_FILE_NOT_FOUND; +cleanupModules: + Free(Modules); + return RET_ERROR(Version, LastError); +} + +_Use_decl_annotations_ +DWORD WINAPI WireGuardGetRunningDriverVersion(VOID) +{ + return MaybeGetRunningDriverVersion(FALSE); +} + +static BOOL EnsureWireGuardUnloaded(VOID) +{ + BOOL Loaded; + for (int i = 0; (Loaded = MaybeGetRunningDriverVersion(TRUE) != 0) != FALSE && i < 300; ++i) + Sleep(50); + return !Loaded; +} + +static VOID +SelectDriverDeferredCleanup(_In_ HDEVINFO DevInfoExistingAdapters, _In_opt_ SP_DEVINFO_DATA_LIST *ExistingAdapters) +{ + if (ExistingAdapters) + { + EnableAllOurAdapters(DevInfoExistingAdapters, ExistingAdapters); + while (ExistingAdapters) + { + SP_DEVINFO_DATA_LIST *Next = ExistingAdapters->Next; + Free(ExistingAdapters->Configuration); + Free(ExistingAdapters); + ExistingAdapters = Next; + } + } + if (DevInfoExistingAdapters != INVALID_HANDLE_VALUE) + SetupDiDestroyDeviceInfoList(DevInfoExistingAdapters); +} + +_Must_inspect_result_ +static _Return_type_success_(return != FALSE) +BOOL +SelectDriver( + _In_ HDEVINFO DevInfo, + _In_ SP_DEVINFO_DATA *DevInfoData, + _Inout_ SP_DEVINSTALL_PARAMS_W *DevInstallParams, + _Out_ HDEVINFO *DevInfoExistingAdaptersForCleanup, + _Out_ SP_DEVINFO_DATA_LIST **ExistingAdaptersForCleanup) +{ + static const FILETIME OurDriverDate = WIREGUARD_INF_FILETIME; + static const DWORDLONG OurDriverVersion = WIREGUARD_INF_VERSION; + HANDLE DriverInstallationLock = NamespaceTakeDriverInstallationMutex(); + if (!DriverInstallationLock) + { + LOG(WIREGUARD_LOG_ERR, L"Failed to take driver installation mutex"); + return FALSE; + } + DWORD LastError; + if (!SetupDiBuildDriverInfoList(DevInfo, DevInfoData, SPDIT_COMPATDRIVER)) + { + LastError = LOG_LAST_ERROR(L"Failed building adapter %u driver info list", DevInfoData->DevInst); + goto cleanupDriverInstallationLock; + } + BOOL DestroyDriverInfoListOnCleanup = TRUE; + FILETIME DriverDate = { 0 }; + DWORDLONG DriverVersion = 0; + HDEVINFO DevInfoExistingAdapters = INVALID_HANDLE_VALUE; + SP_DEVINFO_DATA_LIST *ExistingAdapters = NULL; + for (DWORD EnumIndex = 0;; ++EnumIndex) + { + SP_DRVINFO_DATA_W DrvInfoData = { .cbSize = sizeof(SP_DRVINFO_DATA_W) }; + if (!SetupDiEnumDriverInfoW(DevInfo, DevInfoData, SPDIT_COMPATDRIVER, EnumIndex, &DrvInfoData)) + { + if (GetLastError() == ERROR_NO_MORE_ITEMS) + break; + continue; + } + SP_DRVINFO_DETAIL_DATA_W *DrvInfoDetailData = GetAdapterDrvInfoDetail(DevInfo, DevInfoData, &DrvInfoData); + if (!DrvInfoDetailData) + { + LOG(WIREGUARD_LOG_WARN, L"Failed getting adapter %u driver info detail", DevInfoData->DevInst); + continue; + } + if (!IsOurDrvInfoDetail(DrvInfoDetailData)) + goto next; + if (IsNewer(&OurDriverDate, OurDriverVersion, &DrvInfoData.DriverDate, DrvInfoData.DriverVersion)) + { + if (DevInfoExistingAdapters == INVALID_HANDLE_VALUE) + { + DevInfoExistingAdapters = + SetupDiGetClassDevsExW(&GUID_DEVCLASS_NET, NULL, NULL, DIGCF_PRESENT, NULL, NULL, NULL); + if (DevInfoExistingAdapters == INVALID_HANDLE_VALUE) + { + LastError = LOG_LAST_ERROR(L"Failed to get present adapters"); + Free(DrvInfoDetailData); + goto cleanupExistingAdapters; + } + _Analysis_assume_(DevInfoExistingAdapters != NULL); + DisableAllOurAdapters(DevInfoExistingAdapters, &ExistingAdapters); + LOG(WIREGUARD_LOG_INFO, L"Waiting for existing driver to unload from kernel"); + if (!EnsureWireGuardUnloaded()) + LOG(WIREGUARD_LOG_WARN, + L"Failed to unload existing driver, which means a reboot will likely be required"); + } + LOG(WIREGUARD_LOG_INFO, + L"Removing existing driver %u.%u", + (DWORD)((DrvInfoData.DriverVersion & 0xffff000000000000) >> 48), + (DWORD)((DrvInfoData.DriverVersion & 0x0000ffff00000000) >> 32)); + LPWSTR InfFileName = PathFindFileNameW(DrvInfoDetailData->InfFileName); + if (!SetupUninstallOEMInfW(InfFileName, SUOI_FORCEDELETE, NULL)) + LOG_LAST_ERROR(L"Unable to remove existing driver %s", InfFileName); + goto next; + } + if (!IsNewer(&DrvInfoData.DriverDate, DrvInfoData.DriverVersion, &DriverDate, DriverVersion)) + goto next; + if (!SetupDiSetSelectedDriverW(DevInfo, DevInfoData, &DrvInfoData)) + { + LOG_LAST_ERROR( + L"Failed to select driver %s for adapter %u", DrvInfoDetailData->InfFileName, DevInfoData->DevInst); + goto next; + } + DriverDate = DrvInfoData.DriverDate; + DriverVersion = DrvInfoData.DriverVersion; + next: + Free(DrvInfoDetailData); + } + + if (DriverVersion) + { + LOG(WIREGUARD_LOG_INFO, + L"Using existing driver %u.%u", + (DWORD)((DriverVersion & 0xffff000000000000) >> 48), + (DWORD)((DriverVersion & 0x0000ffff00000000) >> 32)); + LastError = ERROR_SUCCESS; + DestroyDriverInfoListOnCleanup = FALSE; + goto cleanupExistingAdapters; + } + + LOG(WIREGUARD_LOG_INFO, + L"Installing driver %u.%u", + (DWORD)((OurDriverVersion & 0xffff000000000000) >> 48), + (DWORD)((OurDriverVersion & 0x0000ffff00000000) >> 32)); + WCHAR RandomTempSubDirectory[MAX_PATH]; + if (!ResourceCreateTemporaryDirectory(RandomTempSubDirectory)) + { + LastError = LOG_LAST_ERROR(L"Failed to create temporary folder %s", RandomTempSubDirectory); + goto cleanupExistingAdapters; + } + + WCHAR CatPath[MAX_PATH] = { 0 }; + WCHAR SysPath[MAX_PATH] = { 0 }; + WCHAR InfPath[MAX_PATH] = { 0 }; + if (!PathCombineW(CatPath, RandomTempSubDirectory, L"wireguard.cat") || + !PathCombineW(SysPath, RandomTempSubDirectory, L"wireguard.sys") || + !PathCombineW(InfPath, RandomTempSubDirectory, L"wireguard.inf")) + { + LastError = ERROR_BUFFER_OVERFLOW; + goto cleanupDirectory; + } + + BOOL UseWHQL = HaveWHQL(); + if (!UseWHQL && !InstallCertificate(L"wireguard.cat")) + LOG(WIREGUARD_LOG_WARN, L"Failed to install code signing certificate"); + + LOG(WIREGUARD_LOG_INFO, L"Extracting driver"); + if (!ResourceCopyToFile(CatPath, UseWHQL ? L"wireguard-whql.cat" : L"wireguard.cat") || + !ResourceCopyToFile(SysPath, UseWHQL ? L"wireguard-whql.sys" : L"wireguard.sys") || + !ResourceCopyToFile(InfPath, UseWHQL ? L"wireguard-whql.inf" : L"wireguard.inf")) + { + LastError = LOG_LAST_ERROR(L"Failed to extract driver"); + goto cleanupDelete; + } + LOG(WIREGUARD_LOG_INFO, L"Installing driver"); + WCHAR InfStorePath[MAX_PATH]; + if (!SetupCopyOEMInfW(InfPath, NULL, SPOST_NONE, 0, InfStorePath, MAX_PATH, NULL, NULL)) + { + LastError = LOG_LAST_ERROR(L"Could not install driver %s to store", InfPath); + goto cleanupDelete; + } + _Analysis_assume_nullterminated_(InfStorePath); + + SetupDiDestroyDriverInfoList(DevInfo, DevInfoData, SPDIT_COMPATDRIVER); + DestroyDriverInfoListOnCleanup = FALSE; + DevInstallParams->Flags |= DI_ENUMSINGLEINF; + if (wcsncpy_s(DevInstallParams->DriverPath, _countof(DevInstallParams->DriverPath), InfStorePath, _TRUNCATE) == + STRUNCATE) + { + LOG(WIREGUARD_LOG_ERR, L"Inf path too long: %s", InfStorePath); + LastError = ERROR_INVALID_PARAMETER; + goto cleanupDelete; + } + if (!SetupDiSetDeviceInstallParamsW(DevInfo, DevInfoData, DevInstallParams)) + { + LastError = LOG_LAST_ERROR(L"Failed to set adapter %u device installation parameters", DevInfoData->DevInst); + goto cleanupDelete; + } + if (!SetupDiBuildDriverInfoList(DevInfo, DevInfoData, SPDIT_COMPATDRIVER)) + { + LastError = LOG_LAST_ERROR(L"Failed rebuilding adapter %u driver info list", DevInfoData->DevInst); + goto cleanupDelete; + } + DestroyDriverInfoListOnCleanup = TRUE; + SP_DRVINFO_DATA_W DrvInfoData = { .cbSize = sizeof(SP_DRVINFO_DATA_W) }; + if (!SetupDiEnumDriverInfoW(DevInfo, DevInfoData, SPDIT_COMPATDRIVER, 0, &DrvInfoData)) + { + LastError = LOG_LAST_ERROR(L"Failed to get adapter %u driver", DevInfoData->DevInst); + goto cleanupDelete; + } + if (!SetupDiSetSelectedDriverW(DevInfo, DevInfoData, &DrvInfoData)) + { + LastError = LOG_LAST_ERROR(L"Failed to set adapter %u driver", DevInfoData->DevInst); + goto cleanupDelete; + } + LastError = ERROR_SUCCESS; + DestroyDriverInfoListOnCleanup = FALSE; + +cleanupDelete: + DeleteFileW(CatPath); + DeleteFileW(SysPath); + DeleteFileW(InfPath); +cleanupDirectory: + RemoveDirectoryW(RandomTempSubDirectory); +cleanupExistingAdapters: + if (LastError == ERROR_SUCCESS) + { + *DevInfoExistingAdaptersForCleanup = DevInfoExistingAdapters; + *ExistingAdaptersForCleanup = ExistingAdapters; + } + else + SelectDriverDeferredCleanup(DevInfoExistingAdapters, ExistingAdapters); + if (DestroyDriverInfoListOnCleanup) + SetupDiDestroyDriverInfoList(DevInfo, DevInfoData, SPDIT_COMPATDRIVER); +cleanupDriverInstallationLock: + NamespaceReleaseMutex(DriverInstallationLock); + return RET_ERROR(TRUE, LastError); +} + +_Use_decl_annotations_ +WIREGUARD_ADAPTER * +AdapterOpenFromDevInstanceId(LPCWSTR Pool, LPCWSTR DevInstanceID) +{ + WIREGUARD_ADAPTER *Adapter = Zalloc(sizeof(*Adapter)); + if (!Adapter) + return FALSE; + + DWORD LastError = ERROR_SUCCESS; + HANDLE Mutex = NamespaceTakePoolMutex(Pool); + if (!Mutex) + { + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to take %s pool mutex", Pool); + goto cleanup; + } + Adapter->DevInfo = SetupDiCreateDeviceInfoListExW(&GUID_DEVCLASS_NET, NULL, NULL, NULL); + if (Adapter->DevInfo == INVALID_HANDLE_VALUE) + { + LastError = LOG_LAST_ERROR(L"Failed to get present adapters"); + goto cleanupMutex; + } + Adapter->DevInfoData.cbSize = sizeof(Adapter->DevInfoData); + if (!SetupDiOpenDeviceInfoW(Adapter->DevInfo, DevInstanceID, NULL, 0, &Adapter->DevInfoData)) + { + LastError = LOG_LAST_ERROR(L"Failed to open device instance ID %s", DevInstanceID); + goto cleanupMutex; + } + if (!PopulateAdapterData(Adapter, Pool)) + { + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to populate adapter %u data", Adapter->DevInfoData.DevInst); + goto cleanupMutex; + } +cleanupMutex: + NamespaceReleaseMutex(Mutex); +cleanup: + if (LastError != ERROR_SUCCESS) + WireGuardFreeAdapter(Adapter); + return RET_ERROR(Adapter, LastError); +} + +_Use_decl_annotations_ +WIREGUARD_ADAPTER_HANDLE WINAPI +WireGuardCreateAdapter(LPCWSTR Pool, LPCWSTR Name, const GUID *RequestedGUID, BOOL *RebootRequired) +{ + BOOL DummyRebootRequired; + if (!RebootRequired) + RebootRequired = &DummyRebootRequired; + *RebootRequired = FALSE; + +#ifdef MAYBE_WOW64 + if (NativeMachine != IMAGE_FILE_PROCESS) + return CreateAdapterViaRundll32(Pool, Name, RequestedGUID, RebootRequired); +#endif + + DWORD LastError = ERROR_SUCCESS; + LOG(WIREGUARD_LOG_INFO, L"Creating adapter"); + + if (!IsWindows10) + RequestedGUID = NULL; + + WIREGUARD_ADAPTER *Adapter = Zalloc(sizeof(*Adapter)); + if (!Adapter) + return NULL; + + Adapter->DevInfo = SetupDiCreateDeviceInfoListExW(&GUID_DEVCLASS_NET, NULL, NULL, NULL); + if (Adapter->DevInfo == INVALID_HANDLE_VALUE) + { + LastError = LOG_LAST_ERROR(L"Failed to create empty device information set"); + goto cleanupAdapter; + } + WCHAR ClassName[MAX_CLASS_NAME_LEN]; + if (!SetupDiClassNameFromGuidExW(&GUID_DEVCLASS_NET, ClassName, _countof(ClassName), NULL, NULL, NULL)) + { + LastError = LOG_LAST_ERROR(L"Failed to retrieve class name associated with class GUID"); + goto cleanupAdapter; + } + + WCHAR PoolDeviceTypeName[MAX_POOL_DEVICE_TYPE]; + if (!GetPoolDeviceTypeName(Pool, PoolDeviceTypeName)) + { + LastError = GetLastError(); + goto cleanupAdapter; + } + Adapter->DevInfoData.cbSize = sizeof(Adapter->DevInfoData); + if (!SetupDiCreateDeviceInfoW( + Adapter->DevInfo, + ClassName, + &GUID_DEVCLASS_NET, + PoolDeviceTypeName, + NULL, + DICD_GENERATE_ID, + &Adapter->DevInfoData)) + { + LastError = LOG_LAST_ERROR(L"Failed to create new device information element"); + goto cleanupAdapter; + } + SP_DEVINSTALL_PARAMS_W DevInstallParams = { .cbSize = sizeof(DevInstallParams) }; + if (!SetupDiGetDeviceInstallParamsW(Adapter->DevInfo, &Adapter->DevInfoData, &DevInstallParams)) + { + LastError = LOG_LAST_ERROR( + L"Failed to retrieve adapter %u device installation parameters", Adapter->DevInfoData.DevInst); + goto cleanupAdapter; + } + DevInstallParams.Flags |= DI_QUIETINSTALL; + if (!SetupDiSetDeviceInstallParamsW(Adapter->DevInfo, &Adapter->DevInfoData, &DevInstallParams)) + { + LastError = + LOG_LAST_ERROR(L"Failed to set adapter %u device installation parameters", Adapter->DevInfoData.DevInst); + goto cleanupAdapter; + } + if (!SetupDiSetSelectedDevice(Adapter->DevInfo, &Adapter->DevInfoData)) + { + LastError = LOG_LAST_ERROR(L"Failed to select adapter %u device", Adapter->DevInfoData.DevInst); + goto cleanupAdapter; + } + static const WCHAR Hwids[_countof(WIREGUARD_HWID) + 1 /*Multi-string terminator*/] = WIREGUARD_HWID; + if (!SetupDiSetDeviceRegistryPropertyW( + Adapter->DevInfo, &Adapter->DevInfoData, SPDRP_HARDWAREID, (const BYTE *)Hwids, sizeof(Hwids))) + { + LastError = LOG_LAST_ERROR(L"Failed to set adapter %u hardware ID", Adapter->DevInfoData.DevInst); + goto cleanupAdapter; + } + + HDEVINFO DevInfoExistingAdapters; + SP_DEVINFO_DATA_LIST *ExistingAdapters; + if (!SelectDriver( + Adapter->DevInfo, &Adapter->DevInfoData, &DevInstallParams, &DevInfoExistingAdapters, &ExistingAdapters)) + { + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to select adapter %u driver", Adapter->DevInfoData.DevInst); + goto cleanupAdapter; + } + + HANDLE Mutex = NamespaceTakePoolMutex(Pool); + if (!Mutex) + { + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to take %s pool mutex", Pool); + goto cleanupDriverInfoList; + } + + if (!SetupDiCallClassInstaller(DIF_REGISTERDEVICE, Adapter->DevInfo, &Adapter->DevInfoData)) + { + LastError = LOG_LAST_ERROR(L"Failed to register adapter %u device", Adapter->DevInfoData.DevInst); + goto cleanupDevice; + } + if (!SetupDiCallClassInstaller(DIF_REGISTER_COINSTALLERS, Adapter->DevInfo, &Adapter->DevInfoData)) + LOG_LAST_ERROR(L"Failed to register adapter %u coinstallers", Adapter->DevInfoData.DevInst); + + HKEY NetDevRegKey = INVALID_HANDLE_VALUE; + const int PollTimeout = 50 /* ms */; + for (int i = 0; NetDevRegKey == INVALID_HANDLE_VALUE && i < WAIT_FOR_REGISTRY_TIMEOUT / PollTimeout; ++i) + { + if (i) + Sleep(PollTimeout); + NetDevRegKey = SetupDiOpenDevRegKey( + Adapter->DevInfo, + &Adapter->DevInfoData, + DICS_FLAG_GLOBAL, + 0, + DIREG_DRV, + KEY_SET_VALUE | KEY_QUERY_VALUE | KEY_NOTIFY); + } + if (NetDevRegKey == INVALID_HANDLE_VALUE) + { + LastError = + LOG_LAST_ERROR(L"Failed to open adapter %u device-specific registry key", Adapter->DevInfoData.DevInst); + goto cleanupDevice; + } + if (RequestedGUID) + { + LastError = RegSetValueExW( + NetDevRegKey, L"SuggestedInstanceId", 0, REG_BINARY, (const BYTE *)RequestedGUID, sizeof(*RequestedGUID)); + if (LastError != ERROR_SUCCESS) + { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(NetDevRegKey, RegPath); + LOG_ERROR(LastError, L"Failed to set %.*s\\SuggestedInstanceId", MAX_REG_PATH, RegPath); + goto cleanupNetDevRegKey; + } + } + + if (!SetupDiCallClassInstaller(DIF_INSTALLINTERFACES, Adapter->DevInfo, &Adapter->DevInfoData)) + LOG_LAST_ERROR(L"Failed to install adapter %u interfaces", Adapter->DevInfoData.DevInst); + + if (!SetupDiCallClassInstaller(DIF_INSTALLDEVICE, Adapter->DevInfo, &Adapter->DevInfoData)) + { + LastError = LOG_LAST_ERROR(L"Failed to install adapter %u device", Adapter->DevInfoData.DevInst); + goto cleanupNetDevRegKey; + } + *RebootRequired = *RebootRequired || CheckReboot(Adapter->DevInfo, &Adapter->DevInfoData); + + if (!SetupDiSetDevicePropertyW( + Adapter->DevInfo, + &Adapter->DevInfoData, + &DEVPKEY_WireGuard_Pool, + DEVPROP_TYPE_STRING, +#pragma warning(suppress : 4090) + (const BYTE *)Pool, + (DWORD)((wcslen(Pool) + 1) * sizeof(*Pool)), + 0)) + { + LastError = LOG_LAST_ERROR(L"Failed to set adapter %u pool", Adapter->DevInfoData.DevInst); + goto cleanupNetDevRegKey; + } + if (!SetupDiSetDeviceRegistryPropertyW( + Adapter->DevInfo, + &Adapter->DevInfoData, + SPDRP_DEVICEDESC, + (const BYTE *)PoolDeviceTypeName, + (DWORD)((wcslen(PoolDeviceTypeName) + 1) * sizeof(*PoolDeviceTypeName)))) + { + LastError = LOG_LAST_ERROR(L"Failed to set adapter %u description", Adapter->DevInfoData.DevInst); + goto cleanupNetDevRegKey; + } + + /* DIF_INSTALLDEVICE returns almost immediately, while the device installation continues in the background. It might + * take a while, before all registry keys and values are populated. */ + LPWSTR DummyStr = RegistryQueryStringWait(NetDevRegKey, L"NetCfgInstanceId", WAIT_FOR_REGISTRY_TIMEOUT); + if (!DummyStr) + { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(NetDevRegKey, RegPath); + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to get %.*s\\NetCfgInstanceId", MAX_REG_PATH, RegPath); + goto cleanupNetDevRegKey; + } + Free(DummyStr); + DWORD DummyDWORD; + if (!RegistryQueryDWORDWait(NetDevRegKey, L"NetLuidIndex", WAIT_FOR_REGISTRY_TIMEOUT, &DummyDWORD)) + { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(NetDevRegKey, RegPath); + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to get %.*s\\NetLuidIndex", MAX_REG_PATH, RegPath); + goto cleanupNetDevRegKey; + } + if (!RegistryQueryDWORDWait(NetDevRegKey, L"*IfType", WAIT_FOR_REGISTRY_TIMEOUT, &DummyDWORD)) + { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(NetDevRegKey, RegPath); + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to get %.*s\\*IfType", MAX_REG_PATH, RegPath); + goto cleanupNetDevRegKey; + } + + if (!PopulateAdapterData(Adapter, Pool)) + { + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to populate adapter %u data", Adapter->DevInfoData.DevInst); + goto cleanupNetDevRegKey; + } + + HKEY TcpipAdapterRegKey; + WCHAR TcpipAdapterRegPath[MAX_REG_PATH]; + if (!GetTcpipAdapterRegPath(Adapter, TcpipAdapterRegPath)) + { + LastError = GetLastError(); + goto cleanupAdapter; + } + TcpipAdapterRegKey = RegistryOpenKeyWait( + HKEY_LOCAL_MACHINE, TcpipAdapterRegPath, KEY_QUERY_VALUE | KEY_NOTIFY, WAIT_FOR_REGISTRY_TIMEOUT); + if (!TcpipAdapterRegKey) + { + LastError = + LOG(WIREGUARD_LOG_ERR, + L"Failed to open adapter-specific TCP/IP interface registry key %s", + TcpipAdapterRegPath); + goto cleanupAdapter; + } + DummyStr = RegistryQueryStringWait(TcpipAdapterRegKey, L"IpConfig", WAIT_FOR_REGISTRY_TIMEOUT); + if (!DummyStr) + { + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to get %s\\IpConfig", TcpipAdapterRegPath); + goto cleanupTcpipAdapterRegKey; + } + Free(DummyStr); + + WCHAR TcpipInterfaceRegPath[MAX_REG_PATH]; + if (!GetTcpipInterfaceRegPath(Adapter, TcpipInterfaceRegPath)) + { + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to determine interface-specific TCP/IP network registry key path"); + goto cleanupTcpipAdapterRegKey; + } + for (int Tries = 0; Tries < 300; ++Tries) + { + HKEY TcpipInterfaceRegKey = RegistryOpenKeyWait( + HKEY_LOCAL_MACHINE, TcpipInterfaceRegPath, KEY_QUERY_VALUE | KEY_SET_VALUE, WAIT_FOR_REGISTRY_TIMEOUT); + if (!TcpipInterfaceRegKey) + { + LastError = + LOG(WIREGUARD_LOG_ERR, + L"Failed to open interface-specific TCP/IP network registry key %s", + TcpipInterfaceRegPath); + goto cleanupTcpipAdapterRegKey; + } + + static const DWORD EnableDeadGWDetect = 0; + LastError = RegSetKeyValueW( + TcpipInterfaceRegKey, + NULL, + L"EnableDeadGWDetect", + REG_DWORD, + &EnableDeadGWDetect, + sizeof(EnableDeadGWDetect)); + RegCloseKey(TcpipInterfaceRegKey); + if (LastError == ERROR_SUCCESS) + break; + if (LastError != ERROR_TRANSACTION_NOT_ACTIVE) + { + LOG_ERROR(LastError, L"Failed to set %s\\EnableDeadGWDetect", TcpipInterfaceRegPath); + goto cleanupTcpipAdapterRegKey; + } + Sleep(10); + } + + if (!WireGuardSetAdapterName(Adapter, Name)) + { + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to set adapter name %s", Name); + goto cleanupTcpipAdapterRegKey; + } + + for (int Tries = 0; Tries < 1000; ++Tries) + { + DEVPROPTYPE PropertyType; + NTSTATUS ProblemStatus; + if (SetupDiGetDevicePropertyW( + Adapter->DevInfo, + &Adapter->DevInfoData, + &DEVPKEY_Device_ProblemStatus, + &PropertyType, + (PBYTE)&ProblemStatus, + sizeof(ProblemStatus), + NULL, + 0) && + PropertyType == DEVPROP_TYPE_NTSTATUS) + { + if (ProblemStatus != STATUS_PNP_DEVICE_CONFIGURATION_PENDING || Tries == 999) + { + INT32 ProblemCode; + if (!SetupDiGetDevicePropertyW( + Adapter->DevInfo, + &Adapter->DevInfoData, + &DEVPKEY_Device_ProblemCode, + &PropertyType, + (PBYTE)&ProblemCode, + sizeof(ProblemCode), + NULL, + 0) || + PropertyType != DEVPROP_TYPE_INT32) + ProblemCode = 0; + LastError = RtlNtStatusToDosError(ProblemStatus); + if (LastError == ERROR_SUCCESS) + LastError = ERROR_NOT_READY; + LOG_ERROR(LastError, L"Failed to setup adapter (code: 0x%x, status: 0x%x)", ProblemCode, ProblemStatus); + goto cleanupTcpipAdapterRegKey; + } + Sleep(10); + } + else + break; + } + if (!EnsureDeviceObject(Adapter->DevInstanceID)) + { + LastError = LOG_LAST_ERROR(L"Device object file did not appear"); + goto cleanupTcpipAdapterRegKey; + } + LastError = ERROR_SUCCESS; + +cleanupTcpipAdapterRegKey: + RegCloseKey(TcpipAdapterRegKey); +cleanupNetDevRegKey: + RegCloseKey(NetDevRegKey); +cleanupDevice: + if (LastError != ERROR_SUCCESS) + { + SP_REMOVEDEVICE_PARAMS RemoveDeviceParams = { .ClassInstallHeader = { .cbSize = sizeof(SP_CLASSINSTALL_HEADER), + .InstallFunction = DIF_REMOVE }, + .Scope = DI_REMOVEDEVICE_GLOBAL }; + if (SetupDiSetClassInstallParamsW( + Adapter->DevInfo, + &Adapter->DevInfoData, + &RemoveDeviceParams.ClassInstallHeader, + sizeof(RemoveDeviceParams)) && + SetupDiCallClassInstaller(DIF_REMOVE, Adapter->DevInfo, &Adapter->DevInfoData)) + *RebootRequired = *RebootRequired || CheckReboot(Adapter->DevInfo, &Adapter->DevInfoData); + } + NamespaceReleaseMutex(Mutex); +cleanupDriverInfoList: + SelectDriverDeferredCleanup(DevInfoExistingAdapters, ExistingAdapters); + SetupDiDestroyDriverInfoList(Adapter->DevInfo, &Adapter->DevInfoData, SPDIT_COMPATDRIVER); +cleanupAdapter: + if (LastError != ERROR_SUCCESS) + WireGuardFreeAdapter(Adapter); + return RET_ERROR(Adapter, LastError); +} + +_Use_decl_annotations_ +BOOL WINAPI +WireGuardDeleteAdapter(WIREGUARD_ADAPTER *Adapter, BOOL *RebootRequired) +{ + WireGuardSetAdapterLogging(Adapter, WIREGUARD_ADAPTER_LOG_OFF); + + BOOL DummyRebootRequired; + if (!RebootRequired) + RebootRequired = &DummyRebootRequired; + *RebootRequired = FALSE; +#ifdef MAYBE_WOW64 + if (NativeMachine != IMAGE_FILE_PROCESS) + return DeleteAdapterViaRundll32(Adapter, RebootRequired); +#endif + + DWORD LastError = ERROR_SUCCESS; + HANDLE Mutex = NamespaceTakePoolMutex(Adapter->Pool); + if (!Mutex) + { + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to take %s pool mutex", Adapter->Pool); + goto cleanup; + } + + SP_DEVINSTALL_PARAMS_W DevInstallParams = { .cbSize = sizeof(DevInstallParams) }; + if (!SetupDiGetDeviceInstallParamsW(Adapter->DevInfo, &Adapter->DevInfoData, &DevInstallParams)) + { + LastError = LOG_LAST_ERROR( + L"Failed to retrieve adapter %u device installation parameters", Adapter->DevInfoData.DevInst); + goto cleanupMutex; + } + DevInstallParams.Flags |= DI_QUIETINSTALL; + if (!SetupDiSetDeviceInstallParamsW(Adapter->DevInfo, &Adapter->DevInfoData, &DevInstallParams)) + { + LastError = + LOG_LAST_ERROR(L"Failed to set adapter %u device installation parameters", Adapter->DevInfoData.DevInst); + goto cleanupMutex; + } + + SP_REMOVEDEVICE_PARAMS Params = { .ClassInstallHeader = { .cbSize = sizeof(SP_CLASSINSTALL_HEADER), + .InstallFunction = DIF_REMOVE }, + .Scope = DI_REMOVEDEVICE_GLOBAL }; + if ((!SetupDiSetClassInstallParamsW( + Adapter->DevInfo, &Adapter->DevInfoData, &Params.ClassInstallHeader, sizeof(Params)) || + !SetupDiCallClassInstaller(DIF_REMOVE, Adapter->DevInfo, &Adapter->DevInfoData)) && + GetLastError() != ERROR_NO_SUCH_DEVINST) + LastError = LOG_LAST_ERROR(L"Failed to remove adapter %u", Adapter->DevInfoData.DevInst); + + *RebootRequired = *RebootRequired || CheckReboot(Adapter->DevInfo, &Adapter->DevInfoData); + +cleanupMutex: + NamespaceReleaseMutex(Mutex); +cleanup: + return RET_ERROR(TRUE, LastError); +} + +static _Return_type_success_(return != FALSE) +BOOL +DeleteAllOurAdapters(_In_z_ LPCWSTR Pool, _Inout_ BOOL *RebootRequired) +{ + HANDLE Mutex = NamespaceTakePoolMutex(Pool); + if (!Mutex) + { + LOG(WIREGUARD_LOG_ERR, L"Failed to take %s pool mutex", Pool); + return FALSE; + } + DWORD LastError = ERROR_SUCCESS; + HDEVINFO DevInfo = SetupDiGetClassDevsExW(&GUID_DEVCLASS_NET, NULL, NULL, DIGCF_PRESENT, NULL, NULL, NULL); + if (DevInfo == INVALID_HANDLE_VALUE) + { + LastError = LOG_LAST_ERROR(L"Failed to get present adapters"); + goto cleanupMutex; + } + SP_REMOVEDEVICE_PARAMS Params = { .ClassInstallHeader = { .cbSize = sizeof(SP_CLASSINSTALL_HEADER), + .InstallFunction = DIF_REMOVE }, + .Scope = DI_REMOVEDEVICE_GLOBAL }; + for (DWORD EnumIndex = 0;; ++EnumIndex) + { + SP_DEVINFO_DATA DevInfoData = { .cbSize = sizeof(SP_DEVINFO_DATA) }; + if (!SetupDiEnumDeviceInfo(DevInfo, EnumIndex, &DevInfoData)) + { + if (GetLastError() == ERROR_NO_MORE_ITEMS) + break; + continue; + } + + if (!IsOurAdapter(DevInfo, &DevInfoData) || !IsPoolMember(Pool, DevInfo, &DevInfoData)) + continue; + + LOG(WIREGUARD_LOG_INFO, L"Removing adapter %u", DevInfoData.DevInst); + if ((!SetupDiSetClassInstallParamsW(DevInfo, &DevInfoData, &Params.ClassInstallHeader, sizeof(Params)) || + !SetupDiCallClassInstaller(DIF_REMOVE, DevInfo, &DevInfoData)) && + GetLastError() != ERROR_NO_SUCH_DEVINST) + { + LOG_LAST_ERROR(L"Failed to remove adapter %u", DevInfoData.DevInst); + LastError = LastError != ERROR_SUCCESS ? LastError : GetLastError(); + } + *RebootRequired = *RebootRequired || CheckReboot(DevInfo, &DevInfoData); + } + SetupDiDestroyDeviceInfoList(DevInfo); +cleanupMutex: + NamespaceReleaseMutex(Mutex); + return RET_ERROR(TRUE, LastError); +} + +_Use_decl_annotations_ +BOOL WINAPI +WireGuardDeletePoolDriver(LPCWSTR Pool, BOOL *RebootRequired) +{ + BOOL DummyRebootRequired; + if (!RebootRequired) + RebootRequired = &DummyRebootRequired; + *RebootRequired = FALSE; + + DWORD LastError = ERROR_SUCCESS; +#ifdef MAYBE_WOW64 + if (NativeMachine != IMAGE_FILE_PROCESS) + { + LastError = DeletePoolDriverViaRundll32(Pool, RebootRequired) ? ERROR_SUCCESS : GetLastError(); + goto cleanup; + } +#endif + + if (!DeleteAllOurAdapters(Pool, RebootRequired)) + { + LastError = GetLastError(); + goto cleanup; + } + + HANDLE DriverInstallationLock = NamespaceTakeDriverInstallationMutex(); + if (!DriverInstallationLock) + { + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to take driver installation mutex"); + goto cleanup; + } + HDEVINFO DeviceInfoSet = SetupDiGetClassDevsW(&GUID_DEVCLASS_NET, NULL, NULL, 0); + if (!DeviceInfoSet) + { + LastError = LOG_LAST_ERROR(L"Failed to get adapter information"); + goto cleanupDriverInstallationLock; + } + if (!SetupDiBuildDriverInfoList(DeviceInfoSet, NULL, SPDIT_CLASSDRIVER)) + { + LastError = LOG_LAST_ERROR(L"Failed building driver info list"); + goto cleanupDeviceInfoSet; + } + for (DWORD EnumIndex = 0;; ++EnumIndex) + { + SP_DRVINFO_DATA_W DriverInfo = { .cbSize = sizeof(DriverInfo) }; + if (!SetupDiEnumDriverInfoW(DeviceInfoSet, NULL, SPDIT_CLASSDRIVER, EnumIndex, &DriverInfo)) + { + if (GetLastError() == ERROR_NO_MORE_ITEMS) + break; + continue; + } + SP_DRVINFO_DETAIL_DATA_W *DriverDetail = GetAdapterDrvInfoDetail(DeviceInfoSet, NULL, &DriverInfo); + if (!DriverDetail) + continue; + if (!_wcsicmp(DriverDetail->HardwareID, WIREGUARD_HWID)) + { + LPCWSTR Path = PathFindFileNameW(DriverDetail->InfFileName); + LOG(WIREGUARD_LOG_INFO, L"Removing driver %s", Path); + if (!SetupUninstallOEMInfW(Path, 0, NULL)) + { + LOG_LAST_ERROR(L"Unable to remove driver %s", Path); + LastError = LastError != ERROR_SUCCESS ? LastError : GetLastError(); + } + } + Free(DriverDetail); + } + SetupDiDestroyDriverInfoList(DeviceInfoSet, NULL, SPDIT_CLASSDRIVER); +cleanupDeviceInfoSet: + SetupDiDestroyDeviceInfoList(DeviceInfoSet); +cleanupDriverInstallationLock: + NamespaceReleaseMutex(DriverInstallationLock); +cleanup: + return RET_ERROR(TRUE, LastError); +} + +_Use_decl_annotations_ +BOOL WINAPI +WireGuardEnumAdapters(LPCWSTR Pool, WIREGUARD_ENUM_CALLBACK Func, LPARAM Param) +{ + DWORD LastError = ERROR_SUCCESS; + HANDLE Mutex = NamespaceTakePoolMutex(Pool); + if (!Mutex) + { + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to take %s pool mutex", Pool); + goto cleanup; + } + HDEVINFO DevInfo = SetupDiGetClassDevsExW(&GUID_DEVCLASS_NET, NULL, NULL, DIGCF_PRESENT, NULL, NULL, NULL); + if (DevInfo == INVALID_HANDLE_VALUE) + { + LastError = LOG_LAST_ERROR(L"Failed to get present adapters"); + goto cleanupMutex; + } + BOOL Continue = TRUE; + for (DWORD EnumIndex = 0; Continue; ++EnumIndex) + { + WIREGUARD_ADAPTER Adapter = { .DevInfo = DevInfo, .DevInfoData.cbSize = sizeof(Adapter.DevInfoData) }; + if (!SetupDiEnumDeviceInfo(DevInfo, EnumIndex, &Adapter.DevInfoData)) + { + if (GetLastError() == ERROR_NO_MORE_ITEMS) + break; + continue; + } + + if (!IsOurAdapter(DevInfo, &Adapter.DevInfoData) || !IsPoolMember(Pool, DevInfo, &Adapter.DevInfoData)) + continue; + + if (!PopulateAdapterData(&Adapter, Pool)) + { + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to populate adapter %u data", Adapter.DevInfoData.DevInst); + break; + } + Continue = Func(&Adapter, Param); + } + SetupDiDestroyDeviceInfoList(DevInfo); +cleanupMutex: + NamespaceReleaseMutex(Mutex); +cleanup: + return RET_ERROR(TRUE, LastError); +} diff --git a/api/adapter.h b/api/adapter.h new file mode 100644 index 0000000..3ae88c0 --- /dev/null +++ b/api/adapter.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#pragma once + +#include "wireguard.h" +#include <IPExport.h> +#include <SetupAPI.h> +#include <Windows.h> + +#define MAX_INSTANCE_ID MAX_PATH /* TODO: Is MAX_PATH always enough? */ +#define WIREGUARD_HWID L"WireGuard" + +/** + * WireGuard adapter descriptor. + */ +typedef struct _WIREGUARD_ADAPTER +{ + HDEVINFO DevInfo; + SP_DEVINFO_DATA DevInfoData; + GUID CfgInstanceID; + WCHAR DevInstanceID[MAX_INSTANCE_ID]; + DWORD LuidIndex; + DWORD IfType; + DWORD IfIndex; + WCHAR Pool[WIREGUARD_MAX_POOL]; + HANDLE LogThread; + DWORD LogState; +} WIREGUARD_ADAPTER; + +/** + * @copydoc WIREGUARD_FREE_ADAPTER_FUNC + */ +WIREGUARD_FREE_ADAPTER_FUNC WireGuardFreeAdapter; + +/** + * @copydoc WIREGUARD_CREATE_ADAPTER_FUNC + */ +WIREGUARD_CREATE_ADAPTER_FUNC WireGuardCreateAdapter; + +/** + * @copydoc WIREGUARD_OPEN_ADAPTER_FUNC + */ +WIREGUARD_OPEN_ADAPTER_FUNC WireGuardOpenAdapter; + +/** + * @copydoc WIREGUARD_DELETE_ADAPTER_FUNC + */ +WIREGUARD_DELETE_ADAPTER_FUNC WireGuardDeleteAdapter; + +/** + * @copydoc WIREGUARD_ENUM_ADAPTERS_FUNC + */ +WIREGUARD_ENUM_ADAPTERS_FUNC WireGuardEnumAdapters; + +/** + * @copydoc WIREGUARD_DELETE_POOL_DRIVER_FUNC + */ +WIREGUARD_DELETE_POOL_DRIVER_FUNC WireGuardDeletePoolDriver; + +/** + * @copydoc WIREGUARD_GET_ADAPTER_LUID_FUNC + */ +WIREGUARD_GET_ADAPTER_LUID_FUNC WireGuardGetAdapterLUID; + +/** + * @copydoc WIREGUARD_GET_ADAPTER_NAME_FUNC + */ +WIREGUARD_GET_ADAPTER_NAME_FUNC WireGuardGetAdapterName; + +/** + * @copydoc WIREGUARD_SET_ADAPTER_NAME_FUNC + */ +WIREGUARD_SET_ADAPTER_NAME_FUNC WireGuardSetAdapterName; + +/** + * @copydoc WIREGUARD_GET_RUNNING_DRIVER_VERSION_FUNC + */ +WIREGUARD_GET_RUNNING_DRIVER_VERSION_FUNC WireGuardGetRunningDriverVersion; + +/** + * Returns a handle to the adapter device object. + * + * @param Adapter Adapter handle obtained with WireGuardOpenAdapter or WireGuardCreateAdapter. + * + * @return If the function succeeds, the return value is adapter device object handle. + * If the function fails, the return value is INVALID_HANDLE_VALUE. To get extended error + * information, call GetLastError. + */ +_Return_type_success_(return != INVALID_HANDLE_VALUE) +HANDLE WINAPI +AdapterOpenDeviceObject(_In_ const WIREGUARD_ADAPTER *Adapter); + +/** + * Returns an adapter object based on a devnode instance ID. + * + * @param Pool Pool name of adapter object to be opened. + * + * @param DevInstanceID Instance ID of devnode for opening adapter. + * + * @return If the function succeeds, the return value is adapter object.. + * If the function fails, the return value is NULL. To get extended error + * information, call GetLastError. + */ +_Must_inspect_result_ +_Return_type_success_(return != NULL) +_Post_maybenull_ +WIREGUARD_ADAPTER * +AdapterOpenFromDevInstanceId(_In_z_ LPCWSTR Pool, _In_z_ LPCWSTR DevInstanceID); diff --git a/api/api.vcxproj b/api/api.vcxproj new file mode 100644 index 0000000..6e32894 --- /dev/null +++ b/api/api.vcxproj @@ -0,0 +1,94 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Label="Globals"> + <ProjectGuid>{99648503-7DFB-4C06-A87A-E7B66E93FF84}</ProjectGuid> + <RootNamespace>api</RootNamespace> + <ProjectName>api</ProjectName> + </PropertyGroup> + <PropertyGroup Label="Configuration"> + <ConfigurationType>DynamicLibrary</ConfigurationType> + <PlatformToolset>WindowsApplicationForDrivers10.0</PlatformToolset> + </PropertyGroup> + <Import Project="..\wireguard-nt.props" /> + <PropertyGroup> + <TargetName>wireguard</TargetName> + <IgnoreImportLibrary>true</IgnoreImportLibrary> + </PropertyGroup> + <ItemDefinitionGroup> + <ClCompile> + <PreprocessorDefinitions>_WINDOWS;_USRDLL;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions Condition="Exists('$(OutDir)whql\')">HAVE_WHQL;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions Condition="'$(Platform)'=='Win32'">MAYBE_WOW64;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions Condition="'$(Platform)'=='x64'">ACCEPT_WOW64;MAYBE_WOW64;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions Condition="'$(Platform)'=='ARM'">MAYBE_WOW64;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions Condition="'$(Platform)'=='ARM64'">ACCEPT_WOW64;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <AdditionalOptions>/volatile:iso %(AdditionalOptions)</AdditionalOptions> + <DisableSpecificWarnings>4100;4201;$(DisableSpecificWarnings)</DisableSpecificWarnings> + <AdditionalIncludeDirectories>$(IntDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + </ClCompile> + <ResourceCompile> + <AdditionalIncludeDirectories>..\$(Configuration)\$(WireGuardPlatform);..\$(Configuration);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <PreprocessorDefinitions Condition="Exists('$(OutDir)whql\')">HAVE_WHQL;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions Condition="Exists('..\$(Configuration)\arm64\wireguard.dll')">BUILT_ARM64_WOW64;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions Condition="Exists('..\$(Configuration)\amd64\wireguard.dll')">BUILT_AMD64_WOW64;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions Condition="'$(Platform)'=='Win32'">WANT_ARM64_WOW64;WANT_AMD64_WOW64;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions Condition="'$(Platform)'=='x64'">WANT_ARM64_WOW64;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions Condition="'$(Platform)'=='ARM'">WANT_ARM64_WOW64;%(PreprocessorDefinitions)</PreprocessorDefinitions> + </ResourceCompile> + <Link> + <DelayLoadDLLs>advapi32.dll;bcrypt.dll;crypt32.dll;cfgmgr32.dll;iphlpapi.dll;ole32.dll;nci.dll;setupapi.dll;shell32.dll;shlwapi.dll;version.dll</DelayLoadDLLs> + <AdditionalDependencies>Bcrypt.lib;Crypt32.lib;Cfgmgr32.lib;Iphlpapi.lib;$(IntDir)nci.lib;ntdll.lib;Setupapi.lib;shlwapi.lib;version.lib;%(AdditionalDependencies)</AdditionalDependencies> + <ModuleDefinitionFile>exports.def</ModuleDefinitionFile> + <SubSystem>Windows</SubSystem> + <ImportLibrary>$(IntDir)$(TargetName).lib</ImportLibrary> + </Link> + </ItemDefinitionGroup> + <ItemGroup> + <ResourceCompile Include="resources.rc" /> + </ItemGroup> + <ItemGroup> + <None Include="exports.def" /> + <None Include="nci.def" /> + </ItemGroup> + <ItemGroup> + <ClInclude Include="main.h" /> + <ClInclude Include="adapter.h" /> + <ClInclude Include="logger.h" /> + <ClInclude Include="namespace.h" /> + <ClInclude Include="nci.h" /> + <ClInclude Include="ntdll.h" /> + <ClInclude Include="registry.h" /> + <ClInclude Include="resource.h" /> + <ClInclude Include="rundll32.h" /> + <ClInclude Include="wireguard.h" /> + </ItemGroup> + <ItemGroup> + <ClCompile Include="main.c" /> + <ClCompile Include="adapter.c" /> + <ClCompile Include="logger.c" /> + <ClCompile Include="namespace.c" /> + <ClCompile Include="registry.c" /> + <ClCompile Include="resource.c" /> + <ClCompile Include="configuration.c" /> + <ClCompile Include="rundll32.c" /> + </ItemGroup> + <Import Project="..\wireguard-nt.props.user" Condition="exists('..\wireguard-nt.props.user')" /> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> + <ImportGroup Label="ExtensionTargets" /> + <PropertyGroup> + <CleanDependsOn>CleanInfVersion;CleanNci;$(CleanDependsOn)</CleanDependsOn> + </PropertyGroup> + <Target Name="BuildInfVersion" BeforeTargets="ClCompile" Inputs="$(OutDir)driver\wireguard.inf" Outputs="$(IntDir)wireguard-inf.h"> + <Exec Command="cscript.exe /nologo "extract-driverver.js" < "$(OutDir)driver\wireguard.inf" > "$(IntDir)wireguard-inf.h"" /> + </Target> + <Target Name="CleanInfVersion"> + <Delete Files="$(IntDir)wireguard-inf.h" /> + </Target> + <Target Name="BuildNci" BeforeTargets="Link" Inputs="$(ProjectDir)nci.def;$(ProjectDir)nci.h" Outputs="$(IntDir)nci.lib"> + <Exec Command="cl.exe /nologo /DGENERATE_LIB /Ob0 /c /Fo"$(IntDir)nci.obj" /Tc "nci.h"" /> + <Exec Command="lib.exe /def:"$(ProjectDir)nci.def" /out:"$(IntDir)nci.lib" /machine:$(PlatformTarget) /nologo "$(IntDir)nci.obj"" /> + </Target> + <Target Name="CleanNci"> + <Delete Files="$(IntDir)nci.obj;$(IntDir)nci.lib" /> + </Target> +</Project>
\ No newline at end of file diff --git a/api/api.vcxproj.filters b/api/api.vcxproj.filters new file mode 100644 index 0000000..f6bf8fa --- /dev/null +++ b/api/api.vcxproj.filters @@ -0,0 +1,88 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup> + <Filter Include="Source Files"> + <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier> + <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions> + </Filter> + <Filter Include="Header Files"> + <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier> + <Extensions>h;hh;hpp;hxx;hm;inl;inc;ipp;xsd</Extensions> + </Filter> + <Filter Include="Resource Files"> + <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier> + <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions> + </Filter> + </ItemGroup> + <ItemGroup> + <ResourceCompile Include="resources.rc"> + <Filter>Resource Files</Filter> + </ResourceCompile> + </ItemGroup> + <ItemGroup> + <None Include="exports.def"> + <Filter>Source Files</Filter> + </None> + <None Include="nci.def"> + <Filter>Source Files</Filter> + </None> + </ItemGroup> + <ItemGroup> + <ClInclude Include="nci.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="namespace.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="registry.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="logger.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="resource.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="adapter.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="wireguard.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="main.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="ntdll.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="rundll32.h"> + <Filter>Header Files</Filter> + </ClInclude> + </ItemGroup> + <ItemGroup> + <ClCompile Include="namespace.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="rundll32.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="registry.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="logger.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="resource.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="adapter.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="configuration.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="main.c"> + <Filter>Source Files</Filter> + </ClCompile> + </ItemGroup> +</Project>
\ No newline at end of file diff --git a/api/configuration.c b/api/configuration.c new file mode 100644 index 0000000..5bd0030 --- /dev/null +++ b/api/configuration.c @@ -0,0 +1,246 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#include "../driver/ioctl.h" +#include "wireguard.h" +#include "adapter.h" +#include "logger.h" +#include <Windows.h> +#include <stdlib.h> + +static_assert(sizeof(WG_IOCTL_INTERFACE) == sizeof(WIREGUARD_INTERFACE), "Interface struct mismatch"); +static_assert( + offsetof(WG_IOCTL_INTERFACE, Flags) == offsetof(WIREGUARD_INTERFACE, Flags), + "Interface->Flags struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_INTERFACE, Flags) == RTL_FIELD_SIZE(WIREGUARD_INTERFACE, Flags), + "Interface->Flags struct mismatch"); +static_assert( + offsetof(WG_IOCTL_INTERFACE, ListenPort) == offsetof(WIREGUARD_INTERFACE, ListenPort), + "Interface->ListenPort struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_INTERFACE, ListenPort) == RTL_FIELD_SIZE(WIREGUARD_INTERFACE, ListenPort), + "Interface->ListenPort struct mismatch"); +static_assert( + offsetof(WG_IOCTL_INTERFACE, PrivateKey) == offsetof(WIREGUARD_INTERFACE, PrivateKey), + "Interface->PrivateKey struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_INTERFACE, PrivateKey) == RTL_FIELD_SIZE(WIREGUARD_INTERFACE, PrivateKey), + "Interface->PrivateKey struct mismatch"); +static_assert( + offsetof(WG_IOCTL_INTERFACE, PublicKey) == offsetof(WIREGUARD_INTERFACE, PublicKey), + "Interface->PublicKey struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_INTERFACE, PublicKey) == RTL_FIELD_SIZE(WIREGUARD_INTERFACE, PublicKey), + "Interface->PublicKey struct mismatch"); +static_assert( + offsetof(WG_IOCTL_INTERFACE, PeersCount) == offsetof(WIREGUARD_INTERFACE, PeersCount), + "Interface->PeersCount struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_INTERFACE, PeersCount) == RTL_FIELD_SIZE(WIREGUARD_INTERFACE, PeersCount), + "Interface->PeersCount struct mismatch"); +static_assert( + WG_IOCTL_INTERFACE_HAS_PUBLIC_KEY == WIREGUARD_INTERFACE_HAS_PUBLIC_KEY, + "INTERFACE_HAS_PUBLIC_KEY flag mismatch"); +static_assert( + WG_IOCTL_INTERFACE_HAS_PRIVATE_KEY == WIREGUARD_INTERFACE_HAS_PRIVATE_KEY, + "INTERFACE_HAS_PRIVATE_KEY flag mismatch"); +static_assert( + WG_IOCTL_INTERFACE_HAS_LISTEN_PORT == WIREGUARD_INTERFACE_HAS_LISTEN_PORT, + "INTERFACE_HAS_LISTEN_PORT flag mismatch"); +static_assert( + WG_IOCTL_INTERFACE_REPLACE_PEERS == WIREGUARD_INTERFACE_REPLACE_PEERS, + "INTERFACE_REPLACE_PEERS flag mismatch"); +static_assert(sizeof(WG_IOCTL_PEER) == sizeof(WIREGUARD_PEER), "Peer struct mismatch"); +static_assert(offsetof(WG_IOCTL_PEER, Flags) == offsetof(WIREGUARD_PEER, Flags), "Peer->Flags struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_PEER, Flags) == RTL_FIELD_SIZE(WIREGUARD_PEER, Flags), + "Peer->Flags struct mismatch"); +static_assert( + offsetof(WG_IOCTL_PEER, ProtocolVersion) == offsetof(WIREGUARD_PEER, Reserved), + "Peer->Reserved struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_PEER, ProtocolVersion) == RTL_FIELD_SIZE(WIREGUARD_PEER, Reserved), + "Peer->Reserved struct mismatch"); +static_assert( + offsetof(WG_IOCTL_PEER, PublicKey) == offsetof(WIREGUARD_PEER, PublicKey), + "Peer->PublicKey struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_PEER, PublicKey) == RTL_FIELD_SIZE(WIREGUARD_PEER, PublicKey), + "Peer->PublicKey struct mismatch"); +static_assert( + offsetof(WG_IOCTL_PEER, PresharedKey) == offsetof(WIREGUARD_PEER, PresharedKey), + "Peer->PresharedKey struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_PEER, PresharedKey) == RTL_FIELD_SIZE(WIREGUARD_PEER, PresharedKey), + "Peer->PresharedKey struct mismatch"); +static_assert( + offsetof(WG_IOCTL_PEER, PersistentKeepalive) == offsetof(WIREGUARD_PEER, PersistentKeepalive), + "Peer->PersistentKeepalive struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_PEER, PersistentKeepalive) == RTL_FIELD_SIZE(WIREGUARD_PEER, PersistentKeepalive), + "Peer->PersistentKeepalive struct mismatch"); +static_assert( + offsetof(WG_IOCTL_PEER, Endpoint) == offsetof(WIREGUARD_PEER, Endpoint), + "Peer->Endpoint struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_PEER, Endpoint) == RTL_FIELD_SIZE(WIREGUARD_PEER, Endpoint), + "Peer->Endpoint struct mismatch"); +static_assert(offsetof(WG_IOCTL_PEER, TxBytes) == offsetof(WIREGUARD_PEER, TxBytes), "Peer->TxBytes struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_PEER, TxBytes) == RTL_FIELD_SIZE(WIREGUARD_PEER, TxBytes), + "Peer->TxBytes struct mismatch"); +static_assert(offsetof(WG_IOCTL_PEER, RxBytes) == offsetof(WIREGUARD_PEER, RxBytes), "Peer->RxBytes struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_PEER, RxBytes) == RTL_FIELD_SIZE(WIREGUARD_PEER, RxBytes), + "Peer->RxBytes struct mismatch"); +static_assert( + offsetof(WG_IOCTL_PEER, LastHandshake) == offsetof(WIREGUARD_PEER, LastHandshake), + "Peer->LastHandshake struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_PEER, LastHandshake) == RTL_FIELD_SIZE(WIREGUARD_PEER, LastHandshake), + "Peer->LastHandshake struct mismatch"); +static_assert( + offsetof(WG_IOCTL_PEER, AllowedIPsCount) == offsetof(WIREGUARD_PEER, AllowedIPsCount), + "Peer->AllowedIPsCount struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_PEER, AllowedIPsCount) == RTL_FIELD_SIZE(WIREGUARD_PEER, AllowedIPsCount), + "Peer->AllowedIPsCount struct mismatch"); +static_assert(WG_IOCTL_PEER_HAS_PUBLIC_KEY == WIREGUARD_PEER_HAS_PUBLIC_KEY, "PEER_HAS_PUBLIC_KEY flag mismatch"); +static_assert( + WG_IOCTL_PEER_HAS_PRESHARED_KEY == WIREGUARD_PEER_HAS_PRESHARED_KEY, + "PEER_HAS_PRESHARED_KEY flag mismatch"); +static_assert( + WG_IOCTL_PEER_HAS_PERSISTENT_KEEPALIVE == WIREGUARD_PEER_HAS_PERSISTENT_KEEPALIVE, + "PEER_HAS_PERSISTENT_KEEPALIVE flag mismatch"); +static_assert(WG_IOCTL_PEER_HAS_ENDPOINT == WIREGUARD_PEER_HAS_ENDPOINT, "PEER_HAS_ENDPOINT flag mismatch"); +static_assert( + WG_IOCTL_PEER_REPLACE_ALLOWED_IPS == WIREGUARD_PEER_REPLACE_ALLOWED_IPS, + "PEER_REPLACE_ALLOWED_IPS flag mismatch"); +static_assert(WG_IOCTL_PEER_REMOVE == WIREGUARD_PEER_REMOVE, "PEER_REMOVE flag mismatch"); +static_assert(WG_IOCTL_PEER_UPDATE == WIREGUARD_PEER_UPDATE, "PEER_UPDATE flag mismatch"); +static_assert(sizeof(WG_IOCTL_ALLOWED_IP) == sizeof(WIREGUARD_ALLOWED_IP), "Allowed IP struct mismatch"); +static_assert( + offsetof(WG_IOCTL_ALLOWED_IP, AddressFamily) == offsetof(WIREGUARD_ALLOWED_IP, AddressFamily), + "AllowedIp->AddressFamily struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_ALLOWED_IP, AddressFamily) == RTL_FIELD_SIZE(WIREGUARD_ALLOWED_IP, AddressFamily), + "AllowedIp->AddressFamily struct mismatch"); +static_assert( + offsetof(WG_IOCTL_ALLOWED_IP, Cidr) == offsetof(WIREGUARD_ALLOWED_IP, Cidr), + "AllowedIp->cidr struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_ALLOWED_IP, Cidr) == RTL_FIELD_SIZE(WIREGUARD_ALLOWED_IP, Cidr), + "AllowedIp->cidr struct mismatch"); +static_assert( + offsetof(WG_IOCTL_ALLOWED_IP, Address.V4) == offsetof(WIREGUARD_ALLOWED_IP, Address.V4), + "AllowedIp->Address.V4 struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_ALLOWED_IP, Address.V4) == RTL_FIELD_SIZE(WIREGUARD_ALLOWED_IP, Address.V4), + "AllowedIp->Address.V4 struct mismatch"); +static_assert( + offsetof(WG_IOCTL_ALLOWED_IP, Address.V6) == offsetof(WIREGUARD_ALLOWED_IP, Address.V6), + "AllowedIp->Address.V6 struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_ALLOWED_IP, Address.V6) == RTL_FIELD_SIZE(WIREGUARD_ALLOWED_IP, Address.V6), + "AllowedIp->Address.V6 struct mismatch"); +static_assert( + offsetof(WG_IOCTL_ALLOWED_IP, Address) == offsetof(WIREGUARD_ALLOWED_IP, Address), + "AllowedIp->Address struct mismatch"); +static_assert( + RTL_FIELD_SIZE(WG_IOCTL_ALLOWED_IP, Address) == RTL_FIELD_SIZE(WIREGUARD_ALLOWED_IP, Address), + "AllowedIp->Address struct mismatch"); +static_assert(sizeof(WG_IOCTL_ADAPTER_STATE) == sizeof(WIREGUARD_ADAPTER_STATE), "Adapter state mismatch"); +static_assert(WG_IOCTL_ADAPTER_STATE_DOWN == WIREGUARD_ADAPTER_STATE_DOWN, "Adapter state down mismatch"); +static_assert(WG_IOCTL_ADAPTER_STATE_UP == WIREGUARD_ADAPTER_STATE_UP, "Adapter state up mismatch"); + +WIREGUARD_SET_ADAPTER_STATE_FUNC WireGuardSetAdapterState; +_Use_decl_annotations_ +BOOL WINAPI +WireGuardSetAdapterState(WIREGUARD_ADAPTER *Adapter, WIREGUARD_ADAPTER_STATE State) +{ + switch (State) + { + case WIREGUARD_ADAPTER_STATE_UP: + case WIREGUARD_ADAPTER_STATE_DOWN: + break; + default: + SetLastError(ERROR_INVALID_PARAMETER); + return FALSE; + } + HANDLE ControlFile = AdapterOpenDeviceObject(Adapter); + if (ControlFile == INVALID_HANDLE_VALUE) + return FALSE; + DWORD BytesReturned; + if (!DeviceIoControl(ControlFile, WG_IOCTL_SET_ADAPTER_STATE, &State, sizeof(State), NULL, 0, &BytesReturned, NULL)) + { + DWORD LastError = GetLastError(); + CloseHandle(ControlFile); + SetLastError(LastError); + return FALSE; + } + CloseHandle(ControlFile); + return TRUE; +} + +WIREGUARD_GET_ADAPTER_STATE_FUNC WireGuardGetAdapterState; +_Use_decl_annotations_ +BOOL WINAPI +WireGuardGetAdapterState(WIREGUARD_ADAPTER *Adapter, WIREGUARD_ADAPTER_STATE *State) +{ + HANDLE ControlFile = AdapterOpenDeviceObject(Adapter); + if (ControlFile == INVALID_HANDLE_VALUE) + return FALSE; + DWORD BytesReturned; + WG_IOCTL_ADAPTER_STATE Op = WG_IOCTL_ADAPTER_STATE_QUERY; + if (!DeviceIoControl( + ControlFile, WG_IOCTL_SET_ADAPTER_STATE, &Op, sizeof(Op), State, sizeof(*State), &BytesReturned, NULL)) + { + DWORD LastError = GetLastError(); + CloseHandle(ControlFile); + SetLastError(LastError); + return FALSE; + } + CloseHandle(ControlFile); + return TRUE; +} + +WIREGUARD_SET_CONFIGURATION_FUNC WireGuardSetConfiguration; +_Use_decl_annotations_ +BOOL WINAPI +WireGuardSetConfiguration(WIREGUARD_ADAPTER *Adapter, const WIREGUARD_INTERFACE *Config, DWORD Bytes) +{ + HANDLE ControlFile = AdapterOpenDeviceObject(Adapter); + if (ControlFile == INVALID_HANDLE_VALUE) + return FALSE; + if (!DeviceIoControl(ControlFile, WG_IOCTL_SET, NULL, 0, (VOID *)Config, Bytes, &Bytes, NULL)) + { + DWORD LastError = GetLastError(); + CloseHandle(ControlFile); + SetLastError(LastError); + return FALSE; + } + CloseHandle(ControlFile); + return TRUE; +} + +WIREGUARD_GET_CONFIGURATION_FUNC WireGuardGetConfiguration; +_Use_decl_annotations_ +BOOL WINAPI +WireGuardGetConfiguration(WIREGUARD_ADAPTER *Adapter, WIREGUARD_INTERFACE *Config, DWORD *Bytes) +{ + HANDLE ControlFile = AdapterOpenDeviceObject(Adapter); + if (ControlFile == INVALID_HANDLE_VALUE) + return FALSE; + if (!DeviceIoControl(ControlFile, WG_IOCTL_GET, NULL, 0, Config, *Bytes, Bytes, NULL)) + { + DWORD LastError = GetLastError(); + CloseHandle(ControlFile); + SetLastError(LastError); + return FALSE; + } + CloseHandle(ControlFile); + return TRUE; +} diff --git a/api/exports.def b/api/exports.def new file mode 100644 index 0000000..e9879bb --- /dev/null +++ b/api/exports.def @@ -0,0 +1,18 @@ +LIBRARY wireguard.dll +EXPORTS + WireGuardCreateAdapter + WireGuardDeleteAdapter + WireGuardDeletePoolDriver + WireGuardEnumAdapters + WireGuardFreeAdapter + WireGuardOpenAdapter + WireGuardGetAdapterLUID + WireGuardGetAdapterName + WireGuardGetAdapterState + WireGuardGetConfiguration + WireGuardGetRunningDriverVersion + WireGuardSetAdapterLogging + WireGuardSetAdapterName + WireGuardSetAdapterState + WireGuardSetConfiguration + WireGuardSetLogger diff --git a/api/extract-driverver.js b/api/extract-driverver.js new file mode 100644 index 0000000..26ac21b --- /dev/null +++ b/api/extract-driverver.js @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +while (!WScript.StdIn.AtEndOfStream) { + var line = WScript.StdIn.ReadLine(); + if (line.substr(0, 12) != "DriverVer = ") + continue; + var val = line.substr(12).split(","); + var date = val[0].split("/"); + var ver = val[1].split("."); + var time = Date.UTC(date[2], date[0] - 1, date[1]).toString() + WScript.Echo("#define WIREGUARD_INF_FILETIME { (DWORD)((" + time + "0000ULL + 116444736000000000ULL) & 0xffffffffU), (DWORD)((" + time + "0000ULL + 116444736000000000ULL) >> 32) }") + WScript.Echo("#define WIREGUARD_INF_VERSION ((" + ver[0] + "ULL << 48) | (" + ver[1] + "ULL << 32) | (" + ver[2] + "ULL << 16) | (" + ver[3] + "ULL << 0))") + break; +} diff --git a/api/logger.c b/api/logger.c new file mode 100644 index 0000000..5448d97 --- /dev/null +++ b/api/logger.c @@ -0,0 +1,234 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#include "../driver/ioctl.h" +#include "logger.h" +#include "adapter.h" +#include "ntdll.h" +#include <Windows.h> +#include <iphlpapi.h> +#include <winternl.h> +#include <wchar.h> +#include <stdlib.h> + +static BOOL CALLBACK +NopLogger(_In_ WIREGUARD_LOGGER_LEVEL Level, _In_z_ LPCWSTR LogLine) +{ + return TRUE; +} + +WIREGUARD_LOGGER_CALLBACK Logger = NopLogger; + +_Use_decl_annotations_ +VOID WINAPI +WireGuardSetLogger(WIREGUARD_LOGGER_CALLBACK NewLogger) +{ + if (!NewLogger) + NewLogger = NopLogger; + Logger = NewLogger; +} + +static VOID +StrTruncate(_Inout_count_(StrChars) LPWSTR Str, _In_ SIZE_T StrChars) +{ + Str[StrChars - 2] = L'\u2026'; /* Horizontal Ellipsis */ + Str[StrChars - 1] = 0; +} + +_Use_decl_annotations_ +DWORD +LoggerLog(WIREGUARD_LOGGER_LEVEL Level, LPCWSTR Function, LPCWSTR LogLine) +{ + DWORD LastError = GetLastError(); + if (Function) + { + WCHAR Combined[0x400]; + if (_snwprintf_s(Combined, _countof(Combined), _TRUNCATE, L"%s: %s", Function, LogLine) == -1) + StrTruncate(Combined, _countof(Combined)); + Logger(Level, Combined); + } + else + Logger(Level, LogLine); + SetLastError(LastError); + return LastError; +} + +_Use_decl_annotations_ +DWORD +LoggerLogV(WIREGUARD_LOGGER_LEVEL Level, LPCWSTR Function, LPCWSTR Format, va_list Args) +{ + DWORD LastError = GetLastError(); + WCHAR LogLine[0x400]; + if (_vsnwprintf_s(LogLine, _countof(LogLine), _TRUNCATE, Format, Args) == -1) + StrTruncate(LogLine, _countof(LogLine)); + if (Function) + LoggerLog(Level, Function, LogLine); + else + Logger(Level, LogLine); + SetLastError(LastError); + return LastError; +} + +_Use_decl_annotations_ +DWORD +LoggerError(DWORD Error, LPCWSTR Function, LPCWSTR Prefix) +{ + LPWSTR SystemMessage = NULL, FormattedMessage = NULL; + FormatMessageW( + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_MAX_WIDTH_MASK, + NULL, + HRESULT_FROM_SETUPAPI(Error), + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (VOID *)&SystemMessage, + 0, + NULL); + FormatMessageW( + FORMAT_MESSAGE_FROM_STRING | FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_ARGUMENT_ARRAY | + FORMAT_MESSAGE_MAX_WIDTH_MASK, + SystemMessage ? L"%4: %1: %3(Code 0x%2!08X!)" : L"%4: %1: Code 0x%2!08X!", + 0, + 0, + (VOID *)&FormattedMessage, + 0, + (va_list *)(DWORD_PTR[]){ (DWORD_PTR)Prefix, (DWORD_PTR)Error, (DWORD_PTR)SystemMessage, (DWORD_PTR)Function }); + if (FormattedMessage) + Logger(WIREGUARD_LOG_ERR, FormattedMessage); + LocalFree(FormattedMessage); + LocalFree(SystemMessage); + return Error; +} + +_Use_decl_annotations_ +DWORD +LoggerErrorV(DWORD Error, LPCWSTR Function, LPCWSTR Format, va_list Args) +{ + WCHAR Prefix[0x400]; + if (_vsnwprintf_s(Prefix, _countof(Prefix), _TRUNCATE, Format, Args) == -1) + StrTruncate(Prefix, _countof(Prefix)); + return LoggerError(Error, Function, Prefix); +} + +_Use_decl_annotations_ +VOID +LoggerGetRegistryKeyPath(HKEY Key, LPWSTR Path) +{ + DWORD LastError = GetLastError(); + if (Key == NULL) + { + wcsncpy_s(Path, MAX_REG_PATH, L"<null>", _TRUNCATE); + goto out; + } + if (_snwprintf_s(Path, MAX_REG_PATH, _TRUNCATE, L"0x%p", Key) == -1) + StrTruncate(Path, MAX_REG_PATH); + union + { + KEY_NAME_INFORMATION KeyNameInfo; + WCHAR Data[offsetof(KEY_NAME_INFORMATION, Name) + MAX_REG_PATH]; + } Buffer; + DWORD Size; + if (!NT_SUCCESS(NtQueryKey(Key, 3, &Buffer, sizeof(Buffer), &Size)) || + Size < offsetof(KEY_NAME_INFORMATION, Name) || Buffer.KeyNameInfo.NameLength >= MAX_REG_PATH * sizeof(WCHAR)) + goto out; + Buffer.KeyNameInfo.NameLength /= sizeof(WCHAR); + wmemcpy_s(Path, MAX_REG_PATH, Buffer.KeyNameInfo.Name, Buffer.KeyNameInfo.NameLength); + Path[Buffer.KeyNameInfo.NameLength] = L'\0'; +out: + SetLastError(LastError); +} + +static DWORD WINAPI +LogReaderThread(_In_ LPVOID Parameter) +{ + WIREGUARD_ADAPTER *Adapter = Parameter; + HANDLE ControlFile = AdapterOpenDeviceObject(Adapter); + if (ControlFile == INVALID_HANDLE_VALUE) + { + LOG_LAST_ERROR(L"Unable to enable logging"); + WriteULongNoFence(&Adapter->LogState, WIREGUARD_ADAPTER_LOG_OFF); + return 0; + } + while (ReadULongNoFence(&Adapter->LogState) != WIREGUARD_ADAPTER_LOG_OFF) + { + WCHAR WideLine[WG_MAX_LOG_LINE_LEN + 32] = { 0 }; + CHAR Line[WG_MAX_LOG_LINE_LEN] = { 0 }; + DWORD Bytes = sizeof(Line); + if (!DeviceIoControl(ControlFile, WG_IOCTL_READ_LOG_LINE, NULL, 0, Line, Bytes, &Bytes, NULL)) + { + BOOL IsAbort = GetLastError() == ERROR_OPERATION_ABORTED; + CloseHandle(ControlFile); + if (ReadULongNoFence(&Adapter->LogState) == WIREGUARD_ADAPTER_LOG_OFF) + return 0; + if (IsAbort) + Sleep(5000); + for (DWORD i = 0;; ++i) + { + ControlFile = AdapterOpenDeviceObject(Adapter); + if (ControlFile == INVALID_HANDLE_VALUE) + { + if (i < 10 && ReadULongNoFence(&Adapter->LogState) != WIREGUARD_ADAPTER_LOG_OFF) + { + Sleep(1000); + continue; + } + LOG_LAST_ERROR(L"Failed to reopen handle for logging after adapter disappeared"); + WriteULongNoFence(&Adapter->LogState, WIREGUARD_ADAPTER_LOG_OFF); + return 0; + } + else + break; + } + continue; + } + WIREGUARD_LOGGER_LEVEL Level; + if (Line[0] == '1') + Level = WIREGUARD_LOG_ERR; + else if (Line[0] == '2') + Level = WIREGUARD_LOG_WARN; + else if (Line[0] == '3') + Level = WIREGUARD_LOG_INFO; + else + continue; + DWORD Offset = 0; + if (ReadULongNoFence(&Adapter->LogState) == WIREGUARD_ADAPTER_LOG_ON_WITH_PREFIX) + { + if (!Adapter->IfIndex) + { + NET_LUID Luid; + WireGuardGetAdapterLUID(Adapter, &Luid); + ConvertInterfaceLuidToIndex(&Luid, &Adapter->IfIndex); + } + Offset = swprintf_s(WideLine, _countof(WideLine), L"%u: ", Adapter->IfIndex); + } + if (!MultiByteToWideChar(CP_UTF8, 0, &Line[1], -1, WideLine + Offset, _countof(WideLine) - Offset)) + continue; + Logger(Level, WideLine); + } + CloseHandle(ControlFile); + return 0; +} + +_Use_decl_annotations_ +BOOL WINAPI +WireGuardSetAdapterLogging(WIREGUARD_ADAPTER *Adapter, WIREGUARD_ADAPTER_LOG_STATE LogState) +{ + DWORD CurrentState = ReadULongNoFence(&Adapter->LogState); + if (CurrentState == (DWORD)LogState) + return TRUE; + WriteULongNoFence(&Adapter->LogState, LogState); + if (CurrentState != WIREGUARD_ADAPTER_LOG_OFF && LogState == WIREGUARD_ADAPTER_LOG_OFF && Adapter->LogThread) + { + CancelSynchronousIo(Adapter->LogThread); + BOOL Ret = WaitForSingleObject(Adapter->LogThread, INFINITE); + CloseHandle(Adapter->LogThread); + Adapter->LogThread = NULL; + return Ret; + } + if (CurrentState == WIREGUARD_ADAPTER_LOG_OFF && LogState != WIREGUARD_ADAPTER_LOG_OFF && !Adapter->LogThread) + { + Adapter->LogThread = CreateThread(NULL, 0, LogReaderThread, Adapter, 0, NULL); + return Adapter->LogThread != NULL; + } + return TRUE; +} diff --git a/api/logger.h b/api/logger.h new file mode 100644 index 0000000..a878b0f --- /dev/null +++ b/api/logger.h @@ -0,0 +1,191 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#pragma once + +#include "wireguard.h" +#include "main.h" +#include "registry.h" +#include <Windows.h> +#include <intsafe.h> +#include <stdarg.h> +#include <wchar.h> + +extern WIREGUARD_LOGGER_CALLBACK Logger; + +/** + * @copydoc WIREGUARD_SET_LOGGER_FUNC + */ +WIREGUARD_SET_LOGGER_FUNC WireGuardSetLogger; + +/** + * @copydoc WIREGUARD_SET_ADAPTER_LOGGING_FUNC + */ +WIREGUARD_SET_ADAPTER_LOGGING_FUNC WireGuardSetAdapterLogging; + +_Post_equals_last_error_ +DWORD +LoggerLog(_In_ WIREGUARD_LOGGER_LEVEL Level, _In_z_ LPCWSTR Function, _In_z_ LPCWSTR LogLine); + +_Post_equals_last_error_ +DWORD +LoggerLogV( + _In_ WIREGUARD_LOGGER_LEVEL Level, + _In_z_ LPCWSTR Function, + _In_z_ _Printf_format_string_ LPCWSTR Format, + _In_ va_list Args); + +_Post_equals_last_error_ +static inline DWORD +LoggerLogFmt( + _In_ WIREGUARD_LOGGER_LEVEL Level, + _In_z_ LPCWSTR Function, + _In_z_ _Printf_format_string_ LPCWSTR Format, + ...) +{ + va_list Args; + va_start(Args, Format); + DWORD LastError = LoggerLogV(Level, Function, Format, Args); + va_end(Args); + return LastError; +} + +_Post_equals_last_error_ +DWORD +LoggerError(_In_ DWORD Error, _In_z_ LPCWSTR Function, _In_z_ LPCWSTR Prefix); + +_Post_equals_last_error_ +DWORD +LoggerErrorV( + _In_ DWORD Error, + _In_z_ LPCWSTR Function, + _In_z_ _Printf_format_string_ LPCWSTR Format, + _In_ va_list Args); + +_Post_equals_last_error_ +static inline DWORD +LoggerErrorFmt(_In_ DWORD Error, _In_z_ LPCWSTR Function, _In_z_ _Printf_format_string_ LPCWSTR Format, ...) +{ + va_list Args; + va_start(Args, Format); + DWORD LastError = LoggerErrorV(Error, Function, Format, Args); + va_end(Args); + return LastError; +} + +_Post_equals_last_error_ +static inline DWORD +LoggerLastErrorV(_In_z_ LPCWSTR Function, _In_z_ _Printf_format_string_ LPCWSTR Format, _In_ va_list Args) +{ + DWORD LastError = GetLastError(); + LoggerErrorV(LastError, Function, Format, Args); + SetLastError(LastError); + return LastError; +} + +_Post_equals_last_error_ +static inline DWORD +LoggerLastErrorFmt(_In_z_ LPCWSTR Function, _In_z_ _Printf_format_string_ LPCWSTR Format, ...) +{ + va_list Args; + va_start(Args, Format); + DWORD LastError = LoggerLastErrorV(Function, Format, Args); + va_end(Args); + return LastError; +} + +VOID +LoggerGetRegistryKeyPath(_In_ HKEY Key, _Out_writes_z_(MAX_REG_PATH) LPWSTR Path); + +#define __L(x) L##x +#define _L(x) __L(x) +#define LOG(lvl, msg, ...) (LoggerLogFmt((lvl), _L(__FUNCTION__), msg, __VA_ARGS__)) +#define LOG_ERROR(err, msg, ...) (LoggerErrorFmt((err), _L(__FUNCTION__), msg, __VA_ARGS__)) +#define LOG_LAST_ERROR(msg, ...) (LoggerLastErrorFmt(_L(__FUNCTION__), msg, __VA_ARGS__)) + +#define RET_ERROR(Ret, Error) ((Error) == ERROR_SUCCESS ? (Ret) : (SetLastError(Error), 0)) + +_Must_inspect_result_ +DECLSPEC_ALLOCATOR +static inline _Return_type_success_(return != NULL) +_Post_maybenull_ +_Post_writable_byte_size_(Size) +VOID * +LoggerAlloc(_In_z_ LPCWSTR Function, _In_ DWORD Flags, _In_ SIZE_T Size) +{ + VOID *Data = HeapAlloc(ModuleHeap, Flags, Size); + if (!Data) + { + LoggerLogFmt(WIREGUARD_LOG_ERR, Function, L"Out of memory (flags: 0x%x, requested size: 0x%zx)", Flags, Size); + SetLastError(ERROR_OUTOFMEMORY); + } + return Data; +} +_Must_inspect_result_ +DECLSPEC_ALLOCATOR +static inline _Return_type_success_(return != NULL) +_Post_maybenull_ +_Post_writable_byte_size_(Size) +VOID * +LoggerReAlloc(_In_z_ LPCWSTR Function, _In_ DWORD Flags, _Frees_ptr_opt_ LPVOID Mem, _In_ SIZE_T Size) +{ + VOID *Data = Mem ? HeapReAlloc(ModuleHeap, Flags, Mem, Size) : HeapAlloc(ModuleHeap, Flags, Size); + if (!Data) + { + LoggerLogFmt(WIREGUARD_LOG_ERR, Function, L"Out of memory (flags: 0x%x, requested size: 0x%zx)", Flags, Size); + SetLastError(ERROR_OUTOFMEMORY); + } + return Data; +} +#define Alloc(Size) LoggerAlloc(_L(__FUNCTION__), 0, Size) +#define ReAlloc(Mem, Size) LoggerReAlloc(_L(__FUNCTION__), 0, Mem, Size) +#define Zalloc(Size) LoggerAlloc(_L(__FUNCTION__), HEAP_ZERO_MEMORY, Size) +#define ReZalloc(Mem, Size) LoggerReAlloc(_L(__FUNCTION__), HEAP_ZERO_MEMORY, Mem, Size) + +_Must_inspect_result_ +DECLSPEC_ALLOCATOR +static inline _Return_type_success_(return != NULL) +_Post_maybenull_ +_Post_writable_byte_size_((NumberOfElements) * (SizeOfOneElement)) +VOID * +LoggerAllocArray(_In_z_ LPCWSTR Function, _In_ DWORD Flags, _In_ SIZE_T NumberOfElements, _In_ SIZE_T SizeOfOneElement) +{ + SIZE_T Size; + if (FAILED(SIZETMult(NumberOfElements, SizeOfOneElement, &Size))) + return NULL; + return LoggerAlloc(Function, Flags, Size); +} +_Must_inspect_result_ +DECLSPEC_ALLOCATOR +static inline _Return_type_success_(return != NULL) +_Post_maybenull_ +_Post_writable_byte_size_((NumberOfElements) * (SizeOfOneElement)) +VOID * +LoggerReAllocArray( + _In_z_ LPCWSTR Function, + _In_ DWORD Flags, + _Frees_ptr_opt_ LPVOID Mem, + _In_ SIZE_T NumberOfElements, + _In_ SIZE_T SizeOfOneElement) +{ + SIZE_T Size; + if (FAILED(SIZETMult(NumberOfElements, SizeOfOneElement, &Size))) + return NULL; + return LoggerReAlloc(Function, Flags, Mem, Size); +} +#define AllocArray(Count, Size) LoggerAllocArray(_L(__FUNCTION__), 0, Count, Size) +#define ReAllocArray(Mem, Count, Size) LoggerReAllocArray(_L(__FUNCTION__), 0, Mem, Count, Size) +#define ZallocArray(Count, Size) LoggerAllocArray(_L(__FUNCTION__), HEAP_ZERO_MEMORY, Count, Size) +#define ReZallocArray(Mem, Count, Size) LoggerReAllocArray(_L(__FUNCTION__), HEAP_ZERO_MEMORY, Mem, Count, Size) + +static inline VOID +Free(_Frees_ptr_opt_ VOID *Ptr) +{ + if (!Ptr) + return; + DWORD LastError = GetLastError(); + HeapFree(ModuleHeap, 0, Ptr); + SetLastError(LastError); +}
\ No newline at end of file diff --git a/api/main.c b/api/main.c new file mode 100644 index 0000000..7e4e2c9 --- /dev/null +++ b/api/main.c @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#include "logger.h" +#include "adapter.h" +#include "main.h" +#include "namespace.h" +#include "registry.h" +#include "ntdll.h" + +#include <Windows.h> +#include <delayimp.h> +#include <sddl.h> +#include <winefs.h> +#include <stdlib.h> + +HINSTANCE ResourceModule; +HANDLE ModuleHeap; +SECURITY_ATTRIBUTES SecurityAttributes = { .nLength = sizeof(SECURITY_ATTRIBUTES) }; +BOOL IsLocalSystem; +USHORT NativeMachine = IMAGE_FILE_PROCESS; +BOOL IsWindows10; + +static FARPROC WINAPI +DelayedLoadLibraryHook(unsigned dliNotify, PDelayLoadInfo pdli) +{ + if (dliNotify != dliNotePreLoadLibrary) + return NULL; + HMODULE Library = LoadLibraryExA(pdli->szDll, NULL, LOAD_LIBRARY_SEARCH_SYSTEM32); + if (!Library) + abort(); + return (FARPROC)Library; +} + +const PfnDliHook __pfnDliNotifyHook2 = DelayedLoadLibraryHook; + +static BOOL InitializeSecurityObjects(VOID) +{ + BYTE LocalSystemSid[MAX_SID_SIZE]; + DWORD RequiredBytes = sizeof(LocalSystemSid); + HANDLE CurrentProcessToken; + struct + { + TOKEN_USER MaybeLocalSystem; + CHAR LargeEnoughForLocalSystem[MAX_SID_SIZE]; + } TokenUserBuffer; + BOOL Ret = FALSE; + + if (!CreateWellKnownSid(WinLocalSystemSid, NULL, LocalSystemSid, &RequiredBytes)) + return FALSE; + + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &CurrentProcessToken)) + return FALSE; + + if (!GetTokenInformation(CurrentProcessToken, TokenUser, &TokenUserBuffer, sizeof(TokenUserBuffer), &RequiredBytes)) + goto cleanupProcessToken; + + IsLocalSystem = EqualSid(TokenUserBuffer.MaybeLocalSystem.User.Sid, LocalSystemSid); + Ret = ConvertStringSecurityDescriptorToSecurityDescriptorW( + IsLocalSystem ? L"O:SYD:P(A;;GA;;;SY)(A;;GA;;;BA)S:(ML;;NWNRNX;;;HI)" + : L"O:BAD:P(A;;GA;;;SY)(A;;GA;;;BA)S:(ML;;NWNRNX;;;HI)", + SDDL_REVISION_1, + &SecurityAttributes.lpSecurityDescriptor, + NULL); + +cleanupProcessToken: + CloseHandle(CurrentProcessToken); + return Ret; +} + +static void EnvInit(VOID) +{ + DWORD MajorVersion; + RtlGetNtVersionNumbers(&MajorVersion, NULL, NULL); + IsWindows10 = MajorVersion >= 10; + +#ifdef MAYBE_WOW64 + typedef BOOL(WINAPI * IsWow64Process2_t)( + _In_ HANDLE hProcess, _Out_ USHORT * pProcessMachine, _Out_opt_ USHORT * pNativeMachine); + HANDLE Kernel32; + IsWow64Process2_t IsWow64Process2; + USHORT ProcessMachine; + if ((Kernel32 = GetModuleHandleW(L"kernel32.dll")) == NULL || + (IsWow64Process2 = (IsWow64Process2_t)GetProcAddress(Kernel32, "IsWow64Process2")) == NULL || + !IsWow64Process2(GetCurrentProcess(), &ProcessMachine, &NativeMachine)) + { + BOOL IsWoW64; + NativeMachine = + IsWow64Process(GetCurrentProcess(), &IsWoW64) && IsWoW64 ? IMAGE_FILE_MACHINE_AMD64 : IMAGE_FILE_PROCESS; + } +#endif +} + +BOOL APIENTRY +DllMain(_In_ HINSTANCE hinstDLL, _In_ DWORD fdwReason, _In_ LPVOID lpvReserved) +{ + switch (fdwReason) + { + case DLL_PROCESS_ATTACH: + ResourceModule = hinstDLL; + ModuleHeap = HeapCreate(0, 0, 0); + if (!ModuleHeap) + return FALSE; + if (!InitializeSecurityObjects()) + { + HeapDestroy(ModuleHeap); + return FALSE; + } + EnvInit(); + NamespaceInit(); + break; + + case DLL_PROCESS_DETACH: + NamespaceDone(); + LocalFree(SecurityAttributes.lpSecurityDescriptor); + HeapDestroy(ModuleHeap); + break; + } + return TRUE; +} diff --git a/api/main.h b/api/main.h new file mode 100644 index 0000000..5d3ebb1 --- /dev/null +++ b/api/main.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#pragma once + +#include <Windows.h> + +#if defined(_M_IX86) +# define IMAGE_FILE_PROCESS IMAGE_FILE_MACHINE_I386 +#elif defined(_M_AMD64) +# define IMAGE_FILE_PROCESS IMAGE_FILE_MACHINE_AMD64 +#elif defined(_M_ARM) +# define IMAGE_FILE_PROCESS IMAGE_FILE_MACHINE_ARMNT +#elif defined(_M_ARM64) +# define IMAGE_FILE_PROCESS IMAGE_FILE_MACHINE_ARM64 +#else +# error Unsupported architecture +#endif + +extern HINSTANCE ResourceModule; +extern HANDLE ModuleHeap; +extern SECURITY_ATTRIBUTES SecurityAttributes; +extern BOOL IsLocalSystem; +extern USHORT NativeMachine; +extern BOOL IsWindows10;
\ No newline at end of file diff --git a/api/namespace.c b/api/namespace.c new file mode 100644 index 0000000..83f94fe --- /dev/null +++ b/api/namespace.c @@ -0,0 +1,242 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#include "logger.h" +#include "main.h" +#include "namespace.h" + +#include <Windows.h> +#include <winternl.h> +#include <bcrypt.h> +#include <winefs.h> +#include <wchar.h> +#include <stdlib.h> + +static HANDLE PrivateNamespace = NULL; +static HANDLE BoundaryDescriptor = NULL; +static CRITICAL_SECTION Initializing; +static BCRYPT_ALG_HANDLE AlgProvider; + +_Must_inspect_result_ +static _Return_type_success_(return != NULL) +_Post_maybenull_ +LPWSTR +NormalizeStringAlloc(_In_ NORM_FORM NormForm, _In_z_ LPCWSTR Source) +{ + int Len = NormalizeString(NormForm, Source, -1, NULL, 0); + for (;;) + { + LPWSTR Str = AllocArray(Len, sizeof(*Str)); + if (!Str) + return NULL; + Len = NormalizeString(NormForm, Source, -1, Str, Len); + if (Len > 0) + return Str; + Free(Str); + if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) + { + LOG_LAST_ERROR(L"Failed: %s", Source); + return NULL; + } + Len = -Len; + } +} + +static _Return_type_success_(return != FALSE) +BOOL NamespaceRuntimeInit(VOID) +{ + DWORD LastError; + + EnterCriticalSection(&Initializing); + if (PrivateNamespace) + { + LeaveCriticalSection(&Initializing); + return TRUE; + } + + NTSTATUS Status; + if (!BCRYPT_SUCCESS(Status = BCryptOpenAlgorithmProvider(&AlgProvider, BCRYPT_SHA256_ALGORITHM, NULL, 0))) + { + LOG(WIREGUARD_LOG_ERR, L"Failed to open algorithm provider (status: 0x%x)", Status); + LastError = RtlNtStatusToDosError(Status); + goto cleanupLeaveCriticalSection; + } + + BYTE Sid[MAX_SID_SIZE]; + DWORD SidSize = sizeof(Sid); + if (!CreateWellKnownSid(IsLocalSystem ? WinLocalSystemSid : WinBuiltinAdministratorsSid, NULL, Sid, &SidSize)) + { + LastError = LOG_LAST_ERROR(L"Failed to create SID"); + goto cleanupBCryptCloseAlgorithmProvider; + } + + BoundaryDescriptor = CreateBoundaryDescriptorW(L"WireGuard", 0); + if (!BoundaryDescriptor) + { + LastError = LOG_LAST_ERROR(L"Failed to create boundary descriptor"); + goto cleanupBCryptCloseAlgorithmProvider; + } + if (!AddSIDToBoundaryDescriptor(&BoundaryDescriptor, Sid)) + { + LastError = LOG_LAST_ERROR(L"Failed to add SID to boundary descriptor"); + goto cleanupBoundaryDescriptor; + } + + for (;;) + { + if ((PrivateNamespace = CreatePrivateNamespaceW(&SecurityAttributes, BoundaryDescriptor, L"WireGuard")) != NULL) + break; + if ((LastError = GetLastError()) == ERROR_ALREADY_EXISTS) + { + if ((PrivateNamespace = OpenPrivateNamespaceW(BoundaryDescriptor, L"WireGuard")) != NULL) + break; + if ((LastError = GetLastError()) == ERROR_PATH_NOT_FOUND) + continue; + LOG_ERROR(LastError, L"Failed to open private namespace"); + } + else + LOG_ERROR(LastError, L"Failed to create private namespace"); + goto cleanupBoundaryDescriptor; + } + + LeaveCriticalSection(&Initializing); + return TRUE; + +cleanupBoundaryDescriptor: + DeleteBoundaryDescriptor(BoundaryDescriptor); +cleanupBCryptCloseAlgorithmProvider: + BCryptCloseAlgorithmProvider(AlgProvider, 0); +cleanupLeaveCriticalSection: + LeaveCriticalSection(&Initializing); + SetLastError(LastError); + return FALSE; +} + +_Use_decl_annotations_ +HANDLE +NamespaceTakePoolMutex(LPCWSTR Pool) +{ + if (!NamespaceRuntimeInit()) + return NULL; + + BCRYPT_HASH_HANDLE Sha256 = NULL; + NTSTATUS Status; + if (!BCRYPT_SUCCESS(Status = BCryptCreateHash(AlgProvider, &Sha256, NULL, 0, NULL, 0, 0))) + { + LOG(WIREGUARD_LOG_ERR, L"Failed to create hash (status: 0x%x)", Status); + SetLastError(RtlNtStatusToDosError(Status)); + return NULL; + } + DWORD LastError; + static const WCHAR mutex_label[] = L"WireGuard Adapter Name Mutex Stable Suffix v1 jason@zx2c4.com"; + if (!BCRYPT_SUCCESS( + Status = BCryptHashData(Sha256, (PUCHAR)mutex_label, sizeof(mutex_label) /* Including NULL 2 bytes */, 0))) + { + LOG(WIREGUARD_LOG_ERR, L"Failed to hash data (status: 0x%x)", Status); + LastError = RtlNtStatusToDosError(Status); + goto cleanupSha256; + } + LPWSTR PoolNorm = NormalizeStringAlloc(NormalizationC, Pool); + if (!PoolNorm) + { + LastError = GetLastError(); + goto cleanupSha256; + } + if (!BCRYPT_SUCCESS( + Status = BCryptHashData(Sha256, (PUCHAR)PoolNorm, (int)wcslen(PoolNorm) + 2 /* Add in NULL 2 bytes */, 0))) + { + LOG(WIREGUARD_LOG_ERR, L"Failed to hash data (status: 0x%x)", Status); + LastError = RtlNtStatusToDosError(Status); + goto cleanupPoolNorm; + } + BYTE Hash[32]; + if (!BCRYPT_SUCCESS(Status = BCryptFinishHash(Sha256, Hash, sizeof(Hash), 0))) + { + LOG(WIREGUARD_LOG_ERR, L"Failed to calculate hash (status: 0x%x)", Status); + LastError = RtlNtStatusToDosError(Status); + goto cleanupPoolNorm; + } + static const WCHAR MutexNamePrefix[] = L"WireGuard\\WireGuard-Name-Mutex-"; + WCHAR MutexName[_countof(MutexNamePrefix) + sizeof(Hash) * 2]; + memcpy(MutexName, MutexNamePrefix, sizeof(MutexNamePrefix)); + for (size_t i = 0; i < sizeof(Hash); ++i) + swprintf_s(&MutexName[_countof(MutexNamePrefix) - 1 + i * 2], 3, L"%02x", Hash[i]); + HANDLE Mutex = CreateMutexW(&SecurityAttributes, FALSE, MutexName); + if (!Mutex) + { + LastError = LOG_LAST_ERROR(L"Failed to create mutex %s", MutexName); + goto cleanupPoolNorm; + } + DWORD Result = WaitForSingleObject(Mutex, INFINITE); + switch (Result) + { + case WAIT_OBJECT_0: + case WAIT_ABANDONED: + Free(PoolNorm); + BCryptDestroyHash(Sha256); + return Mutex; + } + LOG(WIREGUARD_LOG_ERR, L"Failed to get mutex %s (status: 0x%x)", MutexName, Result); + LastError = ERROR_GEN_FAILURE; + CloseHandle(Mutex); +cleanupPoolNorm: + Free(PoolNorm); +cleanupSha256: + BCryptDestroyHash(Sha256); + SetLastError(LastError); + return NULL; +} + +_Use_decl_annotations_ +HANDLE +NamespaceTakeDriverInstallationMutex(VOID) +{ + if (!NamespaceRuntimeInit()) + return NULL; + HANDLE Mutex = CreateMutexW(&SecurityAttributes, FALSE, L"WireGuard\\WireGuard-Driver-Installation-Mutex"); + if (!Mutex) + { + LOG_LAST_ERROR(L"Failed to create mutex"); + return NULL; + } + DWORD Result = WaitForSingleObject(Mutex, INFINITE); + switch (Result) + { + case WAIT_OBJECT_0: + case WAIT_ABANDONED: + return Mutex; + } + LOG(WIREGUARD_LOG_ERR, L"Failed to get mutex (status: 0x%x)", Result); + CloseHandle(Mutex); + SetLastError(ERROR_GEN_FAILURE); + return NULL; +} + +_Use_decl_annotations_ +VOID +NamespaceReleaseMutex(HANDLE Mutex) +{ + ReleaseMutex(Mutex); + CloseHandle(Mutex); +} + +VOID NamespaceInit(VOID) +{ + InitializeCriticalSection(&Initializing); +} + +VOID NamespaceDone(VOID) +{ + EnterCriticalSection(&Initializing); + if (PrivateNamespace) + { + BCryptCloseAlgorithmProvider(AlgProvider, 0); + ClosePrivateNamespace(PrivateNamespace, 0); + DeleteBoundaryDescriptor(BoundaryDescriptor); + PrivateNamespace = NULL; + } + LeaveCriticalSection(&Initializing); + DeleteCriticalSection(&Initializing); +} diff --git a/api/namespace.h b/api/namespace.h new file mode 100644 index 0000000..cbd9100 --- /dev/null +++ b/api/namespace.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#pragma once + +#include <Windows.h> + +_Must_inspect_result_ +_Return_type_success_(return != NULL) +_Post_maybenull_ +_Acquires_lock_(_Curr_) +HANDLE +NamespaceTakePoolMutex(_In_z_ LPCWSTR Pool); + +_Must_inspect_result_ +_Return_type_success_(return != NULL) +_Post_maybenull_ +_Acquires_lock_(_Curr_) +HANDLE +NamespaceTakeDriverInstallationMutex(VOID); + +_Releases_lock_(Mutex) +VOID +NamespaceReleaseMutex(_In_ HANDLE Mutex); + +VOID NamespaceInit(VOID); + +VOID NamespaceDone(VOID); diff --git a/api/nci.def b/api/nci.def new file mode 100644 index 0000000..db484b7 --- /dev/null +++ b/api/nci.def @@ -0,0 +1,4 @@ +LIBRARY nci.dll +EXPORTS + NciGetConnectionName + NciSetConnectionName diff --git a/api/nci.h b/api/nci.h new file mode 100644 index 0000000..ba99fa6 --- /dev/null +++ b/api/nci.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#pragma once + +#include <Windows.h> + +#ifdef GENERATE_LIB +# define DECLSPEC __declspec(dllexport) +# define STUB \ + { \ + return 0; \ + } +#else +# define DECLSPEC __declspec(dllimport) +# define STUB ; +#endif + +EXTERN_C +DECLSPEC DWORD WINAPI +NciSetConnectionName(_In_ const GUID *Guid, _In_z_ LPCWSTR NewName) STUB + + EXTERN_C +DECLSPEC DWORD WINAPI +NciGetConnectionName( + _In_ const GUID *Guid, + _Out_z_bytecap_(InDestNameBytes) LPWSTR Name, + _In_ DWORD InDestNameBytes, + _Out_opt_ DWORD *OutDestNameBytes) STUB
\ No newline at end of file diff --git a/api/ntdll.h b/api/ntdll.h new file mode 100644 index 0000000..3782a30 --- /dev/null +++ b/api/ntdll.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#pragma once + +#include <Windows.h> + +enum +{ + SystemModuleInformation = 11 +}; + +typedef struct _RTL_PROCESS_MODULE_INFORMATION +{ + HANDLE Section; + PVOID MappedBase; + PVOID ImageBase; + ULONG ImageSize; + ULONG Flags; + USHORT LoadOrderIndex; + USHORT InitOrderIndex; + USHORT LoadCount; + USHORT OffsetToFileName; + UCHAR FullPathName[256]; +} RTL_PROCESS_MODULE_INFORMATION, *PRTL_PROCESS_MODULE_INFORMATION; + +typedef struct _RTL_PROCESS_MODULES +{ + ULONG NumberOfModules; + RTL_PROCESS_MODULE_INFORMATION Modules[1]; +} RTL_PROCESS_MODULES, *PRTL_PROCESS_MODULES; + +typedef struct _KEY_NAME_INFORMATION +{ + ULONG NameLength; + WCHAR Name[1]; +} KEY_NAME_INFORMATION, *PKEY_NAME_INFORMATION; + +#define STATUS_INFO_LENGTH_MISMATCH ((NTSTATUS)0xC0000004L) // TODO: #include <ntstatus.h> instead of this +#define STATUS_PNP_DEVICE_CONFIGURATION_PENDING ((NTSTATUS)0xC0000495L) + +/* We can't use RtlGetVersion, because appcompat's aclayers.dll shims it to report Vista + * when run from legacy contexts. So, we instead use the undocumented RtlGetNtVersionNumbers. + * + * Another way would be reading from the PEB directly: + * ((DWORD *)NtCurrentTeb()->ProcessEnvironmentBlock)[sizeof(VOID *) == 8 ? 70 : 41] + * Or just read from KUSER_SHARED_DATA the same way on 32-bit and 64-bit: + * *(DWORD *)0x7FFE026C + */ +EXTERN_C +DECLSPEC_IMPORT VOID NTAPI +RtlGetNtVersionNumbers(_Out_opt_ DWORD *MajorVersion, _Out_opt_ DWORD *MinorVersion, _Out_opt_ DWORD *BuildNumber); + +EXTERN_C +DECLSPEC_IMPORT DWORD NTAPI +NtQueryKey( + _In_ HANDLE KeyHandle, + _In_ int KeyInformationClass, + _Out_bytecap_post_bytecount_(Length, *ResultLength) PVOID KeyInformation, + _In_ ULONG Length, + _Out_ PULONG ResultLength); diff --git a/api/registry.c b/api/registry.c new file mode 100644 index 0000000..05a62a6 --- /dev/null +++ b/api/registry.c @@ -0,0 +1,398 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#include "logger.h" +#include "registry.h" +#include <Windows.h> +#include <wchar.h> +#include <stdlib.h> +#include <strsafe.h> + +_Must_inspect_result_ +static _Return_type_success_(return != NULL) +_Post_maybenull_ +HKEY +OpenKeyWait(_In_ HKEY Key, _Inout_z_ LPWSTR Path, _In_ DWORD Access, _In_ ULONGLONG Deadline) +{ + DWORD LastError; + LPWSTR PathNext = wcschr(Path, L'\\'); + if (PathNext) + *PathNext = 0; + + HANDLE Event = CreateEventW(NULL, FALSE, FALSE, NULL); + if (!Event) + { + LOG_LAST_ERROR(L"Failed to create event"); + return NULL; + } + for (;;) + { + LastError = RegNotifyChangeKeyValue(Key, FALSE, REG_NOTIFY_CHANGE_NAME, Event, TRUE); + if (LastError != ERROR_SUCCESS) + { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(Key, RegPath); + LOG_ERROR(LastError, L"Failed to setup registry key %.*s notification", MAX_REG_PATH, RegPath); + break; + } + + HKEY Subkey; + LastError = RegOpenKeyExW(Key, Path, 0, PathNext ? KEY_NOTIFY : Access, &Subkey); + if (LastError == ERROR_SUCCESS) + { + if (PathNext) + { + HKEY KeyOut = OpenKeyWait(Subkey, PathNext + 1, Access, Deadline); + if (KeyOut) + { + RegCloseKey(Subkey); + CloseHandle(Event); + return KeyOut; + } + LastError = GetLastError(); + break; + } + else + { + CloseHandle(Event); + return Subkey; + } + } + if (LastError != ERROR_FILE_NOT_FOUND && LastError != ERROR_PATH_NOT_FOUND) + { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(Key, RegPath); + LOG_ERROR(LastError, L"Failed to open registry key %.*s\\%s", MAX_REG_PATH, RegPath, Path); + break; + } + + LONGLONG TimeLeft = Deadline - GetTickCount64(); + if (TimeLeft < 0) + TimeLeft = 0; + DWORD Result = WaitForSingleObject(Event, (DWORD)TimeLeft); + if (Result != WAIT_OBJECT_0) + { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(Key, RegPath); + LOG(WIREGUARD_LOG_ERR, + L"Timeout waiting for registry key %.*s\\%s (status: 0x%x)", + MAX_REG_PATH, + RegPath, + Path, + Result); + break; + } + } + CloseHandle(Event); + SetLastError(LastError); + return NULL; +} + +_Use_decl_annotations_ +HKEY +RegistryOpenKeyWait(HKEY Key, LPCWSTR Path, DWORD Access, DWORD Timeout) +{ + WCHAR Buf[MAX_REG_PATH]; + if (wcsncpy_s(Buf, _countof(Buf), Path, _TRUNCATE) == STRUNCATE) + { + LOG(WIREGUARD_LOG_ERR, L"Registry path too long: %s", Path); + SetLastError(ERROR_INVALID_PARAMETER); + return NULL; + } + return OpenKeyWait(Key, Buf, Access, GetTickCount64() + Timeout); +} + +_Use_decl_annotations_ +BOOL +RegistryGetString(LPWSTR *Buf, DWORD Len, DWORD ValueType) +{ + if (wcsnlen(*Buf, Len) >= Len) + { + /* String is missing zero-terminator. */ + LPWSTR BufZ = ReZallocArray(*Buf, (SIZE_T)Len + 1, sizeof(*BufZ)); + if (!BufZ) + return FALSE; + _Analysis_assume_((wmemset(BufZ, L'A', (SIZE_T)Len + 1), TRUE)); + *Buf = BufZ; + } + + if (ValueType != REG_EXPAND_SZ) + return TRUE; + + /* ExpandEnvironmentStringsW() returns strlen on success or 0 on error. Bail out on empty input strings to + * disambiguate. */ + if (!(*Buf)[0]) + return TRUE; + + for (;;) + { + LPWSTR Expanded = AllocArray(Len, sizeof(*Expanded)); + if (!Expanded) + return FALSE; + DWORD Result = ExpandEnvironmentStringsW(*Buf, Expanded, Len); + if (!Result) + { + LOG_LAST_ERROR(L"Failed to expand environment variables: %s", *Buf); + Free(Expanded); + return FALSE; + } + if (Result > Len) + { + Free(Expanded); + Len = Result; + continue; + } + Free(*Buf); + *Buf = Expanded; + return TRUE; + } +} + +_Use_decl_annotations_ +BOOL +RegistryGetMultiString(PZZWSTR *Buf, DWORD Len, DWORD ValueType) +{ + if (ValueType == REG_MULTI_SZ) + { + for (size_t i = 0;; i += wcsnlen(*Buf + i, Len - i) + 1) + { + if (i > Len) + { + /* Missing string and list terminators. */ + PZZWSTR BufZ = ReZallocArray(*Buf, (SIZE_T)Len + 2, sizeof(*BufZ)); + if (!BufZ) + return FALSE; + *Buf = BufZ; + return TRUE; + } + if (i == Len) + { + /* Missing list terminator. */ + PZZWSTR BufZ = ReZallocArray(*Buf, (SIZE_T)Len + 1, sizeof(*BufZ)); + if (!BufZ) + return FALSE; + *Buf = BufZ; + return TRUE; + } + if (!(*Buf)[i]) + return TRUE; + } + } + + /* Sanitize REG_SZ/REG_EXPAND_SZ and append a list terminator to make a multi-string. */ + if (!RegistryGetString(Buf, Len, ValueType)) + return FALSE; + Len = (DWORD)wcslen(*Buf) + 1; + PZZWSTR BufZ = ReZallocArray(*Buf, (SIZE_T)Len + 1, sizeof(WCHAR)); + if (!BufZ) + return FALSE; + *Buf = BufZ; + return TRUE; +} + +_Must_inspect_result_ +static _Return_type_success_(return != NULL) +_Post_maybenull_ +_Post_writable_byte_size_(*BufLen) +VOID * +RegistryQuery(_In_ HKEY Key, _In_opt_z_ LPCWSTR Name, _Out_opt_ DWORD *ValueType, _Inout_ DWORD *BufLen, _In_ BOOL Log) +{ + for (;;) + { + BYTE *p = Alloc(*BufLen); + if (!p) + return NULL; + LSTATUS LastError = RegQueryValueExW(Key, Name, NULL, ValueType, p, BufLen); + if (LastError == ERROR_SUCCESS) + return p; + Free(p); + if (LastError != ERROR_MORE_DATA) + { + if (Log) + { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(Key, RegPath); + LOG_ERROR(LastError, L"Failed to query registry value %.*s\\%s", MAX_REG_PATH, RegPath, Name); + } + SetLastError(LastError); + return NULL; + } + } +} + +_Use_decl_annotations_ +LPWSTR +RegistryQueryString(HKEY Key, LPCWSTR Name, BOOL Log) +{ + DWORD LastError, ValueType, Size = 256 * sizeof(WCHAR); + LPWSTR Value = RegistryQuery(Key, Name, &ValueType, &Size, Log); + if (!Value) + return NULL; + switch (ValueType) + { + case REG_SZ: + case REG_EXPAND_SZ: + case REG_MULTI_SZ: + if (RegistryGetString(&Value, Size / sizeof(*Value), ValueType)) + return Value; + LastError = GetLastError(); + break; + default: { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(Key, RegPath); + LOG(WIREGUARD_LOG_ERR, + L"Registry value %.*s\\%s is not a string (type: %u)", + MAX_REG_PATH, + RegPath, + Name, + ValueType); + LastError = ERROR_INVALID_DATATYPE; + } + } + Free(Value); + SetLastError(LastError); + return NULL; +} + +_Use_decl_annotations_ +LPWSTR +RegistryQueryStringWait(HKEY Key, LPCWSTR Name, DWORD Timeout) +{ + DWORD LastError; + ULONGLONG Deadline = GetTickCount64() + Timeout; + HANDLE Event = CreateEventW(NULL, FALSE, FALSE, NULL); + if (!Event) + { + LOG_LAST_ERROR(L"Failed to create event"); + return NULL; + } + for (;;) + { + LastError = RegNotifyChangeKeyValue(Key, FALSE, REG_NOTIFY_CHANGE_LAST_SET, Event, TRUE); + if (LastError != ERROR_SUCCESS) + { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(Key, RegPath); + LOG_ERROR(LastError, L"Failed to setup registry key %.*s notification", MAX_REG_PATH, RegPath); + break; + } + LPWSTR Value = RegistryQueryString(Key, Name, FALSE); + if (Value) + { + CloseHandle(Event); + return Value; + } + LastError = GetLastError(); + if (LastError != ERROR_FILE_NOT_FOUND && LastError != ERROR_PATH_NOT_FOUND) + break; + LONGLONG TimeLeft = Deadline - GetTickCount64(); + if (TimeLeft < 0) + TimeLeft = 0; + DWORD Result = WaitForSingleObject(Event, (DWORD)TimeLeft); + if (Result != WAIT_OBJECT_0) + { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(Key, RegPath); + LOG(WIREGUARD_LOG_ERR, + L"Timeout waiting for registry value %.*s\\%s (status: 0x%x)", + MAX_REG_PATH, + RegPath, + Name, + Result); + break; + } + } + CloseHandle(Event); + SetLastError(LastError); + return NULL; +} + +_Use_decl_annotations_ +BOOL +RegistryQueryDWORD(HKEY Key, LPCWSTR Name, DWORD *Value, BOOL Log) +{ + DWORD ValueType, Size = sizeof(DWORD); + DWORD LastError = RegQueryValueExW(Key, Name, NULL, &ValueType, (BYTE *)Value, &Size); + if (LastError != ERROR_SUCCESS) + { + if (Log) + { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(Key, RegPath); + LOG_ERROR(LastError, L"Failed to query registry value %.*s\\%s", MAX_REG_PATH, RegPath, Name); + } + SetLastError(LastError); + return FALSE; + } + if (ValueType != REG_DWORD) + { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(Key, RegPath); + LOG(WIREGUARD_LOG_ERR, L"Value %.*s\\%s is not a DWORD (type: %u)", MAX_REG_PATH, RegPath, Name, ValueType); + SetLastError(ERROR_INVALID_DATATYPE); + return FALSE; + } + if (Size != sizeof(DWORD)) + { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(Key, RegPath); + LOG(WIREGUARD_LOG_ERR, L"Value %.*s\\%s size is not 4 bytes (size: %u)", MAX_REG_PATH, RegPath, Name, Size); + SetLastError(ERROR_INVALID_DATA); + return FALSE; + } + return TRUE; +} + +_Use_decl_annotations_ +BOOL +RegistryQueryDWORDWait(HKEY Key, LPCWSTR Name, DWORD Timeout, DWORD *Value) +{ + DWORD LastError; + ULONGLONG Deadline = GetTickCount64() + Timeout; + HANDLE Event = CreateEventW(NULL, FALSE, FALSE, NULL); + if (!Event) + { + LOG_LAST_ERROR(L"Failed to create event"); + return FALSE; + } + for (;;) + { + LastError = RegNotifyChangeKeyValue(Key, FALSE, REG_NOTIFY_CHANGE_LAST_SET, Event, TRUE); + if (LastError != ERROR_SUCCESS) + { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(Key, RegPath); + LOG_ERROR(LastError, L"Failed to setup registry key %.*s notification", MAX_REG_PATH, RegPath); + break; + } + if (RegistryQueryDWORD(Key, Name, Value, FALSE)) + { + CloseHandle(Event); + return TRUE; + } + LastError = GetLastError(); + if (LastError != ERROR_FILE_NOT_FOUND && LastError != ERROR_PATH_NOT_FOUND) + break; + LONGLONG TimeLeft = Deadline - GetTickCount64(); + if (TimeLeft < 0) + TimeLeft = 0; + DWORD Result = WaitForSingleObject(Event, (DWORD)TimeLeft); + if (Result != WAIT_OBJECT_0) + { + WCHAR RegPath[MAX_REG_PATH]; + LoggerGetRegistryKeyPath(Key, RegPath); + LOG(WIREGUARD_LOG_ERR, + L"Timeout waiting registry value %.*s\\%s (status: 0x%x)", + MAX_REG_PATH, + RegPath, + Name, + Result); + break; + } + } + CloseHandle(Event); + SetLastError(LastError); + return FALSE; +} diff --git a/api/registry.h b/api/registry.h new file mode 100644 index 0000000..40fc7c7 --- /dev/null +++ b/api/registry.h @@ -0,0 +1,160 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#pragma once + +#include "wireguard.h" +#include <Windows.h> + +#define MAX_REG_PATH \ + 256 /* Maximum registry path length \ + https://support.microsoft.com/en-us/help/256986/windows-registry-information-for-advanced-users */ + +/** + * Opens the specified registry key. It waits for the registry key to become available. + * + * @param Key Handle of the parent registry key. Must be opened with notify access. + * + * @param Path Subpath of the registry key to open. Zero-terminated string of up to MAX_REG_PATH-1 characters. + * + * @param Access A mask that specifies the desired access rights to the key to be opened. + * + * @param Timeout Timeout to wait for the value in milliseconds. + * + * @return Key handle on success. If the function fails, the return value is zero. To get extended error information, + * call GetLastError. + */ +_Must_inspect_result_ +_Return_type_success_(return != NULL) +_Post_maybenull_ +HKEY +RegistryOpenKeyWait(_In_ HKEY Key, _In_z_ LPCWSTR Path, _In_ DWORD Access, _In_ DWORD Timeout); + +/** + * Validates and/or sanitizes string value read from registry. + * + * @param Buf On input, it contains a pointer to pointer where the data is stored. The data must be allocated + * using HeapAlloc(ModuleHeap, 0). On output, it contains a pointer to pointer where the sanitized + * data is stored. It must be released with HeapFree(ModuleHeap, 0, *Buf) after use. + * + * @param Len Length of data string in wide characters. + * + * @param ValueType Type of data. Must be either REG_SZ or REG_EXPAND_SZ. REG_MULTI_SZ is treated like REG_SZ; only + * the first string of a multi-string is to be used. + * + * @return If the function succeeds, the return value is nonzero. If the function fails, the return value is zero. To + * get extended error information, call GetLastError. + */ +_Must_inspect_result_ +_Return_type_success_(return != FALSE) +BOOL +RegistryGetString(_Inout_ LPWSTR *Buf, _In_ DWORD Len, _In_ DWORD ValueType); + +/** + * Validates and/or sanitizes multi-string value read from registry. + * + * @param Buf On input, it contains a pointer to pointer where the data is stored. The data must be allocated + * using HeapAlloc(ModuleHeap, 0). On output, it contains a pointer to pointer where the sanitized + * data is stored. It must be released with HeapFree(ModuleHeap, 0, *Buf) after use. + * + * @param Len Length of data string in wide characters. + * + * @param ValueType Type of data. Must be one of REG_MULTI_SZ, REG_SZ or REG_EXPAND_SZ. + * + * @return If the function succeeds, the return value is nonzero. If the function fails, the return value is zero. To + * get extended error information, call GetLastError. + */ +_Must_inspect_result_ +_Return_type_success_(return != FALSE) +BOOL +RegistryGetMultiString(_Inout_ PZZWSTR *Buf, _In_ DWORD Len, _In_ DWORD ValueType); + +/** + * Reads string value from registry key. + * + * @param Key Handle of the registry key to read from. Must be opened with read access. + * + * @param Name Name of the value to read. + * + * @param Value Pointer to string to retrieve registry value. If the value type is REG_EXPAND_SZ the value is + * expanded using ExpandEnvironmentStrings(). If the value type is REG_MULTI_SZ, only the first + * string from the multi-string is returned. The string must be released with + * HeapFree(ModuleHeap, 0, Value) after use. + * + * @Log Set to TRUE to log all failures; FALSE to skip logging the innermost errors. Skipping innermost + * errors reduces log clutter when we are using RegistryQueryString() from + * RegistryQueryStringWait() and some errors are expected to occur. + * + * @return String with registry value on success; If the function fails, the return value is zero. To get extended error + * information, call GetLastError. + */ +_Must_inspect_result_ +_Return_type_success_(return != NULL) +_Post_maybenull_ +LPWSTR +RegistryQueryString(_In_ HKEY Key, _In_opt_z_ LPCWSTR Name, _In_ BOOL Log); + +/** + * Reads string value from registry key. It waits for the registry value to become available. + * + * @param Key Handle of the registry key to read from. Must be opened with read and notify access. + * + * @param Name Name of the value to read. + * + * @param Timeout Timeout to wait for the value in milliseconds. + * + * @return Registry value. If the value type is REG_EXPAND_SZ the value is expanded using ExpandEnvironmentStrings(). If + * the value type is REG_MULTI_SZ, only the first string from the multi-string is returned. The string must be + * released with HeapFree(ModuleHeap, 0, Value) after use. If the function fails, the return value is zero. To + * get extended error information, call GetLastError. Possible errors include the following: + * ERROR_INVALID_DATATYPE when the registry value is not a string + */ +_Must_inspect_result_ +_Return_type_success_(return != NULL) +_Post_maybenull_ +LPWSTR +RegistryQueryStringWait(_In_ HKEY Key, _In_opt_z_ LPCWSTR Name, _In_ DWORD Timeout); + +/** + * Reads a 32-bit DWORD value from registry key. + * + * @param Key Handle of the registry key to read from. Must be opened with read access. + * + * @param Name Name of the value to read. + * + * @param Value Pointer to DWORD to retrieve registry value. + * + * @Log Set to TRUE to log all failures; FALSE to skip logging the innermost errors. Skipping innermost + * errors reduces log clutter when we are using RegistryQueryDWORD() from + * RegistryQueryDWORDWait() and some errors are expected to occur. + * + * @return If the function succeeds, the return value is nonzero. If the function fails, the return value is zero. To + * get extended error information, call GetLastError. + */ +_Must_inspect_result_ +_Return_type_success_(return != FALSE) +BOOL +RegistryQueryDWORD(_In_ HKEY Key, _In_opt_z_ LPCWSTR Name, _Out_ DWORD *Value, _In_ BOOL Log); + +/** + * Reads a 32-bit DWORD value from registry key. It waits for the registry value to become available. + * + * @param Key Handle of the registry key to read from. Must be opened with read access. + * + * @param Name Name of the value to read. + * + * @param Timeout Timeout to wait for the value in milliseconds. + * + * @param Value Pointer to DWORD to retrieve registry value. + * + * @return If the function succeeds, the return value is nonzero. If the function fails, the return value is zero. To + * get extended error information, call GetLastError. Possible errors include the following: + * ERROR_INVALID_DATATYPE when registry value exist but not REG_DWORD type; + * ERROR_INVALID_DATA when registry value size is not 4 bytes + */ +_Must_inspect_result_ +_Return_type_success_(return != FALSE) +BOOL +RegistryQueryDWORDWait(_In_ HKEY Key, _In_opt_z_ LPCWSTR Name, _In_ DWORD Timeout, _Out_ DWORD *Value); diff --git a/api/resource.c b/api/resource.c new file mode 100644 index 0000000..6dd93a3 --- /dev/null +++ b/api/resource.c @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#include "logger.h" +#include "main.h" +#include "resource.h" +#include <Windows.h> +#include <Shlwapi.h> +#include <NTSecAPI.h> + +_Use_decl_annotations_ +const VOID * +ResourceGetAddress(LPCWSTR ResourceName, DWORD *Size) +{ + HRSRC FoundResource = FindResourceW(ResourceModule, ResourceName, RT_RCDATA); + if (!FoundResource) + { + LOG_LAST_ERROR(L"Failed to find resource %s", ResourceName); + return NULL; + } + *Size = SizeofResource(ResourceModule, FoundResource); + if (!*Size) + { + LOG_LAST_ERROR(L"Failed to query resource %s size", ResourceName); + return NULL; + } + HGLOBAL LoadedResource = LoadResource(ResourceModule, FoundResource); + if (!LoadedResource) + { + LOG_LAST_ERROR(L"Failed to load resource %s", ResourceName); + return NULL; + } + BYTE *Address = LockResource(LoadedResource); + if (!Address) + { + LOG(WIREGUARD_LOG_ERR, L"Failed to lock resource %s", ResourceName); + SetLastError(ERROR_LOCK_FAILED); + return NULL; + } + return Address; +} + +_Use_decl_annotations_ +BOOL +ResourceCopyToFile(LPCWSTR DestinationPath, LPCWSTR ResourceName) +{ + DWORD SizeResource; + const VOID *LockedResource = ResourceGetAddress(ResourceName, &SizeResource); + if (!LockedResource) + { + LOG(WIREGUARD_LOG_ERR, L"Failed to locate resource %s", ResourceName); + return FALSE; + } + HANDLE DestinationHandle = CreateFileW( + DestinationPath, + GENERIC_WRITE, + 0, + &SecurityAttributes, + CREATE_NEW, + FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_TEMPORARY, + NULL); + if (DestinationHandle == INVALID_HANDLE_VALUE) + { + LOG_LAST_ERROR(L"Failed to create file %s", DestinationPath); + return FALSE; + } + DWORD BytesWritten; + DWORD LastError; + if (!WriteFile(DestinationHandle, LockedResource, SizeResource, &BytesWritten, NULL)) + { + LastError = LOG_LAST_ERROR(L"Failed to write file %s", DestinationPath); + goto cleanupDestinationHandle; + } + if (BytesWritten != SizeResource) + { + LOG(WIREGUARD_LOG_ERR, + L"Incomplete write to %s (written: %u, expected: %u)", + DestinationPath, + BytesWritten, + SizeResource); + LastError = ERROR_WRITE_FAULT; + goto cleanupDestinationHandle; + } + LastError = ERROR_SUCCESS; +cleanupDestinationHandle: + CloseHandle(DestinationHandle); + return RET_ERROR(TRUE, LastError); +} + +_Return_type_success_(return != FALSE) +BOOL +ResourceCreateTemporaryDirectory(_Out_writes_z_(MAX_PATH) LPWSTR RandomTempSubDirectory) +{ + WCHAR WindowsDirectory[MAX_PATH]; + if (!GetWindowsDirectoryW(WindowsDirectory, _countof(WindowsDirectory))) + { + LOG_LAST_ERROR(L"Failed to get Windows folder"); + return FALSE; + } + WCHAR WindowsTempDirectory[MAX_PATH]; + if (!PathCombineW(WindowsTempDirectory, WindowsDirectory, L"Temp")) + { + SetLastError(ERROR_BUFFER_OVERFLOW); + return FALSE; + } + UCHAR RandomBytes[32] = { 0 }; + if (!RtlGenRandom(RandomBytes, sizeof(RandomBytes))) + { + LOG(WIREGUARD_LOG_ERR, L"Failed to generate random"); + SetLastError(ERROR_GEN_FAILURE); + return FALSE; + } + WCHAR RandomSubDirectory[sizeof(RandomBytes) * 2 + 1]; + for (int i = 0; i < sizeof(RandomBytes); ++i) + swprintf_s(&RandomSubDirectory[i * 2], 3, L"%02x", RandomBytes[i]); + if (!PathCombineW(RandomTempSubDirectory, WindowsTempDirectory, RandomSubDirectory)) + { + SetLastError(ERROR_BUFFER_OVERFLOW); + return FALSE; + } + if (!CreateDirectoryW(RandomTempSubDirectory, &SecurityAttributes)) + { + LOG_LAST_ERROR(L"Failed to create temporary folder %s", RandomTempSubDirectory); + return FALSE; + } + return TRUE; +} diff --git a/api/resource.h b/api/resource.h new file mode 100644 index 0000000..7cf3240 --- /dev/null +++ b/api/resource.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#pragma once + +#include "wireguard.h" +#include <Windows.h> + +/** + * Locates RT_RCDATA resource memory address and size. + * + * @param ResourceName Name of the RT_RCDATA resource. Use MAKEINTRESOURCEW to locate resource by ID. + * + * @param Size Pointer to a variable to receive resource size. + * + * @return Resource address on success. If the function fails, the return value is NULL. To get extended error + * information, call GetLastError. + */ +_Must_inspect_result_ +_Return_type_success_(return != NULL) +_Post_maybenull_ +_Post_readable_byte_size_(*Size) const VOID *ResourceGetAddress(_In_z_ LPCWSTR ResourceName, _Out_ DWORD *Size); + +/** + * Copies resource to a file. + * + * @param DestinationPath File path + * + * @param ResourceName Name of the RT_RCDATA resource. Use MAKEINTRESOURCEW to locate resource by ID. + * + * @return If the function succeeds, the return value is nonzero. If the function fails, the return value is zero. To + * get extended error information, call GetLastError. + */ +_Return_type_success_(return != FALSE) +BOOL +ResourceCopyToFile(_In_z_ LPCWSTR DestinationPath, _In_z_ LPCWSTR ResourceName); + +/** + * Creates a temporary directory. + * + * @param RandomTempSubDirectory Name of random temporary directory. + * + * @return If the function succeeds, the return value is nonzero. If the function fails, the return value is zero. To + * get extended error information, call GetLastError. + */ +_Return_type_success_(return != FALSE) +BOOL +ResourceCreateTemporaryDirectory(_Out_writes_z_(MAX_PATH) LPWSTR RandomTempSubDirectory); diff --git a/api/resources.rc b/api/resources.rc new file mode 100644 index 0000000..44b6e0c --- /dev/null +++ b/api/resources.rc @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#include <windows.h> +#include <ntverp.h> + +#pragma code_page(1252) + +wireguard.cat RCDATA "driver\\wireguard.cat" +wireguard.inf RCDATA "driver\\wireguard.inf" +wireguard.sys RCDATA "driver\\wireguard.sys" + +#ifdef HAVE_WHQL +wireguard-whql.cat RCDATA "whql\\wireguard.cat" +wireguard-whql.inf RCDATA "whql\\wireguard.inf" +wireguard-whql.sys RCDATA "whql\\wireguard.sys" +#endif + +#if defined(WANT_AMD64_WOW64) +# if defined(BUILT_AMD64_WOW64) +wireguard-amd64.dll RCDATA "amd64\\wireguard.dll" +# else +# pragma message("AMD64 wireguard.dll was not built, so this will not work from WOW64") +# endif +#endif +#if defined(WANT_ARM64_WOW64) +# if defined(BUILT_ARM64_WOW64) +wireguard-arm64.dll RCDATA "arm64\\wireguard.dll" +# else +# pragma message("ARM64 wireguard.dll was not built, so this will not work from WOW64") +# endif +#endif + +#define STRINGIZE(x) #x +#define EXPAND(x) STRINGIZE(x) + +VS_VERSION_INFO VERSIONINFO +FILEVERSION WIREGUARD_VERSION_MAJ, WIREGUARD_VERSION_MIN, WIREGUARD_VERSION_REL, 0 +PRODUCTVERSION WIREGUARD_VERSION_MAJ, WIREGUARD_VERSION_MIN, WIREGUARD_VERSION_REL, 0 +FILEOS VOS_NT_WINDOWS32 +FILETYPE VFT_DLL +FILESUBTYPE VFT2_UNKNOWN +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904b0" + BEGIN + VALUE "CompanyName", "WireGuard LLC" + VALUE "FileDescription", "WireGuard API Library" + VALUE "FileVersion", EXPAND(WIREGUARD_VERSION) + VALUE "InternalName", "wireguard.dll" + VALUE "LegalCopyright", "Copyright \xa9 2018-2021 WireGuard LLC. All Rights Reserved." + VALUE "OriginalFilename", "wireguard.dll" + VALUE "ProductName", "WireGuard Driver" + VALUE "ProductVersion", EXPAND(WIREGUARD_VERSION) + VALUE "Comments", "https://www.wireguard.net/" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1200 + END +END diff --git a/api/rundll32.c b/api/rundll32.c new file mode 100644 index 0000000..c12e0c2 --- /dev/null +++ b/api/rundll32.c @@ -0,0 +1,594 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#include "rundll32.h" +#include "adapter.h" +#include "main.h" +#include "logger.h" +#include "resource.h" +#include <Windows.h> +#include <shellapi.h> +#include <Shlwapi.h> +#include <cfgmgr32.h> +#include <objbase.h> +#include <assert.h> + +#ifdef ACCEPT_WOW64 + +# define EXPORT comment(linker, "/EXPORT:" __FUNCTION__ "=" __FUNCDNAME__) + +static DWORD +WriteFormatted(_In_ DWORD StdHandle, _In_z_ LPCWSTR Template, ...) +{ + LPWSTR FormattedMessage = NULL; + DWORD Size; + va_list Arguments; + va_start(Arguments, Template); + DWORD Len = FormatMessageW( + FORMAT_MESSAGE_FROM_STRING | FORMAT_MESSAGE_ALLOCATE_BUFFER, + Template, + 0, + 0, + (VOID *)&FormattedMessage, + 0, + &Arguments); + if (SUCCEEDED(DWordMult(Len, sizeof(*FormattedMessage), &Size))) + WriteFile(GetStdHandle(StdHandle), FormattedMessage, Size, &Size, NULL); + else + Size = 0; + LocalFree(FormattedMessage); + va_end(Arguments); + return Size / sizeof(*FormattedMessage); +} + +static VOID CALLBACK +ConsoleLogger(_In_ WIREGUARD_LOGGER_LEVEL Level, _In_z_ LPCWSTR LogLine) +{ + LPCWSTR Template; + switch (Level) + { + case WIREGUARD_LOG_INFO: + Template = L"[+] %1\n"; + break; + case WIREGUARD_LOG_WARN: + Template = L"[-] %1\n"; + break; + case WIREGUARD_LOG_ERR: + Template = L"[!] %1\n"; + break; + default: + return; + } + WriteFormatted(STD_ERROR_HANDLE, Template, LogLine); +} + +VOID __stdcall CreateAdapter(HWND hwnd, HINSTANCE hinst, LPSTR lpszCmdLine, int nCmdShow) +{ +# pragma EXPORT + + int Argc; + LPWSTR *Argv = CommandLineToArgvW(GetCommandLineW(), &Argc); + WireGuardSetLogger(ConsoleLogger); + + if (Argc < 4) + goto cleanup; + if (wcslen(Argv[2]) >= WIREGUARD_MAX_POOL) + goto cleanup; + if (wcslen(Argv[3]) >= MAX_ADAPTER_NAME) + goto cleanup; + GUID RequestedGUID; + if (Argc > 4 && FAILED(CLSIDFromString(Argv[4], &RequestedGUID))) + goto cleanup; + + BOOL RebootRequired; + WIREGUARD_ADAPTER *Adapter = + WireGuardCreateAdapter(Argv[2], Argv[3], Argc > 4 ? &RequestedGUID : NULL, &RebootRequired); + DWORD LastError = Adapter ? ERROR_SUCCESS : GetLastError(); + WriteFormatted( + STD_OUTPUT_HANDLE, L"%1!X! %2!s! %3!X!", LastError, Adapter ? Adapter->DevInstanceID : L"", RebootRequired); + if (Adapter) + WireGuardFreeAdapter(Adapter); + +cleanup: + LocalFree(Argv); +} + +VOID __stdcall DeleteAdapter(HWND hwnd, HINSTANCE hinst, LPSTR lpszCmdLine, int nCmdShow) +{ +# pragma EXPORT + + int Argc; + LPWSTR *Argv = CommandLineToArgvW(GetCommandLineW(), &Argc); + WireGuardSetLogger(ConsoleLogger); + + if (Argc < 3) + goto cleanup; + + DWORD LastError; + BOOL RebootRequired = FALSE; + WIREGUARD_ADAPTER *Adapter = AdapterOpenFromDevInstanceId(Argv[2], Argv[3]); + if (!Adapter) + { + LastError = GetLastError(); + goto write; + } + LastError = WireGuardDeleteAdapter(Adapter, &RebootRequired) ? ERROR_SUCCESS : GetLastError(); + WireGuardFreeAdapter(Adapter); +write: + WriteFormatted(STD_OUTPUT_HANDLE, L"%1!X! %2!X!", LastError, RebootRequired); + +cleanup: + LocalFree(Argv); +} + +VOID __stdcall DeletePoolDriver(HWND hwnd, HINSTANCE hinst, LPSTR lpszCmdLine, int nCmdShow) +{ +# pragma EXPORT + + int Argc; + LPWSTR *Argv = CommandLineToArgvW(GetCommandLineW(), &Argc); + WireGuardSetLogger(ConsoleLogger); + + if (Argc < 2) + goto cleanup; + + BOOL RebootRequired; + DWORD LastError = WireGuardDeletePoolDriver(Argv[2], &RebootRequired) ? ERROR_SUCCESS : GetLastError(); + WriteFormatted(STD_OUTPUT_HANDLE, L"%1!X! %2!X!", LastError, RebootRequired); + +cleanup: + LocalFree(Argv); +} +#endif + +#ifdef MAYBE_WOW64 + +_Return_type_success_(return != FALSE) +static BOOL +AppendToBuffer(_Inout_ LPWSTR *Buffer, _In_ CONST WCHAR Addition, _Inout_ SIZE_T *BufferPos, _Inout_ SIZE_T *BufferLen) +{ + SIZE_T NewPos; + if (FAILED(SIZETAdd(*BufferPos, sizeof(Addition), &NewPos))) + return FALSE; + if (NewPos >= *BufferLen) + { + SIZE_T NewLen; + if (FAILED(SIZETMult(NewPos, 3, &NewLen))) + return FALSE; + LPWSTR NewBuffer = ReZalloc(*Buffer, NewLen); + if (!NewBuffer) + return FALSE; + *Buffer = NewBuffer; + *BufferLen = NewLen; + } + SIZE_T NewIndex = *BufferPos / sizeof(**Buffer); + if (*Buffer + NewIndex < *Buffer) + return FALSE; + (*Buffer)[NewIndex] = Addition; + *BufferPos = NewPos; + return TRUE; +} + +_Must_inspect_result_ +static _Return_type_success_(return != NULL) +_Post_maybenull_ +LPWSTR +ArgvToCommandLineW(_In_ SIZE_T ArgCount, ...) +{ + LPWSTR Output = NULL; + SIZE_T BufferPos = 0, BufferLen = 0; +# define Append(Char) \ + do \ + { \ + if (!AppendToBuffer(&Output, Char, &BufferPos, &BufferLen)) \ + goto cleanupBuffer; \ + } while (0) + + va_list Args; + va_start(Args, ArgCount); + for (SIZE_T i = 0; i < ArgCount; ++i) + { + LPCWSTR Arg = va_arg(Args, LPCWSTR); + SIZE_T ArgLen = wcslen(Arg); + if (ArgLen >= DWORD_MAX >> 3) + goto cleanupBuffer; + if (i) + Append(L' '); + Append(L'"'); + for (SIZE_T j = 0;; ++j) + { + SIZE_T NumberBackslashes = 0; + + while (j < ArgLen && Arg[j] == L'\\') + { + ++j; + ++NumberBackslashes; + } + if (j >= ArgLen) + { + for (SIZE_T k = 0; k < NumberBackslashes * 2; ++k) + Append(L'\\'); + break; + } + else if (Arg[j] == L'"') + { + for (SIZE_T k = 0; k < NumberBackslashes * 2 + 1; ++k) + Append(L'\\'); + Append(Arg[j]); + } + else + { + for (SIZE_T k = 0; k < NumberBackslashes; ++k) + Append(L'\\'); + Append(Arg[j]); + } + } + Append(L'"'); + } + va_end(Args); + return Output; + +cleanupBuffer: + Free(Output); + return NULL; +# undef Append +} + +typedef struct _PROCESS_STDOUT_STATE +{ + HANDLE Stdout; + LPWSTR Response; + DWORD ResponseCapacity; +} PROCESS_STDOUT_STATE; + +_Return_type_success_(return != ERROR_SUCCESS) +static DWORD WINAPI +ProcessStdout(_Inout_ PROCESS_STDOUT_STATE *State) +{ + for (DWORD Offset = 0, MaxLen = State->ResponseCapacity - 1; Offset < MaxLen;) + { + DWORD Size; + if (FAILED(DWordMult(MaxLen - Offset, sizeof(WCHAR), &Size))) + return ERROR_BUFFER_OVERFLOW; + if (!ReadFile(State->Stdout, State->Response + Offset, Size, &Size, NULL)) + return ERROR_SUCCESS; + if (Size % sizeof(WCHAR)) + return ERROR_INVALID_DATA; + Offset += Size / sizeof(WCHAR); + State->Response[Offset] = 0; + } + return ERROR_BUFFER_OVERFLOW; +} + +static DWORD WINAPI +ProcessStderr(_In_ HANDLE Stderr) +{ + enum + { + OnNone, + OnLevelStart, + OnLevel, + OnLevelEnd, + OnSpace, + OnMsg + } State = OnNone; + WCHAR Msg[0x200]; + DWORD Count = 0; + WIREGUARD_LOGGER_LEVEL Level = WIREGUARD_LOG_INFO; + for (;;) + { + WCHAR Buf[0x200]; + DWORD SizeRead; + if (!ReadFile(Stderr, Buf, sizeof(Buf), &SizeRead, NULL)) + return ERROR_SUCCESS; + if (SizeRead % sizeof(*Buf)) + return ERROR_INVALID_DATA; + SizeRead /= sizeof(*Buf); + for (DWORD i = 0; i < SizeRead; ++i) + { + WCHAR c = Buf[i]; + if (State == OnNone && c == L'[') + State = OnLevelStart; + else if ( + State == OnLevelStart && + ((Level = WIREGUARD_LOG_INFO, c == L'+') || (Level = WIREGUARD_LOG_WARN, c == L'-') || + (Level = WIREGUARD_LOG_ERR, c == L'!'))) + State = OnLevelEnd; + else if (State == OnLevelEnd && c == L']') + State = OnSpace; + else if (State == OnSpace && !iswspace(c) || State == OnMsg && c != L'\r' && c != L'\n') + { + if (Count < _countof(Msg) - 1) + Msg[Count++] = c; + State = OnMsg; + } + else if (State == OnMsg && c == L'\n') + { + Msg[Count] = 0; + LoggerLog(Level, NULL, Msg); + State = OnNone; + Count = 0; + } + } + } +} + +static _Return_type_success_(return != FALSE) +BOOL +ExecuteRunDll32( + _In_z_ LPCWSTR Function, + _In_z_ LPCWSTR Arguments, + _Out_z_cap_c_(ResponseCapacity) LPWSTR Response, + _In_ DWORD ResponseCapacity) +{ + WCHAR WindowsDirectory[MAX_PATH]; + if (!GetWindowsDirectoryW(WindowsDirectory, _countof(WindowsDirectory))) + { + LOG_LAST_ERROR(L"Failed to get Windows folder"); + return FALSE; + } + WCHAR RunDll32Path[MAX_PATH]; + if (!PathCombineW(RunDll32Path, WindowsDirectory, L"Sysnative\\rundll32.exe")) + { + SetLastError(ERROR_BUFFER_OVERFLOW); + return FALSE; + } + + DWORD LastError; + WCHAR RandomTempSubDirectory[MAX_PATH]; + if (!ResourceCreateTemporaryDirectory(RandomTempSubDirectory)) + { + LOG(WIREGUARD_LOG_ERR, L"Failed to create temporary folder"); + return FALSE; + } + WCHAR DllPath[MAX_PATH] = { 0 }; + if (!PathCombineW(DllPath, RandomTempSubDirectory, L"wireguard.dll")) + { + LastError = ERROR_BUFFER_OVERFLOW; + goto cleanupDirectory; + } + LPCWSTR WireGuardDllResourceName; + switch (NativeMachine) + { + case IMAGE_FILE_MACHINE_AMD64: + WireGuardDllResourceName = L"wireguard-amd64.dll"; + break; + case IMAGE_FILE_MACHINE_ARM64: + WireGuardDllResourceName = L"wireguard-arm64.dll"; + break; + default: + LOG(WIREGUARD_LOG_ERR, L"Unsupported platform 0x%x", NativeMachine); + LastError = ERROR_NOT_SUPPORTED; + goto cleanupDirectory; + } + if (!ResourceCopyToFile(DllPath, WireGuardDllResourceName)) + { + LastError = LOG(WIREGUARD_LOG_ERR, L"Failed to copy resource %s to %s", WireGuardDllResourceName, DllPath); + goto cleanupDelete; + } + size_t CommandLineLen = 10 + MAX_PATH + 2 + wcslen(Arguments) + 1 + wcslen(Function) + 1; + LPWSTR CommandLine = AllocArray(CommandLineLen, sizeof(*CommandLine)); + if (!CommandLine) + { + LastError = GetLastError(); + goto cleanupDelete; + } + if (_snwprintf_s( + CommandLine, + CommandLineLen, + _TRUNCATE, + L"rundll32 \"%.*s\",%s %s", + MAX_PATH, + DllPath, + Function, + Arguments) == -1) + { + LOG(WIREGUARD_LOG_ERR, L"Command line too long"); + LastError = ERROR_INVALID_PARAMETER; + goto cleanupDelete; + } + HANDLE StreamRStdout = INVALID_HANDLE_VALUE, StreamRStderr = INVALID_HANDLE_VALUE, + StreamWStdout = INVALID_HANDLE_VALUE, StreamWStderr = INVALID_HANDLE_VALUE; + if (!CreatePipe(&StreamRStdout, &StreamWStdout, &SecurityAttributes, 0) || + !CreatePipe(&StreamRStderr, &StreamWStderr, &SecurityAttributes, 0)) + { + LastError = LOG_LAST_ERROR(L"Failed to create pipes"); + goto cleanupPipes; + } + if (!SetHandleInformation(StreamWStdout, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT) || + !SetHandleInformation(StreamWStderr, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT)) + { + LastError = LOG_LAST_ERROR(L"Failed to set handle info"); + goto cleanupPipes; + } + if (ResponseCapacity) + Response[0] = 0; + PROCESS_STDOUT_STATE ProcessStdoutState = { .Stdout = StreamRStdout, + .Response = Response, + .ResponseCapacity = ResponseCapacity }; + HANDLE ThreadStdout = NULL, ThreadStderr = NULL; + if ((ThreadStdout = CreateThread(NULL, 0, ProcessStdout, &ProcessStdoutState, 0, NULL)) == NULL || + (ThreadStderr = CreateThread(NULL, 0, ProcessStderr, StreamRStderr, 0, NULL)) == NULL) + { + LastError = LOG_LAST_ERROR(L"Failed to spawn readers"); + goto cleanupThreads; + } + STARTUPINFOW si = { .cb = sizeof(STARTUPINFO), + .dwFlags = STARTF_USESHOWWINDOW | STARTF_USESTDHANDLES, + .wShowWindow = SW_HIDE, + .hStdOutput = StreamWStdout, + .hStdError = StreamWStderr }; + PROCESS_INFORMATION pi; + if (!CreateProcessW(RunDll32Path, CommandLine, NULL, NULL, TRUE, 0, NULL, NULL, &si, &pi)) + { + LastError = LOG_LAST_ERROR(L"Failed to create process: %s", CommandLine); + goto cleanupThreads; + } + LastError = ERROR_SUCCESS; + WaitForSingleObject(pi.hProcess, INFINITE); + CloseHandle(pi.hProcess); + CloseHandle(pi.hThread); +cleanupThreads: + if (ThreadStderr) + { + CloseHandle(StreamWStderr); + StreamWStderr = INVALID_HANDLE_VALUE; + WaitForSingleObject(ThreadStderr, INFINITE); + CloseHandle(ThreadStderr); + } + if (ThreadStdout) + { + CloseHandle(StreamWStdout); + StreamWStdout = INVALID_HANDLE_VALUE; + WaitForSingleObject(ThreadStdout, INFINITE); + DWORD ThreadResult; + if (!GetExitCodeThread(ThreadStdout, &ThreadResult)) + LOG_LAST_ERROR(L"Failed to retrieve stdout reader result"); + else if (ThreadResult != ERROR_SUCCESS) + LOG_ERROR(LastError, L"Failed to read process output"); + CloseHandle(ThreadStdout); + } +cleanupPipes: + CloseHandle(StreamRStderr); + CloseHandle(StreamWStderr); + CloseHandle(StreamRStdout); + CloseHandle(StreamWStdout); + Free(CommandLine); +cleanupDelete: + DeleteFileW(DllPath); +cleanupDirectory: + RemoveDirectoryW(RandomTempSubDirectory); + return RET_ERROR(TRUE, LastError); +} + +_Use_decl_annotations_ +WIREGUARD_ADAPTER * +CreateAdapterViaRundll32(LPCWSTR Pool, LPCWSTR Name, const GUID *RequestedGUID, BOOL *RebootRequired) +{ + LOG(WIREGUARD_LOG_INFO, L"Spawning native process"); + LPWSTR Arguments = NULL; + if (RequestedGUID) + { + WCHAR RequestedGUIDStr[MAX_GUID_STRING_LEN]; + if (StringFromGUID2(RequestedGUID, RequestedGUIDStr, _countof(RequestedGUIDStr))) + Arguments = ArgvToCommandLineW(3, Pool, Name, RequestedGUIDStr); + } + else + Arguments = ArgvToCommandLineW(2, Pool, Name); + if (!Arguments) + { + LOG(WIREGUARD_LOG_ERR, L"Command line too long"); + SetLastError(ERROR_INVALID_PARAMETER); + return NULL; + } + WIREGUARD_ADAPTER *Adapter = NULL; + DWORD LastError; + WCHAR Response[8 + 1 + MAX_GUID_STRING_LEN + 1 + 8 + 1]; + if (!ExecuteRunDll32(L"CreateAdapter", Arguments, Response, _countof(Response))) + { + LastError = GetLastError(); + LOG(WIREGUARD_LOG_ERR, L"Error executing worker process: %s", Arguments); + goto cleanupArguments; + } + int Argc; + LPWSTR *Argv = CommandLineToArgvW(Response, &Argc); + if (Argc < 3) + { + LOG(WIREGUARD_LOG_ERR, L"Incomplete response: %s", Response); + LastError = ERROR_INVALID_PARAMETER; + goto cleanupArgv; + } + LastError = wcstoul(Argv[0], NULL, 16); + if (LastError == ERROR_SUCCESS && (Adapter = AdapterOpenFromDevInstanceId(Pool, Argv[1])) == NULL) + { + LOG(WIREGUARD_LOG_ERR, L"Failed to get adapter %s", Argv[1]); + LastError = ERROR_FILE_NOT_FOUND; + } + if (wcstoul(Argv[2], NULL, 16)) + *RebootRequired = TRUE; +cleanupArgv: + LocalFree(Argv); +cleanupArguments: + Free(Arguments); + SetLastError(LastError); + return Adapter; +} + +_Use_decl_annotations_ +BOOL +DeleteAdapterViaRundll32(const WIREGUARD_ADAPTER *Adapter, BOOL *RebootRequired) +{ + LOG(WIREGUARD_LOG_INFO, L"Spawning native process"); + LPWSTR Arguments = ArgvToCommandLineW(2, Adapter->Pool, Adapter->DevInstanceID); + if (!Arguments) + { + LOG(WIREGUARD_LOG_ERR, L"Command line too long"); + SetLastError(ERROR_INVALID_PARAMETER); + return FALSE; + } + WCHAR Response[8 + 1 + 8 + 1]; + DWORD LastError; + if (!ExecuteRunDll32(L"DeleteAdapter", Arguments, Response, _countof(Response))) + { + LastError = GetLastError(); + LOG(WIREGUARD_LOG_ERR, L"Error executing worker process: %s", Arguments); + goto cleanupArguments; + } + int Argc; + LPWSTR *Argv = CommandLineToArgvW(Response, &Argc); + if (Argc < 2) + { + LOG(WIREGUARD_LOG_ERR, L"Incomplete response: %s", Response); + LastError = ERROR_INVALID_PARAMETER; + goto cleanupArgv; + } + LastError = wcstoul(Argv[0], NULL, 16); + if (wcstoul(Argv[1], NULL, 16)) + *RebootRequired = TRUE; +cleanupArgv: + LocalFree(Argv); +cleanupArguments: + Free(Arguments); + return RET_ERROR(TRUE, LastError); +} + +_Use_decl_annotations_ +BOOL +DeletePoolDriverViaRundll32(LPCWSTR Pool, BOOL *RebootRequired) +{ + LOG(WIREGUARD_LOG_INFO, L"Spawning native process"); + LPWSTR Arguments = ArgvToCommandLineW(1, Pool); + if (!Arguments) + { + LOG(WIREGUARD_LOG_ERR, L"Command line too long"); + SetLastError(ERROR_INVALID_PARAMETER); + return FALSE; + } + WCHAR Response[8 + 1 + 8 + 1]; + DWORD LastError; + if (!ExecuteRunDll32(L"DeletePoolDriver", Arguments, Response, _countof(Response))) + { + LastError = GetLastError(); + LOG(WIREGUARD_LOG_ERR, L"Error executing worker process: %s", Arguments); + goto cleanupArguments; + } + int Argc; + LPWSTR *Argv = CommandLineToArgvW(Response, &Argc); + if (Argc < 2) + { + LOG(WIREGUARD_LOG_ERR, L"Incomplete response: %s", Response); + LastError = ERROR_INVALID_PARAMETER; + goto cleanupArgv; + } + LastError = wcstoul(Argv[0], NULL, 16); + if (wcstoul(Argv[1], NULL, 16)) + *RebootRequired = TRUE; +cleanupArgv: + LocalFree(Argv); +cleanupArguments: + Free(Arguments); + return RET_ERROR(TRUE, LastError); +} +#endif diff --git a/api/rundll32.h b/api/rundll32.h new file mode 100644 index 0000000..03a88cb --- /dev/null +++ b/api/rundll32.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#pragma once + +#include "adapter.h" + +_Must_inspect_result_ +_Return_type_success_(return != NULL) +_Post_maybenull_ +WIREGUARD_ADAPTER * +CreateAdapterViaRundll32( + _In_z_ LPCWSTR Pool, + _In_z_ LPCWSTR Name, + _In_opt_ const GUID *RequestedGUID, + _Inout_ BOOL *RebootRequired); + +_Return_type_success_(return != FALSE) +BOOL +DeleteAdapterViaRundll32(_In_ const WIREGUARD_ADAPTER *Adapter, _Inout_ BOOL *RebootRequired); + +_Return_type_success_(return != FALSE) +BOOL +DeletePoolDriverViaRundll32(_In_z_ LPCWSTR Pool, _Inout_ BOOL *RebootRequired); diff --git a/api/wireguard.h b/api/wireguard.h new file mode 100644 index 0000000..2f63be7 --- /dev/null +++ b/api/wireguard.h @@ -0,0 +1,371 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#pragma once + +#include <winsock2.h> +#include <windows.h> +#include <ipexport.h> +#include <ifdef.h> +#include <ws2ipdef.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#pragma warning(push) +#pragma warning(disable : 4324) /* structure was padded due to alignment specifier */ + +/** + * A handle representing WireGuard adapter + */ +typedef struct _WIREGUARD_ADAPTER *WIREGUARD_ADAPTER_HANDLE; + +/** + * Maximum pool name length including zero terminator + */ +#define WIREGUARD_MAX_POOL 256 + +/** + * Creates a new WireGuard adapter. + * + * @param Pool Name of the adapter pool. Zero-terminated string of up to WIREGUARD_MAX_POOL-1 characters. + * + * @param Name The requested name of the adapter. Zero-terminated string of up to MAX_ADAPTER_NAME-1 + * characters. + * + * @param RequestedGUID The GUID of the created network adapter, which then influences NLA generation deterministically. + * If it is set to NULL, the GUID is chosen by the system at random, and hence a new NLA entry is + * created for each new adapter. It is called "requested" GUID because the API it uses is + * completely undocumented, and so there could be minor interesting complications with its usage. + * + * @param RebootRequired Optional pointer to a boolean flag to be set to TRUE in case SetupAPI suggests a reboot. + * + * @return If the function succeeds, the return value is the adapter handle. Must be released with WireGuardFreeAdapter. + * If the function fails, the return value is NULL. To get extended error information, call GetLastError. + */ +typedef _Must_inspect_result_ +_Return_type_success_(return != NULL) +_Post_maybenull_ +WIREGUARD_ADAPTER_HANDLE(WINAPI WIREGUARD_CREATE_ADAPTER_FUNC) +(_In_z_ LPCWSTR Pool, _In_z_ LPCWSTR Name, _In_opt_ const GUID *RequestedGUID, _Out_opt_ BOOL *RebootRequired); + +/** + * Opens an existing WireGuard adapter. + * + * @param Pool Name of the adapter pool. Zero-terminated string of up to WIREGUARD_MAX_POOL-1 characters. + * + * @param Name Adapter name. Zero-terminated string of up to MAX_ADAPTER_NAME-1 characters. + * + * @return If the function succeeds, the return value is adapter handle. Must be released with WireGuardFreeAdapter. If + * the function fails, the return value is NULL. To get extended error information, call GetLastError. Possible errors + * include the following: ERROR_FILE_NOT_FOUND if adapter with given name is not found; ERROR_ALREADY_EXISTS if adapter + * is found but not a WireGuard-class or not a member of the pool + */ +typedef _Must_inspect_result_ +_Return_type_success_(return != NULL) +_Post_maybenull_ +WIREGUARD_ADAPTER_HANDLE(WINAPI WIREGUARD_OPEN_ADAPTER_FUNC)(_In_z_ LPCWSTR Pool, _In_z_ LPCWSTR Name); + +/** + * Deletes a WireGuard adapter. + * + * @param Adapter Adapter handle obtained with WireGuardOpenAdapter or WireGuardCreateAdapter. + * + * @param RebootRequired Optional pointer to a boolean flag to be set to TRUE in case SetupAPI suggests a reboot. + * + * @return If the function succeeds, the return value is nonzero. If the function fails, the return value is zero. To + * get extended error information, call GetLastError. + */ +typedef _Return_type_success_(return != FALSE) +BOOL(WINAPI WIREGUARD_DELETE_ADAPTER_FUNC) +(_In_ WIREGUARD_ADAPTER_HANDLE Adapter, _Out_opt_ BOOL *RebootRequired); + +/** + * Called by WireGuardEnumAdapters for each adapter in the pool. + * + * @param Adapter Adapter handle, which will be freed when this function returns. + * + * @param Param An application-defined value passed to the WireGuardEnumAdapters. + * + * @return Non-zero to continue iterating adapters; zero to stop. + */ +typedef BOOL(CALLBACK *WIREGUARD_ENUM_CALLBACK)(_In_ WIREGUARD_ADAPTER_HANDLE Adapter, _In_ LPARAM Param); + +/** + * Enumerates all WireGuard adapters. + * + * @param Pool Name of the adapter pool. Zero-terminated string of up to WIREGUARD_MAX_POOL-1 characters. + * + * @param Callback Callback function. To continue enumeration, the callback function must return TRUE; to stop + * enumeration, it must return FALSE. + * + * @param Param An application-defined value to be passed to the callback function. + * + * @return If the function succeeds, the return value is nonzero. If the function fails, the return value is zero. To + * get extended error information, call GetLastError. + */ +typedef _Return_type_success_(return != FALSE) +BOOL(WINAPI WIREGUARD_ENUM_ADAPTERS_FUNC) +(_In_z_ LPCWSTR Pool, _In_ WIREGUARD_ENUM_CALLBACK Callback, _In_ LPARAM Param); + +/** + * Releases WireGuard adapter resources. + * + * @param Adapter Adapter handle obtained with WireGuardOpenAdapter or WireGuardCreateAdapter. + */ +typedef VOID(WINAPI WIREGUARD_FREE_ADAPTER_FUNC)(_In_opt_ WIREGUARD_ADAPTER_HANDLE Adapter); + +/** + * Deletes all WireGuard adapters in a pool and if there are no more adapters in any other pools, also removes WireGuard + * from the driver store, usually called by uninstallers. + * + * @param Pool Name of the adapter pool. Zero-terminated string of up to WIREGUARD_MAX_POOL-1 characters. + * + * @param RebootRequired Optional pointer to a boolean flag to be set to TRUE in case SetupAPI suggests a reboot. + * + * @return If the function succeeds, the return value is nonzero. If the function fails, the return value is zero. To + * get extended error information, call GetLastError. + */ +typedef _Return_type_success_(return != FALSE) +BOOL(WINAPI WIREGUARD_DELETE_POOL_DRIVER_FUNC)(_In_z_ LPCWSTR Pool, _Out_opt_ BOOL *RebootRequired); + +/** + * Returns the LUID of the adapter. + * + * @param Adapter Adapter handle obtained with WireGuardOpenAdapter or WireGuardCreateAdapter + * + * @param Luid Pointer to LUID to receive adapter LUID. + */ +typedef VOID(WINAPI WIREGUARD_GET_ADAPTER_LUID_FUNC)(_In_ WIREGUARD_ADAPTER_HANDLE Adapter, _Out_ NET_LUID *Luid); + +/** + * Returns the name of the WireGuard adapter. + * + * @param Adapter Adapter handle obtained with WireGuardOpenAdapter or WireGuardCreateAdapter + * + * @param Name Pointer to a string to receive adapter name + * + * @return If the function succeeds, the return value is nonzero. If the function fails, the return value is zero. To + * get extended error information, call GetLastError. + */ +typedef _Must_inspect_result_ +_Return_type_success_(return != FALSE) +BOOL(WINAPI WIREGUARD_GET_ADAPTER_NAME_FUNC) +(_In_ WIREGUARD_ADAPTER_HANDLE Adapter, _Out_writes_z_(MAX_ADAPTER_NAME) LPWSTR Name); + +/** + * Sets name of the WireGuard adapter. + * + * @param Adapter Adapter handle obtained with WireGuardOpenAdapter or WireGuardCreateAdapter + * + * @param Name Adapter name. Zero-terminated string of up to MAX_ADAPTER_NAME-1 characters. + * + * @return If the function succeeds, the return value is nonzero. If the function fails, the return value is zero. To + * get extended error information, call GetLastError. + */ +typedef _Return_type_success_(return != FALSE) +BOOL(WINAPI WIREGUARD_SET_ADAPTER_NAME_FUNC) +(_In_ WIREGUARD_ADAPTER_HANDLE Adapter, _In_reads_or_z_(MAX_ADAPTER_NAME) LPCWSTR Name); + +/** + * Determines the version of the WireGuard driver currently loaded. + * + * @return If the function succeeds, the return value is the version number. If the function fails, the return value is + * zero. To get extended error information, call GetLastError. Possible errors include the following: + * ERROR_FILE_NOT_FOUND WireGuard not loaded + */ +typedef _Return_type_success_(return != 0) +DWORD(WINAPI WIREGUARD_GET_RUNNING_DRIVER_VERSION_FUNC)(VOID); + +/** + * Determines the level of logging, passed to WIREGUARD_LOGGER_CALLBACK. + */ +typedef enum +{ + WIREGUARD_LOG_INFO, /**< Informational */ + WIREGUARD_LOG_WARN, /**< Warning */ + WIREGUARD_LOG_ERR /**< Error */ +} WIREGUARD_LOGGER_LEVEL; + +/** + * Called by internal logger to report diagnostic messages + * + * @param Level Message level. + * + * @param Message Message text. + */ +typedef VOID(CALLBACK *WIREGUARD_LOGGER_CALLBACK)(_In_ WIREGUARD_LOGGER_LEVEL Level, _In_z_ LPCWSTR Message); + +/** + * Sets logger callback function. + * + * @param NewLogger Pointer to callback function to use as a new global logger. NewLogger may be called from various + * threads concurrently. Should the logging require serialization, you must handle serialization in + * NewLogger. Set to NULL to disable. + */ +typedef VOID(WINAPI WIREGUARD_SET_LOGGER_FUNC)(_In_ WIREGUARD_LOGGER_CALLBACK NewLogger); + +/** + * Whether and how logs from the driver are collected for the callback function. + */ +typedef enum +{ + WIREGUARD_ADAPTER_LOG_OFF, /**< No logs are generated from the driver. */ + WIREGUARD_ADAPTER_LOG_ON, /**< Logs are generated from the driver. */ + WIREGUARD_ADAPTER_LOG_ON_WITH_PREFIX /**< Logs are generated from the driver, index-prefixed. */ +} WIREGUARD_ADAPTER_LOG_STATE; + +/** + * Sets whether and how the adapter logs to the logger previously set up with WireGuardSetLoggerFunc. + * + * @param Adapter Adapter handle obtained with WireGuardOpenAdapter or WireGuardCreateAdapter + * + * @param LogState Adapter logging state. + * + * @return If the function succeeds, the return value is nonzero. If the function fails, the return value is zero. To + * get extended error information, call GetLastError. + */ +typedef _Return_type_success_(return != FALSE) +BOOL(WINAPI WIREGUARD_SET_ADAPTER_LOGGING_FUNC) +(_In_ WIREGUARD_ADAPTER_HANDLE Adapter, _In_ WIREGUARD_ADAPTER_LOG_STATE LogState); + +/** + * Determines the state of the adapter. + */ +typedef enum +{ + WIREGUARD_ADAPTER_STATE_DOWN, /**< Down */ + WIREGUARD_ADAPTER_STATE_UP, /**< Up */ +} WIREGUARD_ADAPTER_STATE; + +/** + * Sets the adapter state of the WireGuard adapter. Note: sockets are owned by the process that sets the state to up. + * + * @param Adapter Adapter handle obtained with WireGuardOpenAdapter or WireGuardCreateAdapter + * + * @param State Adapter state. + * + * @return If the function succeeds, the return value is nonzero. If the function fails, the return value is zero. To + * get extended error information, call GetLastError. + */ +typedef _Return_type_success_(return != FALSE) +BOOL(WINAPI WIREGUARD_SET_ADAPTER_STATE_FUNC) +(_In_ WIREGUARD_ADAPTER_HANDLE Adapter, _In_ WIREGUARD_ADAPTER_STATE State); + +/** + * Gets the adapter state of the WireGuard adapter. + * + * @param Adapter Adapter handle obtained with WireGuardOpenAdapter or WireGuardCreateAdapter + * + * @param State Pointer to adapter state. + * + * @return If the function succeeds, the return value is nonzero. If the function fails, the return value is zero. To + * get extended error information, call GetLastError. + */ +typedef _Must_inspect_result_ +_Return_type_success_(return != FALSE) +BOOL(WINAPI WIREGUARD_GET_ADAPTER_STATE_FUNC) +(_In_ WIREGUARD_ADAPTER_HANDLE Adapter, _Out_ WIREGUARD_ADAPTER_STATE *State); + +#define WIREGUARD_KEY_LENGTH 32 + +typedef struct _WIREGUARD_ALLOWED_IP WIREGUARD_ALLOWED_IP; +__declspec(align(8)) struct _WIREGUARD_ALLOWED_IP +{ + union + { + IN_ADDR V4; + IN6_ADDR V6; + } Address; /**< IP address */ + ADDRESS_FAMILY AddressFamily; /**< Address family, either AF_INET or AF_INET6 */ + BYTE Cidr; /**< CIDR of allowed IPs */ +}; + +typedef enum +{ + WIREGUARD_PEER_HAS_PUBLIC_KEY = 1 << 0, /**< The PublicKey field is set */ + WIREGUARD_PEER_HAS_PRESHARED_KEY = 1 << 1, /**< The PresharedKey field is set */ + WIREGUARD_PEER_HAS_PERSISTENT_KEEPALIVE = 1 << 2, /**< The PersistentKeepAlive field is set */ + WIREGUARD_PEER_HAS_ENDPOINT = 1 << 3, /**< The Endpoint field is set */ + WIREGUARD_PEER_REPLACE_ALLOWED_IPS = 1 << 5, /**< Remove all allowed IPs before adding new ones */ + WIREGUARD_PEER_REMOVE = 1 << 6, /**< Remove specified peer */ + WIREGUARD_PEER_UPDATE = 1 << 7 /**< Do not add a new peer */ +} WIREGUARD_PEER_FLAG; + +typedef struct _WIREGUARD_PEER WIREGUARD_PEER; +__declspec(align(8)) struct _WIREGUARD_PEER +{ + WIREGUARD_PEER_FLAG Flags; /**< Bitwise combination of flags */ + DWORD Reserved; /**< Reserved; must be zero */ + BYTE PublicKey[WIREGUARD_KEY_LENGTH]; /**< Public key, the peer's primary identifier */ + BYTE PresharedKey[WIREGUARD_KEY_LENGTH]; /**< Preshared key for additional layer of post-quantum resistance */ + WORD PersistentKeepalive; /**< Seconds interval, or 0 to disable */ + SOCKADDR_INET Endpoint; /**< Endpoint, with IP address and UDP port number*/ + DWORD64 TxBytes; /**< Number of bytes transmitted */ + DWORD64 RxBytes; /**< Number of bytes received */ + DWORD64 LastHandshake; /**< Time of the last handshake, in 100ns intervals since 1601-01-01 UTC */ + DWORD AllowedIPsCount; /**< Number of allowed IP structs following this struct */ +}; + +typedef enum +{ + WIREGUARD_INTERFACE_HAS_PUBLIC_KEY = (1 << 0), /**< The PublicKey field is set */ + WIREGUARD_INTERFACE_HAS_PRIVATE_KEY = (1 << 1), /**< The PrivateKey field is set */ + WIREGUARD_INTERFACE_HAS_LISTEN_PORT = (1 << 2), /**< The ListenPort field is set */ + WIREGUARD_INTERFACE_REPLACE_PEERS = (1 << 3) /**< Remove all peers before adding new ones */ +} WIREGUARD_INTERFACE_FLAG; + +typedef struct _WIREGUARD_INTERFACE WIREGUARD_INTERFACE; +__declspec(align(8)) struct _WIREGUARD_INTERFACE +{ + WIREGUARD_INTERFACE_FLAG Flags; /**< Bitwise combination of flags */ + WORD ListenPort; /**< Port for UDP listen socket, or 0 to choose randomly */ + BYTE PrivateKey[WIREGUARD_KEY_LENGTH]; /**< Private key of interface */ + BYTE PublicKey[WIREGUARD_KEY_LENGTH]; /**< Corresponding public key of private key */ + DWORD PeersCount; /**< Number of peer structs following this struct */ +}; + +/** + * Sets the configuration of the WireGuard adapter. + * + * @param Adapter Adapter handle obtained with WireGuardOpenAdapter or WireGuardCreateAdapter + * + * @param Config Configuration for the adapter. + * + * @param Bytes Number of bytes in Config allocation. + * + * @return If the function succeeds, the return value is nonzero. If the function fails, the return value is zero. To + * get extended error information, call GetLastError. + */ +typedef _Return_type_success_(return != FALSE) +BOOL(WINAPI WIREGUARD_SET_CONFIGURATION_FUNC) +(_In_ WIREGUARD_ADAPTER_HANDLE Adapter, _In_reads_bytes_(Bytes) const WIREGUARD_INTERFACE *Config, _In_ DWORD Bytes); + +/** + * Gets the configuration of the WireGuard adapter. + * + * @param Adapter Adapter handle obtained with WireGuardOpenAdapter or WireGuardCreateAdapter + * + * @param Config Configuration for the adapter. + * + * @param Bytes Pointer to number of bytes in Config allocation. + * + * @return If the function succeeds, the return value is nonzero. If the function fails, the return value is zero. To + * get extended error information, call GetLastError, which if ERROR_MORE_DATA, Bytes is updated with the + * required size. + */ +typedef _Must_inspect_result_ +_Return_type_success_(return != FALSE) +BOOL(WINAPI WIREGUARD_GET_CONFIGURATION_FUNC) +(_In_ WIREGUARD_ADAPTER_HANDLE Adapter, + _Out_writes_bytes_all_(*Bytes) WIREGUARD_INTERFACE *Config, + _Inout_ DWORD *Bytes); + +#pragma warning(pop) + +#ifdef __cplusplus +} +#endif diff --git a/driver/allowedips.c b/driver/allowedips.c new file mode 100644 index 0000000..a8928ce --- /dev/null +++ b/driver/allowedips.c @@ -0,0 +1,456 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include "allowedips.h" +#include "containers.h" +#include "peer.h" +#include "logging.h" + +#define STACK_ENTRIES 128 + +static LOOKASIDE_ALIGN LOOKASIDE_LIST_EX NodeCache; + +static VOID +SwapEndian(_Out_writes_bytes_all_(Bits / 8) UINT8 *Dst, _In_reads_bytes_(Bits / 8) CONST UINT8 *Src, _In_ UINT8 Bits) +{ + if (Bits == 32) + { + *(UINT32 *)Dst = Be32ToCpu(*(CONST UINT32_BE *)Src); + } + else if (Bits == 128) + { + ((UINT64 *)Dst)[0] = Be64ToCpu(((CONST UINT64_BE *)Src)[0]); + ((UINT64 *)Dst)[1] = Be64ToCpu(((CONST UINT64_BE *)Src)[1]); + } +} + +static VOID +CopyAndAssignCidr( + _Out_ ALLOWEDIPS_NODE *Node, + _In_reads_bytes_(Bits / 8) CONST UINT8 *Src, + _In_ UINT8 Cidr, + _In_ UINT8 Bits) +{ + Node->Cidr = Cidr; + Node->BitAtA = Cidr / 8U; +#if REG_DWORD == REG_DWORD_LITTLE_ENDIAN + Node->BitAtA ^= (Bits / 8U - 1U) % 8U; +#endif + Node->BitAtB = 7U - (Cidr % 8U); + Node->Bitlen = Bits; + RtlCopyMemory(Node->Bits, Src, Bits / 8U); +} + +static inline UINT8 +Choose(_In_ CONST ALLOWEDIPS_NODE *Node, _In_ CONST UINT8 *Key) +{ + return (Key[Node->BitAtA] >> Node->BitAtB) & 1; +} + +static VOID +PushRcu(_Inout_ ALLOWEDIPS_NODE *Stack[STACK_ENTRIES], _In_ ALLOWEDIPS_NODE __rcu *P, _In_ ULONG *Len) +{ + /* This lets us use it from mutex-protected or cleanup functions too. */ + _Analysis_assume_rcu_held_; + if (RcuAccessPointer(P)) + { + NT_ASSERT(*Len < STACK_ENTRIES); + Stack[(*Len)++] = RcuDereference(ALLOWEDIPS_NODE, P); + } + _Analysis_assume_rcu_not_held_; +} + +static RCU_CALLBACK_FN NodeFreeRcu; +_Use_decl_annotations_ +static VOID +NodeFreeRcu(RCU_CALLBACK *Rcu) +{ + ExFreeToLookasideListEx(&NodeCache, CONTAINING_RECORD(Rcu, ALLOWEDIPS_NODE, Rcu)); +} + +static RCU_CALLBACK_FN RootFreeRcu; +#pragma warning(suppress : 6262) /* Using 1044 bytes of stack is still below 1280. */ +_Use_decl_annotations_ +static VOID +RootFreeRcu(RCU_CALLBACK *Rcu) +{ + ALLOWEDIPS_NODE *Node, *Stack[STACK_ENTRIES] = { CONTAINING_RECORD(Rcu, ALLOWEDIPS_NODE, Rcu) }; + ULONG Len = 1; + + while (Len > 0 && (Node = Stack[--Len]) != NULL) + { + PushRcu(Stack, Node->Bit[0], &Len); + PushRcu(Stack, Node->Bit[1], &Len); + ExFreeToLookasideListEx(&NodeCache, Node); + } +} + +#pragma warning(suppress : 6262) /* Using 1044 bytes of stack is still below 1280. */ +static VOID +RootRemovePeerLists(_In_ ALLOWEDIPS_NODE *Root) +{ + ALLOWEDIPS_NODE *Node, *Stack[STACK_ENTRIES] = { Root }; + ULONG Len = 1; + while (Len > 0 && (Node = Stack[--Len]) != NULL) + { + PushRcu(Stack, Node->Bit[0], &Len); + PushRcu(Stack, Node->Bit[1], &Len); + if (RcuAccessPointer(Node->Peer)) + RemoveEntryList(&Node->PeerList); + } +} + +static UINT8 +CommonBits(_In_ CONST ALLOWEDIPS_NODE *Node, _In_reads_bytes_(Bits / 8) CONST UINT8 *Key, _In_ UINT8 Bits) +{ + if (Bits == 32) + return 32 - (UINT8)FindLastSet32(*(CONST UINT32 *)Node->Bits ^ *(CONST UINT32 *)Key); + else if (Bits == 128) + return 128 - (UINT8)FindLastSet128( + *(CONST UINT64 *)&Node->Bits[0] ^ *(CONST UINT64 *)&Key[0], + *(CONST UINT64 *)&Node->Bits[8] ^ *(CONST UINT64 *)&Key[8]); + return 0; +} + +static BOOLEAN +PrefixMatches(_In_ CONST ALLOWEDIPS_NODE *Node, _In_reads_bytes_(Bits / 8) CONST UINT8 *Key, _In_ UINT8 Bits) +{ + /* This could be much faster if it actually just compared the common + * bits properly, by precomputing a mask bswap(~0 << (32 - cidr)), and + * the rest, but it turns out that common_bits is already super fast on + * modern processors, even taking into account the unfortunate bswap. + * So, we just inline it like this instead. + */ + return CommonBits(Node, Key, Bits) >= Node->Cidr; +} + +_Requires_rcu_held_ +_Must_inspect_result_ +_Post_maybenull_ +static ALLOWEDIPS_NODE * +FindNode(_In_ ALLOWEDIPS_NODE *Trie, _In_ UINT8 Bits, _In_reads_bytes_(Bits / 8) CONST UINT8 *Key) +{ + ALLOWEDIPS_NODE *Node = Trie, *Found = NULL; + + while (Node && PrefixMatches(Node, Key, Bits)) + { + if (RcuAccessPointer(Node->Peer)) + Found = Node; + if (Node->Cidr == Bits) + break; + Node = RcuDereference(ALLOWEDIPS_NODE, Node->Bit[Choose(Node, Key)]); + } + return Found; +} + +/* Returns a strong reference to a peer */ +_IRQL_requires_max_(DISPATCH_LEVEL) +_Must_inspect_result_ +_Post_maybenull_ +static WG_PEER * +Lookup(_In_ ALLOWEDIPS_NODE __rcu *Root, _In_ UINT8 Bits, _In_reads_bytes_(Bits / 8) CONST VOID *BeIp) +{ + /* Aligned so it can be passed to FindLastSet/FindLastSet64 */ + __declspec(align(8)) UINT8 Ip[16]; + ALLOWEDIPS_NODE *Node; + WG_PEER *Peer = NULL; + KIRQL Irql; + + SwapEndian(Ip, BeIp, Bits); + + Irql = RcuReadLock(); +retry: + Node = FindNode(RcuDereference(ALLOWEDIPS_NODE, Root), Bits, Ip); + if (Node) + { + Peer = PeerGetMaybeZero(RcuDereference(WG_PEER, Node->Peer)); + if (!Peer) + goto retry; + } + RcuReadUnlock(Irql); + return Peer; +} + +_Requires_lock_held_(Lock) +static BOOLEAN +NodePlacement( + _In_ ALLOWEDIPS_NODE __rcu *Trie, + _In_reads_bytes_(Bits / 8) CONST UINT8 *Key, + _In_ UINT8 Cidr, + _In_ UINT8 Bits, + _Out_ ALLOWEDIPS_NODE **Rnode, + _In_ EX_PUSH_LOCK *Lock) +{ + ALLOWEDIPS_NODE *Node = RcuDereferenceProtected(ALLOWEDIPS_NODE, Trie, Lock); + ALLOWEDIPS_NODE *Parent = NULL; + BOOLEAN Exact = FALSE; + + while (Node && Node->Cidr <= Cidr && PrefixMatches(Node, Key, Bits)) + { + Parent = Node; + if (Parent->Cidr == Cidr) + { + Exact = TRUE; + break; + } + Node = RcuDereferenceProtected(ALLOWEDIPS_NODE, Parent->Bit[Choose(Parent, Key)], Lock); + } + *Rnode = Parent; + return Exact; +} + +static inline VOID +ConnectNode(_Inout_ ALLOWEDIPS_NODE __rcu **Parent, _In_ UINT8 Bit, _In_ __drv_aliasesMem ALLOWEDIPS_NODE *Node) +{ + Node->ParentBitPacked = (ULONG_PTR)Parent | Bit; + RcuAssignPointer(*Parent, Node); +} + +static inline VOID +ChooseAndConnectNode(_Inout_ ALLOWEDIPS_NODE *Parent, _In_ __drv_aliasesMem ALLOWEDIPS_NODE *Node) +{ + UINT8 Bit = Choose(Parent, Node->Bits); + ConnectNode(&Parent->Bit[Bit], Bit, Node); +} + +_Requires_lock_held_(Lock) +static NTSTATUS +Add(_Inout_ ALLOWEDIPS_NODE __rcu **Trie, + _In_ UINT8 Bits, + _In_ CONST UINT8 *Key, + _In_ UINT8 Cidr, + _In_ WG_PEER *Peer, + _In_ EX_PUSH_LOCK *Lock) +{ + ALLOWEDIPS_NODE *Node, *Parent, *Down, *Newnode; + + if (Cidr > Bits || !Peer) + return STATUS_INVALID_PARAMETER; + + if (!RcuAccessPointer(*Trie)) + { + Node = ExAllocateFromLookasideListEx(&NodeCache); + if (!Node) + return STATUS_INSUFFICIENT_RESOURCES; + RtlZeroMemory(Node, sizeof(*Node)); + RcuInitPointer(Node->Peer, Peer); + InsertTailList(&Peer->AllowedIpsList, &Node->PeerList); + CopyAndAssignCidr(Node, Key, Cidr, Bits); + ConnectNode(Trie, 2, Node); + return STATUS_SUCCESS; + } + if (NodePlacement(*Trie, Key, Cidr, Bits, &Node, Lock)) + { + RcuAssignPointer(Node->Peer, Peer); + RemoveEntryList(&Node->PeerList); + InsertTailList(&Peer->AllowedIpsList, &Node->PeerList); + return STATUS_SUCCESS; + } + + Newnode = ExAllocateFromLookasideListEx(&NodeCache); + if (!Newnode) + return STATUS_INSUFFICIENT_RESOURCES; + RtlZeroMemory(Newnode, sizeof(*Newnode)); + RcuInitPointer(Newnode->Peer, Peer); + InsertTailList(&Peer->AllowedIpsList, &Newnode->PeerList); + CopyAndAssignCidr(Newnode, Key, Cidr, Bits); + + if (!Node) + { + Down = RcuDereferenceProtected(ALLOWEDIPS_NODE, *Trie, Lock); + } + else + { + CONST UINT8 Bit = Choose(Node, Key); + Down = RcuDereferenceProtected(ALLOWEDIPS_NODE, Node->Bit[Bit], Lock); + if (!Down) + { + ConnectNode(&Node->Bit[Bit], Bit, Newnode); + return STATUS_SUCCESS; + } + } + Cidr = min(Cidr, CommonBits(Down, Key, Bits)); + Parent = Node; + + if (Newnode->Cidr == Cidr) + { + ChooseAndConnectNode(Newnode, Down); + if (!Parent) + ConnectNode(Trie, 2, Newnode); + else + ChooseAndConnectNode(Parent, Newnode); + return 0; + } + + Node = ExAllocateFromLookasideListEx(&NodeCache); + if (!Node) + { + RemoveEntryList(&Newnode->PeerList); + ExFreeToLookasideListEx(&NodeCache, Newnode); + return STATUS_INSUFFICIENT_RESOURCES; + } + RtlZeroMemory(Node, sizeof(*Node)); + InitializeListHead(&Node->PeerList); + CopyAndAssignCidr(Node, Newnode->Bits, Cidr, Bits); + + ChooseAndConnectNode(Node, Down); + ChooseAndConnectNode(Node, Newnode); + if (!Parent) + ConnectNode(Trie, 2, Node); + else + ChooseAndConnectNode(Parent, Node); + return STATUS_SUCCESS; +} + +_Use_decl_annotations_ +VOID +AllowedIpsInit(ALLOWEDIPS_TABLE *Table) +{ + Table->Root4 = Table->Root6 = NULL; + Table->Seq = 1; +} + +_Use_decl_annotations_ +VOID +AllowedIpsFree(ALLOWEDIPS_TABLE *Table, EX_PUSH_LOCK *Lock) +{ + ALLOWEDIPS_NODE *Old4 = RcuDereferenceProtected(ALLOWEDIPS_NODE, Table->Root4, Lock); + ALLOWEDIPS_NODE *Old6 = RcuDereferenceProtected(ALLOWEDIPS_NODE, Table->Root6, Lock); + + ++Table->Seq; + RcuInitPointer(Table->Root4, NULL); + RcuInitPointer(Table->Root6, NULL); + if (Old4) + { + RootRemovePeerLists(Old4); + RcuCall(&Old4->Rcu, RootFreeRcu); + } + if (Old6) + { + RootRemovePeerLists(Old6); + RcuCall(&Old6->Rcu, RootFreeRcu); + } +} + +_Use_decl_annotations_ +NTSTATUS +AllowedIpsInsertV4(ALLOWEDIPS_TABLE *Table, CONST IN_ADDR *Ip, UINT8 Cidr, WG_PEER *Peer, EX_PUSH_LOCK *Lock) +{ + /* Aligned so it can be passed to FindLastSet */ + __declspec(align(4)) UINT8 Key[4]; + + ++Table->Seq; + SwapEndian(Key, (CONST UINT8 *)Ip, 32); + return Add(&Table->Root4, 32, Key, Cidr, Peer, Lock); +} + +_Use_decl_annotations_ +NTSTATUS +AllowedIpsInsertV6(ALLOWEDIPS_TABLE *Table, CONST IN6_ADDR *Ip, UINT8 Cidr, WG_PEER *Peer, EX_PUSH_LOCK *Lock) +{ + /* Aligned so it can be passed to FindLastSet64 */ + __declspec(align(8)) UINT8 Key[16]; + + ++Table->Seq; + SwapEndian(Key, (CONST UINT8 *)Ip, 128); + return Add(&Table->Root6, 128, Key, Cidr, Peer, Lock); +} + +_Use_decl_annotations_ +VOID +AllowedIpsRemoveByPeer(ALLOWEDIPS_TABLE *Table, WG_PEER *Peer, EX_PUSH_LOCK *Lock) +{ + ALLOWEDIPS_NODE *Node, *Child, **ParentBit, *Parent, *Tmp; + BOOLEAN FreeParent; + + if (IsListEmpty(&Peer->AllowedIpsList)) + return; + ++Table->Seq; + LIST_FOR_EACH_ENTRY_SAFE (Node, Tmp, &Peer->AllowedIpsList, ALLOWEDIPS_NODE, PeerList) + { + RemoveEntryList(&Node->PeerList); + InitializeListHead(&Node->PeerList); + RcuInitPointer(Node->Peer, NULL); + if (Node->Bit[0] && Node->Bit[1]) + continue; + Child = RcuDereferenceProtected(ALLOWEDIPS_NODE, Node->Bit[!RcuAccessPointer(Node->Bit[0])], Lock); + if (Child) + Child->ParentBitPacked = Node->ParentBitPacked; + ParentBit = (ALLOWEDIPS_NODE **)(Node->ParentBitPacked & ~(ULONG_PTR)3); + *ParentBit = Child; + Parent = + (ALLOWEDIPS_NODE *)((UCHAR *)ParentBit - FIELD_OFFSET(ALLOWEDIPS_NODE, Bit[Node->ParentBitPacked & 1])); + FreeParent = !RcuAccessPointer(Node->Bit[0]) && !RcuAccessPointer(Node->Bit[1]) && + (Node->ParentBitPacked & 3) <= 1 && !RcuAccessPointer(Parent->Peer); + if (FreeParent) + Child = RcuDereferenceProtected(ALLOWEDIPS_NODE, Parent->Bit[!(Node->ParentBitPacked & 1)], Lock); + RcuCall(&Node->Rcu, NodeFreeRcu); + if (!FreeParent) + continue; + if (Child) + Child->ParentBitPacked = Parent->ParentBitPacked; + *(ALLOWEDIPS_NODE **)(Parent->ParentBitPacked & ~(ULONG_PTR)3) = Child; + RcuCall(&Parent->Rcu, NodeFreeRcu); + } +} + +_Use_decl_annotations_ +ADDRESS_FAMILY +AllowedIpsReadNode(CONST ALLOWEDIPS_NODE *Node, UINT8 Ip[16], UINT8 *Cidr) +{ + CONST ULONG CidrBytes = DIV_ROUND_UP(Node->Cidr, 8U); + SwapEndian(Ip, Node->Bits, Node->Bitlen); + RtlZeroMemory(Ip + CidrBytes, Node->Bitlen / 8U - CidrBytes); + if (Node->Cidr) + Ip[CidrBytes - 1U] &= ~0U << (-Node->Cidr % 8U); + + *Cidr = Node->Cidr; + return Node->Bitlen == 32 ? AF_INET : AF_INET6; +} + +/* Returns a strong reference to a peer */ +_Use_decl_annotations_ +WG_PEER * +AllowedIpsLookupDst(ALLOWEDIPS_TABLE *Table, UINT16_BE Proto, CONST VOID *IpHdr) +{ + if (Proto == Htons(NDIS_ETH_TYPE_IPV4)) + return Lookup(Table->Root4, 32, &((IPV4HDR *)IpHdr)->Daddr); + else if (Proto == Htons(NDIS_ETH_TYPE_IPV6)) + return Lookup(Table->Root6, 128, &((IPV6HDR *)IpHdr)->Daddr); + return NULL; +} + +/* Returns a strong reference to a peer */ +_Use_decl_annotations_ +WG_PEER * +AllowedIpsLookupSrc(ALLOWEDIPS_TABLE *Table, UINT16_BE Proto, CONST VOID *IpHdr) +{ + if (Proto == Htons(NDIS_ETH_TYPE_IPV4)) + return Lookup(Table->Root4, 32, &((IPV4HDR *)IpHdr)->Saddr); + else if (Proto == Htons(NDIS_ETH_TYPE_IPV6)) + return Lookup(Table->Root6, 128, &((IPV6HDR *)IpHdr)->Saddr); + return NULL; +} + +#ifdef ALLOC_PRAGMA +# pragma alloc_text(INIT, AllowedIpsDriverEntry) +#endif +_Use_decl_annotations_ +NTSTATUS +AllowedIpsDriverEntry(VOID) +{ + return ExInitializeLookasideListEx(&NodeCache, NULL, NULL, NonPagedPool, 0, sizeof(ALLOWEDIPS_NODE), MEMORY_TAG, 0); +} + +_Use_decl_annotations_ +VOID AllowedIpsUnload(VOID) +{ + RcuBarrier(); + ExDeleteLookasideListEx(&NodeCache); +} + +#ifdef DBG +# include "selftest/allowedips.c" +#endif diff --git a/driver/allowedips.h b/driver/allowedips.h new file mode 100644 index 0000000..0e17b92 --- /dev/null +++ b/driver/allowedips.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +#include "rcu.h" +#include "arithmetic.h" +#include <ntifs.h> /* Must be included before <wdm.h> */ +#include <wdm.h> +#include <wsk.h> + +typedef struct _WG_PEER WG_PEER; + +typedef struct _ALLOWEDIPS_NODE ALLOWEDIPS_NODE; +struct _ALLOWEDIPS_NODE +{ + WG_PEER __rcu *Peer; + ALLOWEDIPS_NODE __rcu *Bit[2]; + UINT8 Cidr, BitAtA, BitAtB, Bitlen; + __declspec(align(8)) UINT8 Bits[16]; + + /* Keep rarely used members at bottom to be beyond cache line. */ + ULONG_PTR ParentBitPacked; + union + { + LIST_ENTRY PeerList; + RCU_CALLBACK Rcu; + }; +}; + +typedef __declspec(align(4)) struct _ALLOWEDIPS_TABLE +{ + ALLOWEDIPS_NODE __rcu *Root4; + ALLOWEDIPS_NODE __rcu *Root6; + UINT64 Seq; +} ALLOWEDIPS_TABLE; + +VOID +AllowedIpsInit(_Out_ ALLOWEDIPS_TABLE *Table); + +_Requires_lock_held_(Lock) +VOID +AllowedIpsFree(_Inout_ ALLOWEDIPS_TABLE *Table, _In_ EX_PUSH_LOCK *Lock); + +_Requires_lock_held_(Lock) +NTSTATUS +AllowedIpsInsertV4( + _Inout_ ALLOWEDIPS_TABLE *Table, + _In_ CONST IN_ADDR *Ip, + _In_ UINT8 Cidr, + _In_ WG_PEER *Peer, + _In_ EX_PUSH_LOCK *Lock); + +_Requires_lock_held_(Lock) +NTSTATUS +AllowedIpsInsertV6( + _Inout_ ALLOWEDIPS_TABLE *Table, + _In_ CONST IN6_ADDR *Ip, + _In_ UINT8 Cidr, + _In_ WG_PEER *Peer, + _In_ EX_PUSH_LOCK *Lock); + +_Requires_lock_held_(Lock) +VOID +AllowedIpsRemoveByPeer(_Inout_ ALLOWEDIPS_TABLE *Table, _In_ WG_PEER *Peer, _In_ EX_PUSH_LOCK *Lock); + +/* The Ip pointer should be 8 byte aligned */ +ADDRESS_FAMILY +AllowedIpsReadNode(_In_ CONST ALLOWEDIPS_NODE *Node, _Out_ UINT8 Ip[16], _Out_ UINT8 *Cidr); + +/* These return a strong reference to a peer: */ +_Must_inspect_result_ +_Post_maybenull_ +WG_PEER * +AllowedIpsLookupDst(_In_ ALLOWEDIPS_TABLE *Table, _In_ UINT16_BE Proto, _In_ CONST VOID *IpHdr); + +_Must_inspect_result_ +_Post_maybenull_ +WG_PEER * +AllowedIpsLookupSrc(_In_ ALLOWEDIPS_TABLE *Table, _In_ UINT16_BE Proto, _In_ CONST VOID *IpHdr); + +#ifdef DBG +_IRQL_requires_max_(PASSIVE_LEVEL) +BOOLEAN +AllowedIpsSelftest(VOID); +#endif + +_IRQL_requires_max_(DISPATCH_LEVEL) +NTSTATUS +AllowedIpsDriverEntry(VOID); + +_IRQL_requires_max_(APC_LEVEL) +VOID AllowedIpsUnload(VOID); diff --git a/driver/arithmetic.h b/driver/arithmetic.h new file mode 100644 index 0000000..4a7198f --- /dev/null +++ b/driver/arithmetic.h @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +#include <ntifs.h> /* Must be included before <wdm.h> */ +#include <wdm.h> +#include <ndis.h> +#include <ntintsafe.h> + +typedef _Strict_type_match_ UINT16 UINT16_BE; +typedef _Strict_type_match_ UINT16 UINT16_LE; +typedef _Strict_type_match_ UINT32 UINT32_BE; +typedef _Strict_type_match_ UINT32 UINT32_LE; +typedef _Strict_type_match_ UINT64 UINT64_BE; +typedef _Strict_type_match_ UINT64 UINT64_LE; + +#if REG_DWORD == REG_DWORD_BIG_ENDIAN +# define Le16ToCpu(X) ((UINT16)RtlUshortByteSwap(X)) +# define Le32ToCpu(X) ((UINT32)RtlUlongByteSwap(X)) +# define Le64ToCpu(X) ((UINT64)RtlUlonglongByteSwap(X)) +# define Be16ToCpu(X) ((UINT16)(X)) +# define Be32ToCpu(X) ((UINT32)(X)) +# define Be64ToCpu(X) ((UINT64)(X)) +# define CpuToLe16(X) ((UINT16_LE)RtlUshortByteSwap(X)) +# define CpuToLe32(X) ((UINT32_LE)RtlUlongByteSwap(X)) +# define CpuToLe64(X) ((UINT64_LE)RtlUlonglongByteSwap(X)) +# define CpuToBe16(X) ((UINT16_BE)(X)) +# define CpuToBe32(X) ((UINT32_BE)(X)) +# define CpuToBe64(X) ((UINT64_BE)(X)) +#elif REG_DWORD == REG_DWORD_LITTLE_ENDIAN +# define Be16ToCpu(X) ((UINT16)RtlUshortByteSwap(X)) +# define Be32ToCpu(X) ((UINT32)RtlUlongByteSwap(X)) +# define Be64ToCpu(X) ((UINT64)RtlUlonglongByteSwap(X)) +# define Le16ToCpu(X) ((UINT16)(X)) +# define Le32ToCpu(X) ((UINT32)(X)) +# define Le64ToCpu(X) ((UINT64)(X)) +# define CpuToBe16(X) ((UINT16_BE)RtlUshortByteSwap(X)) +# define CpuToBe32(X) ((UINT32_BE)RtlUlongByteSwap(X)) +# define CpuToBe64(X) ((UINT64_BE)RtlUlonglongByteSwap(X)) +# define CpuToLe16(X) ((UINT16_LE)(X)) +# define CpuToLe32(X) ((UINT32_LE)(X)) +# define CpuToLe64(X) ((UINT64_LE)(X)) +#else +# error "Unable to determine endianess" +#endif + +#define Ntohs(X) Be16ToCpu(X) +#define Ntohl(X) Be32ToCpu(X) +#define Htons(X) CpuToBe16(X) +#define Htonl(X) CpuToBe32(X) + +#ifdef _WIN64 +# define BITS_PER_POINTER 64 +# define BITS_PER_POINTER_SHIFT 6 +#else +# define BITS_PER_POINTER 32 +# define BITS_PER_POINTER_SHIFT 5 +#endif + +static inline ULONG +FindLastSet32(_In_ UINT32 Word) +{ + ULONG Index; + return BitScanReverse(&Index, Word) ? Index + 1 : 0; +} + +static inline ULONG +FindLastSet64(_In_ UINT64 Word) +{ +#ifndef BitScanReverse64 + UINT32 H = Word >> 32; + return H ? FindLastSet32(H) + 32 : FindLastSet32((UINT32)Word); +#else + ULONG Index; + return BitScanReverse64(&Index, Word) ? Index + 1 : 0; +#endif +} + +static inline ULONG +FindLastSet128(_In_ UINT64 A, _In_ UINT64 B) +{ + return A ? FindLastSet64(A) + 64U : FindLastSet64(B); +} + +static inline ULONG_PTR +RounddownPowOfTwo(_In_ ULONG_PTR N) +{ + N = N | (N >> 1); + N = N | (N >> 2); + N = N | (N >> 4); + N = N | (N >> 8); + N = N | (N >> 16); +#ifdef _WIN64 + N = N | (N >> 32); +#endif + return N - (N >> 1); +} + +#define DIV_ROUND_UP(N, D) (((N) + (D)-1) / (D)) +#define ALIGN_DOWN_BY_T(T, Length, Alignment) ((T)(Length) & ~((T)(Alignment)-1)) +#define ALIGN_UP_BY_T(T, Length, Alignment) (ALIGN_DOWN_BY_T(T, ((T)(Length) + (Alignment)-1), Alignment))
\ No newline at end of file diff --git a/driver/containers.h b/driver/containers.h new file mode 100644 index 0000000..8a030d0 --- /dev/null +++ b/driver/containers.h @@ -0,0 +1,386 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +#include "interlocked.h" +#include "memory.h" +#include "rcu.h" +#include <ntifs.h> /* Must be included before <wdm.h> */ +#include <wdm.h> +#include <ndis.h> + +#define LIST_FOR_EACH_ENTRY(Pos, Head, Type, Member) \ + for (Pos = CONTAINING_RECORD((Head)->Flink, Type, Member); &Pos->Member != (Head); \ + Pos = CONTAINING_RECORD((Pos)->Member.Flink, Type, Member)) +#define LIST_FOR_EACH_ENTRY_SAFE(Pos, Tmp, Head, Type, Member) \ + for (Pos = CONTAINING_RECORD((Head)->Flink, Type, Member), \ + Tmp = CONTAINING_RECORD((Pos)->Member.Flink, Type, Member); \ + &Pos->Member != (Head); \ + Pos = Tmp, Tmp = CONTAINING_RECORD((Tmp)->Member.Flink, Type, Member)) + +typedef struct _HLIST_NODE HLIST_NODE; +struct _HLIST_NODE +{ + HLIST_NODE *Next, **Pprev; +}; +typedef struct _HLIST_HEAD +{ + HLIST_NODE *First; +} HLIST_HEAD; + +static inline VOID +HlistHeadInit(_Out_ HLIST_HEAD *H) +{ + H->First = NULL; +} + +static inline VOID +HlistInit(_Out_ HLIST_NODE *H) +{ + H->Next = NULL; + H->Pprev = NULL; +} + +_Must_inspect_result_ +static inline BOOLEAN +HlistUnhashed(_In_ CONST HLIST_NODE *Head) +{ + return !Head->Pprev; +} + +static inline VOID +__HlistDel(_Inout_ HLIST_NODE *Node) +{ + HLIST_NODE *Next = Node->Next; + HLIST_NODE **Pprev = Node->Pprev; + + WritePointerNoFence(Pprev, Next); + if (Next) + WritePointerNoFence(&Next->Pprev, Pprev); +} + +static inline VOID +HlistDelInitRcu(_Inout_ HLIST_NODE *Node) +{ + if (!HlistUnhashed(Node)) + { + __HlistDel(Node); + WritePointerNoFence(&Node->Pprev, NULL); + } +} + +static inline VOID +HlistDelRcu(_Inout_ HLIST_NODE *Node) +{ + __HlistDel(Node); + WritePointerNoFence(&Node->Pprev, NULL); +} + +static inline VOID +HlistReplaceRcu(_Inout_ HLIST_NODE *Old, _Out_ HLIST_NODE *New) +{ + HLIST_NODE *Next = Old->Next; + + New->Next = Next; + WritePointerNoFence(&New->Pprev, Old->Pprev); + RcuAssignPointer(*(HLIST_NODE __rcu **)New->Pprev, New); + if (Next) + WritePointerNoFence(&New->Next->Pprev, &New->Next); + WritePointerNoFence(&Old->Pprev, NULL); +} + +#define HlistFirstRcu(Head) (*((HLIST_NODE __rcu **)(&(Head)->First))) +#define HlistNextRcu(Node) (*((HLIST_NODE __rcu **)(&(Node)->Next))) + +static inline VOID +HlistAddHeadRcu(_Inout_ __drv_aliasesMem HLIST_NODE *Node, _Inout_ HLIST_HEAD *Head) +{ + HLIST_NODE *First = Head->First; + + Node->Next = First; + WritePointerNoFence(&Node->Pprev, &Head->First); + RcuAssignPointer(HlistFirstRcu(Head), Node); + if (First) + WritePointerNoFence(&First->Pprev, &Node->Next); +} + +#define HlistEntry(Ptr, Type, Member) CONTAINING_RECORD(Ptr, Type, Member) +#define HlistEntrySafe(Ptr, Type, Member) ((Ptr) ? HlistEntry(Ptr, Type, Member) : NULL) +#define HLIST_FOR_EACH_ENTRY_SAFE(Pos, Tmp, Head, Type, Member) \ + for (Pos = HlistEntrySafe((Head)->First, Type, Member); Pos && (Tmp = Pos->Member.Next, 1); \ + Pos = HlistEntrySafe(Tmp, Type, Member)) + +#define HLIST_FOR_EACH_ENTRY_RCU(Pos, Head, Type, Member) \ + for (Pos = HlistEntrySafe(RcuDereference(Type, HlistFirstRcu(Head)), Type, Member); Pos; \ + Pos = HlistEntrySafe(RcuDereference(Type, HlistNextRcu(&(Pos)->Member)), Type, Member)) + +#define DECLARE_HASHTABLE(Name, Bits) HLIST_HEAD Name[1 << (Bits)] +#define HASH_SIZE(Name) (ARRAYSIZE(Name)) + +static inline VOID +__HashInit(_Out_writes_bytes_all_(Sz) HLIST_HEAD *Ht, _In_ SIZE_T Sz) +{ + for (SIZE_T i = 0; i < Sz; ++i) + HlistHeadInit(&Ht[i]); +} + +#define HashInit(Hashtable) __HashInit(Hashtable, HASH_SIZE(Hashtable)) + +typedef struct _PTR_RING +{ + DECLSPEC_CACHEALIGN LONG Producer; + KSPIN_LOCK ProducerLock; + DECLSPEC_CACHEALIGN LONG ConsumerHead; + LONG ConsumerTail; + KSPIN_LOCK ConsumerLock; + DECLSPEC_CACHEALIGN LONG Size; + LONG Batch; + VOID **Queue; +} PTR_RING; + +_Requires_lock_held_(Ring->ProducerLock) +_Must_inspect_result_ +static inline NTSTATUS +__PtrRingProduce(_Inout_ PTR_RING *Ring, _In_ __drv_aliasesMem VOID *Ptr) +{ + if (!Ring->Size || Ring->Queue[Ring->Producer]) + return STATUS_BUFFER_TOO_SMALL; + + WriteMemoryBarrier(); + + WritePointerNoFence(&Ring->Queue[Ring->Producer++], Ptr); + if (Ring->Producer >= Ring->Size) + Ring->Producer = 0; + return STATUS_SUCCESS; +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Requires_lock_not_held_(Ring->ProducerLock) +_Must_inspect_result_ +static inline NTSTATUS +PtrRingProduce(_Inout_ PTR_RING *Ring, _In_ __drv_aliasesMem VOID *Ptr) +{ + KIRQL Irql; + NTSTATUS Ret; + + KeAcquireSpinLock(&Ring->ProducerLock, &Irql); + Ret = __PtrRingProduce(Ring, Ptr); + KeReleaseSpinLock(&Ring->ProducerLock, Irql); + + return Ret; +} + +_Requires_lock_held_(Ring->ConsumerLock) +_Must_inspect_result_ +_Post_maybenull_ +static inline VOID * +__PtrRingPeek(_In_ CONST PTR_RING *Ring) +{ + if (Ring->Size) + return ReadPointerNoFence(&Ring->Queue[Ring->ConsumerHead]); + return NULL; +} + +_Requires_lock_held_(Ring->ConsumerLock) +static inline VOID +__PtrRingDiscardOne(_Inout_ PTR_RING *Ring) +{ + LONG ConsumerHead = Ring->ConsumerHead; + LONG Head = ConsumerHead++; + + if (ConsumerHead - Ring->ConsumerTail >= Ring->Batch || ConsumerHead >= Ring->Size) + { + while (Head >= Ring->ConsumerTail) + Ring->Queue[Head--] = NULL; + Ring->ConsumerTail = ConsumerHead; + } + if (ConsumerHead >= Ring->Size) + { + ConsumerHead = 0; + Ring->ConsumerTail = 0; + } + WriteNoFence(&Ring->ConsumerHead, ConsumerHead); +} + +_Requires_lock_held_(Ring->ConsumerLock) +_Must_inspect_result_ +_Post_maybenull_ +static inline VOID * +__PtrRingConsume(_Inout_ PTR_RING *Ring) +{ + VOID *Ptr; + + Ptr = __PtrRingPeek(Ring); + if (Ptr) + __PtrRingDiscardOne(Ring); + + return Ptr; +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Requires_lock_not_held_(Ring->ConsumerLock) +_Must_inspect_result_ +_Post_maybenull_ +static inline VOID * +PtrRingConsume(_Inout_ PTR_RING *Ring) +{ + KIRQL Irql; + VOID *Ptr; + + KeAcquireSpinLock(&Ring->ConsumerLock, &Irql); + Ptr = __PtrRingConsume(Ring); + KeReleaseSpinLock(&Ring->ConsumerLock, Irql); + + return Ptr; +} + +static inline VOID +__PtrRingSetSize(_Inout_ PTR_RING *Ring, _In_ LONG Size) +{ + Ring->Size = Size; + Ring->Batch = SYSTEM_CACHE_ALIGNMENT_SIZE * 2 / sizeof(*(Ring->Queue)); + if (Ring->Batch > Ring->Size / 2 || !Ring->Batch) + Ring->Batch = 1; +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Must_inspect_result_ +static inline NTSTATUS +PtrRingInit(_Inout_ PTR_RING *Ring, _In_ LONG Size) +{ + Ring->Queue = MemAllocateArrayAndZero(Size, sizeof(VOID *)); + if (!Ring->Queue) + return STATUS_INSUFFICIENT_RESOURCES; + + __PtrRingSetSize(Ring, Size); + Ring->Producer = Ring->ConsumerHead = Ring->ConsumerTail = 0; + KeInitializeSpinLock(&Ring->ProducerLock); + KeInitializeSpinLock(&Ring->ConsumerLock); + + return STATUS_SUCCESS; +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +_When_(Destroy != 0, _Requires_lock_not_held_(Ring->ConsumerLock)) +static inline VOID +PtrRingCleanup(_In_ PTR_RING *Ring, _In_opt_ VOID (*Destroy)(VOID *)) +{ + VOID *Ptr; + + if (Destroy) + while ((Ptr = PtrRingConsume(Ring)) != NULL) + Destroy(Ptr); + MemFree(Ring->Queue); +} + +typedef struct _NET_BUFFER_LIST_QUEUE +{ + PNET_BUFFER_LIST Head, Tail; + ULONG Length; + KSPIN_LOCK Lock; +} NET_BUFFER_LIST_QUEUE; + +static inline VOID +NetBufferListInitQueue(_Out_ NET_BUFFER_LIST_QUEUE *NblQueue) +{ + NblQueue->Head = NblQueue->Tail = NULL; + NblQueue->Length = 0; + KeInitializeSpinLock(&NblQueue->Lock); +} + +_Must_inspect_result_ +static inline BOOLEAN +NetBufferListIsQueueEmpty(_In_ CONST NET_BUFFER_LIST_QUEUE *NblQueue) +{ + return !NblQueue->Head; +} + +static inline ULONG +NetBufferListQueueLength(_In_ CONST NET_BUFFER_LIST_QUEUE *NblQueue) +{ + return NblQueue->Length; +} + +_Requires_lock_held_(NblQueue->Lock) +_Requires_lock_held_(Head->Lock) +static inline VOID +NetBufferListSpliceAndReinitQueue(_Inout_ NET_BUFFER_LIST_QUEUE *NblQueue, _Inout_ NET_BUFFER_LIST_QUEUE *Head) +{ + if (!NetBufferListIsQueueEmpty(NblQueue)) + { + if (!Head->Tail) + Head->Tail = NblQueue->Tail; + NET_BUFFER_LIST_NEXT_NBL(NblQueue->Tail) = Head->Head; + Head->Head = NblQueue->Head; + Head->Length += NblQueue->Length; + NblQueue->Head = NblQueue->Tail = NULL; + NblQueue->Length = 0; + } +} + +_Requires_lock_held_(Head->Lock) +static inline VOID +NetBufferListSpliceTail(_In_ CONST NET_BUFFER_LIST_QUEUE *NblQueue, _Inout_ NET_BUFFER_LIST_QUEUE *Head) +{ + if (!NetBufferListIsQueueEmpty(NblQueue)) + { + *(Head->Tail ? &NET_BUFFER_LIST_NEXT_NBL(Head->Tail) : &Head->Head) = NblQueue->Head; + Head->Tail = NblQueue->Tail; + Head->Length += NblQueue->Length; + } +} + +_Requires_lock_held_(NblQueue->Lock) +static inline VOID +NetBufferListEnqueue(_Inout_ NET_BUFFER_LIST_QUEUE *NblQueue, __drv_aliasesMem _In_ PNET_BUFFER_LIST Nbl) +{ + NET_BUFFER_LIST_NEXT_NBL(Nbl) = NULL; + *(NblQueue->Tail ? &NET_BUFFER_LIST_NEXT_NBL(NblQueue->Tail) : &NblQueue->Head) = Nbl; + NblQueue->Tail = Nbl; + ++NblQueue->Length; +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Requires_lock_not_held_(NblQueue->Lock) +static inline VOID +NetBufferListInterlockedEnqueue(_Inout_ NET_BUFFER_LIST_QUEUE *NblQueue, __drv_aliasesMem _In_ PNET_BUFFER_LIST Nbl) +{ + KIRQL Irql; + KeAcquireSpinLock(&NblQueue->Lock, &Irql); + NetBufferListEnqueue(NblQueue, Nbl); + KeReleaseSpinLock(&NblQueue->Lock, Irql); +} + +_Requires_lock_held_(NblQueue->Lock) +_Must_inspect_result_ +_Post_maybenull_ +static inline PNET_BUFFER_LIST +NetBufferListDequeue(_Inout_ NET_BUFFER_LIST_QUEUE *NblQueue) +{ + PNET_BUFFER_LIST Nbl = NblQueue->Head; + if (!Nbl) + return NULL; + NblQueue->Head = NET_BUFFER_LIST_NEXT_NBL(Nbl); + NET_BUFFER_LIST_NEXT_NBL(Nbl) = NULL; + if (!NblQueue->Head) + NblQueue->Tail = NULL; + NblQueue->Length--; + return Nbl; +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Requires_lock_not_held_(NblQueue->Lock) +_Must_inspect_result_ +_Post_maybenull_ +static inline PNET_BUFFER_LIST +NetBufferListInterlockedDequeue(NET_BUFFER_LIST_QUEUE *NblQueue) +{ + KIRQL Irql; + KeAcquireSpinLock(&NblQueue->Lock, &Irql); + PNET_BUFFER_LIST Nbl = NetBufferListDequeue(NblQueue); + KeReleaseSpinLock(&NblQueue->Lock, Irql); + return Nbl; +} diff --git a/driver/cookie.c b/driver/cookie.c new file mode 100644 index 0000000..6fb4c58 --- /dev/null +++ b/driver/cookie.c @@ -0,0 +1,259 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include "cookie.h" +#include "peer.h" +#include "device.h" +#include "messages.h" +#include "ratelimiter.h" +#include "timers.h" +#include "crypto.h" +#include "queueing.h" +#include "logging.h" + +#pragma warning(disable : 4295) /* array is too small to include a terminating null character */ + +_Use_decl_annotations_ +VOID +CookieCheckerInit(COOKIE_CHECKER *Checker, WG_DEVICE *Wg) +{ + MuInitializePushLock(&Checker->SecretLock); + Checker->SecretBirthdate = KeQueryInterruptTime(); + CryptoRandom(Checker->Secret, NOISE_HASH_LEN); + Checker->Device = Wg; +} + +enum +{ + COOKIE_KEY_LABEL_LEN = 8 +}; +static CONST UINT8 Mac1KeyLabel[COOKIE_KEY_LABEL_LEN] = "mac1----"; +static CONST UINT8 CookieKeyLabel[COOKIE_KEY_LABEL_LEN] = "cookie--"; + +static VOID +PrecomputeKey( + _Out_writes_bytes_all_(NOISE_SYMMETRIC_KEY_LEN) UINT8 Key[NOISE_SYMMETRIC_KEY_LEN], + _In_reads_bytes_(NOISE_PUBLIC_KEY_LEN) CONST UINT8 Pubkey[NOISE_PUBLIC_KEY_LEN], + _In_reads_bytes_(COOKIE_KEY_LABEL_LEN) CONST UINT8 Label[COOKIE_KEY_LABEL_LEN]) +{ + BLAKE2S_STATE Blake; + + Blake2sInit(&Blake, NOISE_SYMMETRIC_KEY_LEN); + Blake2sUpdate(&Blake, Label, COOKIE_KEY_LABEL_LEN); + Blake2sUpdate(&Blake, Pubkey, NOISE_PUBLIC_KEY_LEN); + Blake2sFinal(&Blake, Key); +} + +_Use_decl_annotations_ +VOID +CookieCheckerPrecomputeDeviceKeys(COOKIE_CHECKER *Checker) +{ + if (Checker->Device->StaticIdentity.HasIdentity) + { + PrecomputeKey(Checker->CookieEncryptionKey, Checker->Device->StaticIdentity.StaticPublic, CookieKeyLabel); + PrecomputeKey(Checker->MessageMac1Key, Checker->Device->StaticIdentity.StaticPublic, Mac1KeyLabel); + } + else + { + RtlZeroMemory(Checker->CookieEncryptionKey, NOISE_SYMMETRIC_KEY_LEN); + RtlZeroMemory(Checker->MessageMac1Key, NOISE_SYMMETRIC_KEY_LEN); + } +} + +_Use_decl_annotations_ +VOID +CookieCheckerPrecomputePeerKeys(WG_PEER *Peer) +{ + PrecomputeKey(Peer->LatestCookie.CookieDecryptionKey, Peer->Handshake.RemoteStatic, CookieKeyLabel); + PrecomputeKey(Peer->LatestCookie.MessageMac1Key, Peer->Handshake.RemoteStatic, Mac1KeyLabel); +} + +_Use_decl_annotations_ +VOID +CookieInit(COOKIE *Cookie) +{ + RtlZeroMemory(Cookie, sizeof(*Cookie)); + MuInitializePushLock(&Cookie->Lock); +} + +static VOID +ComputeMac1( + _Out_writes_bytes_all_(COOKIE_LEN) UINT8 Mac1[COOKIE_LEN], + _In_reads_bytes_(Len - sizeof(MESSAGE_MACS)) CONST VOID *Message, + _In_ SIZE_T Len, + _In_ CONST UINT8 Key[NOISE_SYMMETRIC_KEY_LEN]) +{ + Len = Len - sizeof(MESSAGE_MACS) + FIELD_OFFSET(MESSAGE_MACS, Mac1); + Blake2s(Mac1, Message, Key, COOKIE_LEN, Len, NOISE_SYMMETRIC_KEY_LEN); +} + +static VOID +ComputeMac2( + _Out_writes_bytes_all_(COOKIE_LEN) UINT8 Mac2[COOKIE_LEN], + _In_reads_bytes_(Len - COOKIE_LEN) CONST VOID *Message, + _In_ SIZE_T Len, + _In_ CONST UINT8 Cookie[COOKIE_LEN]) +{ + Len = Len - sizeof(MESSAGE_MACS) + FIELD_OFFSET(MESSAGE_MACS, Mac2); + Blake2s(Mac2, Message, Cookie, COOKIE_LEN, Len, COOKIE_LEN); +} + +_Requires_lock_not_held_(Checker->SecretLock) +_IRQL_requires_max_(APC_LEVEL) +static VOID +MakeCookie( + _Out_writes_bytes_all_(COOKIE_LEN) UINT8 Cookie[COOKIE_LEN], + _In_ CONST SOCKADDR *Src, + _Inout_ COOKIE_CHECKER *Checker) +{ + BLAKE2S_STATE State; + + if (BirthdateHasExpired(Checker->SecretBirthdate, COOKIE_SECRET_MAX_AGE)) + { + MuAcquirePushLockExclusive(&Checker->SecretLock); + Checker->SecretBirthdate = KeQueryInterruptTime(); + CryptoRandom(Checker->Secret, NOISE_HASH_LEN); + MuReleasePushLockExclusive(&Checker->SecretLock); + } + + MuAcquirePushLockShared(&Checker->SecretLock); + + Blake2sInitKey(&State, COOKIE_LEN, Checker->Secret, NOISE_HASH_LEN); + if (Src->sa_family == AF_INET) + { + CONST SOCKADDR_IN *Src4 = (CONST SOCKADDR_IN *)Src; + Blake2sUpdate(&State, (UINT8 *)&Src4->sin_addr, sizeof(Src4->sin_addr)); + Blake2sUpdate(&State, (UINT8 *)&Src4->sin_port, sizeof(Src4->sin_port)); + } + else if (Src->sa_family == AF_INET6) + { + CONST SOCKADDR_IN6 *Src6 = (CONST SOCKADDR_IN6 *)Src; + Blake2sUpdate(&State, (UINT8 *)&Src6->sin6_addr, sizeof(Src6->sin6_addr)); + Blake2sUpdate(&State, (UINT8 *)&Src6->sin6_port, sizeof(Src6->sin6_port)); + } + Blake2sFinal(&State, Cookie); + + MuReleasePushLockShared(&Checker->SecretLock); +} + +_Use_decl_annotations_ +COOKIE_MAC_STATE +CookieValidatePacket(COOKIE_CHECKER *Checker, NET_BUFFER_LIST *Nbl, BOOLEAN CheckCookie) +{ + CONST ULONG NblLen = NET_BUFFER_DATA_LENGTH(NET_BUFFER_LIST_FIRST_NB(Nbl)); + UCHAR *NblData = MemGetValidatedNetBufferListData(Nbl); + MESSAGE_MACS *Macs = (MESSAGE_MACS *)(NblData + NblLen - sizeof(*Macs)); + COOKIE_MAC_STATE Ret; + UINT8 ComputedMac[COOKIE_LEN]; + UINT8 Cookie[COOKIE_LEN]; + + Ret = INVALID_MAC; + ComputeMac1(ComputedMac, NblData, NblLen, Checker->MessageMac1Key); + if (!CryptoEqualMemory16(ComputedMac, Macs->Mac1)) + goto out; + + Ret = VALID_MAC_BUT_NO_COOKIE; + + if (!CheckCookie) + goto out; + + MakeCookie(Cookie, NET_BUFFER_LIST_DATAGRAM_INDICATION(Nbl)->RemoteAddress, Checker); + + ComputeMac2(ComputedMac, NblData, NblLen, Cookie); + if (!CryptoEqualMemory16(ComputedMac, Macs->Mac2)) + goto out; + + Ret = VALID_MAC_WITH_COOKIE_BUT_RATELIMITED; + if (!RatelimiterAllow(NET_BUFFER_LIST_DATAGRAM_INDICATION(Nbl)->RemoteAddress)) + goto out; + + Ret = VALID_MAC_WITH_COOKIE; + +out: + return Ret; +} + +_Use_decl_annotations_ +VOID +CookieAddMacToPacket(VOID *Message, SIZE_T Len, WG_PEER *Peer) +{ + MESSAGE_MACS *Macs = (MESSAGE_MACS *)((UINT8 *)Message + Len - sizeof(*Macs)); + + MuAcquirePushLockExclusive(&Peer->LatestCookie.Lock); + ComputeMac1(Macs->Mac1, Message, Len, Peer->LatestCookie.MessageMac1Key); + RtlCopyMemory(Peer->LatestCookie.LastMac1Sent, Macs->Mac1, COOKIE_LEN); + Peer->LatestCookie.HaveSentMac1 = TRUE; + MuReleasePushLockExclusive(&Peer->LatestCookie.Lock); + + MuAcquirePushLockShared(&Peer->LatestCookie.Lock); + if (Peer->LatestCookie.IsValid && + !BirthdateHasExpired(Peer->LatestCookie.Birthdate, COOKIE_SECRET_MAX_AGE - COOKIE_SECRET_LATENCY)) + ComputeMac2(Macs->Mac2, Message, Len, Peer->LatestCookie.Cookie); + else + RtlZeroMemory(Macs->Mac2, COOKIE_LEN); + MuReleasePushLockShared(&Peer->LatestCookie.Lock); +} + +_Use_decl_annotations_ +VOID +CookieMessageCreate(MESSAGE_HANDSHAKE_COOKIE *Dst, CONST NET_BUFFER_LIST *Nbl, UINT32_LE Index, COOKIE_CHECKER *Checker) +{ + CONST ULONG NblLen = NET_BUFFER_DATA_LENGTH(NET_BUFFER_LIST_FIRST_NB(Nbl)); + UCHAR *NblData = MemGetValidatedNetBufferListData(Nbl); + MESSAGE_MACS *Macs = (MESSAGE_MACS *)(NblData + NblLen - sizeof(*Macs)); + UINT8 Cookie[COOKIE_LEN]; + + Dst->Header.Type = CpuToLe32(MESSAGE_TYPE_HANDSHAKE_COOKIE); + Dst->ReceiverIndex = Index; + CryptoRandom(Dst->Nonce, COOKIE_NONCE_LEN); + + MakeCookie(Cookie, NET_BUFFER_LIST_DATAGRAM_INDICATION(Nbl)->RemoteAddress, Checker); + XChaCha20Poly1305Encrypt( + Dst->EncryptedCookie, Cookie, COOKIE_LEN, Macs->Mac1, COOKIE_LEN, Dst->Nonce, Checker->CookieEncryptionKey); +} + +_Use_decl_annotations_ +VOID +CookieMessageConsume(MESSAGE_HANDSHAKE_COOKIE *Src, WG_DEVICE *Wg) +{ + WG_PEER *Peer = NULL; + UINT8 Cookie[COOKIE_LEN]; + BOOLEAN Ret; + + if (!IndexHashtableLookup( + Wg->IndexHashtable, INDEX_HASHTABLE_HANDSHAKE | INDEX_HASHTABLE_KEYPAIR, Src->ReceiverIndex, &Peer)) + return; + + MuAcquirePushLockShared(&Peer->LatestCookie.Lock); + if (!Peer->LatestCookie.HaveSentMac1) + { + MuReleasePushLockShared(&Peer->LatestCookie.Lock); + goto out; + } + Ret = XChaCha20Poly1305Decrypt( + Cookie, + Src->EncryptedCookie, + sizeof(Src->EncryptedCookie), + Peer->LatestCookie.LastMac1Sent, + COOKIE_LEN, + Src->Nonce, + Peer->LatestCookie.CookieDecryptionKey); + MuReleasePushLockShared(&Peer->LatestCookie.Lock); + + if (Ret) + { + MuAcquirePushLockExclusive(&Peer->LatestCookie.Lock); + RtlCopyMemory(Peer->LatestCookie.Cookie, Cookie, COOKIE_LEN); + Peer->LatestCookie.Birthdate = KeQueryInterruptTime(); + Peer->LatestCookie.IsValid = TRUE; + Peer->LatestCookie.HaveSentMac1 = FALSE; + MuReleasePushLockExclusive(&Peer->LatestCookie.Lock); + } + else + LogInfoRatelimited(Wg, "Could not decrypt invalid cookie response"); + +out: + PeerPut(Peer); +} diff --git a/driver/cookie.h b/driver/cookie.h new file mode 100644 index 0000000..b0d9d01 --- /dev/null +++ b/driver/cookie.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +#include "messages.h" + +typedef struct _WG_PEER WG_PEER; +typedef struct _WG_DEVICE WG_DEVICE; + +typedef struct _COOKIE_CHECKER +{ + UINT8 Secret[NOISE_HASH_LEN]; + UINT8 CookieEncryptionKey[NOISE_SYMMETRIC_KEY_LEN]; + UINT8 MessageMac1Key[NOISE_SYMMETRIC_KEY_LEN]; + UINT64 SecretBirthdate; + EX_PUSH_LOCK SecretLock; + WG_DEVICE *Device; +} COOKIE_CHECKER; + +typedef struct _COOKIE +{ + UINT64 Birthdate; + BOOLEAN IsValid; + UINT8 Cookie[COOKIE_LEN]; + BOOLEAN HaveSentMac1; + UINT8 LastMac1Sent[COOKIE_LEN]; + UINT8 CookieDecryptionKey[NOISE_SYMMETRIC_KEY_LEN]; + UINT8 MessageMac1Key[NOISE_SYMMETRIC_KEY_LEN]; + EX_PUSH_LOCK Lock; +} COOKIE; + +typedef enum _COOKIE_MAC_STATE +{ + INVALID_MAC, + VALID_MAC_BUT_NO_COOKIE, + VALID_MAC_WITH_COOKIE_BUT_RATELIMITED, + VALID_MAC_WITH_COOKIE +} COOKIE_MAC_STATE; + +_IRQL_requires_max_(APC_LEVEL) +VOID +CookieCheckerInit(_Out_ COOKIE_CHECKER *Checker, _In_ WG_DEVICE *Wg); + +_Requires_lock_held_(Checker->Device->DeviceUpdateLock) +VOID +CookieCheckerPrecomputeDeviceKeys(_Inout_ COOKIE_CHECKER *Checker); + +VOID +CookieCheckerPrecomputePeerKeys(_Inout_ WG_PEER *Peer); + +_IRQL_requires_max_(APC_LEVEL) +VOID +CookieInit(_Out_ COOKIE *Cookie); + +_Must_inspect_result_ +COOKIE_MAC_STATE +CookieValidatePacket(_Inout_ COOKIE_CHECKER *Checker, _In_ NET_BUFFER_LIST *Nbl, _In_ BOOLEAN CheckCookie); + +_IRQL_requires_max_(APC_LEVEL) +VOID +CookieAddMacToPacket(_Inout_updates_bytes_(Len) VOID *Message, _In_ SIZE_T Len, _Inout_ WG_PEER *Peer); + +_IRQL_requires_max_(APC_LEVEL) +VOID +CookieMessageCreate( + _Out_ MESSAGE_HANDSHAKE_COOKIE *Src, + _In_ CONST NET_BUFFER_LIST *Nbl, + _In_ UINT32_LE Index, + _Inout_ COOKIE_CHECKER *Checker); + +_IRQL_requires_max_(APC_LEVEL) +VOID +CookieMessageConsume(_In_ MESSAGE_HANDSHAKE_COOKIE *Src, _Inout_ WG_DEVICE *Wg); diff --git a/driver/crypto-amd64.asm b/driver/crypto-amd64.asm new file mode 100644 index 0000000..7aaccc7 --- /dev/null +++ b/driver/crypto-amd64.asm @@ -0,0 +1,5724 @@ +; SPDX-License-Identifier: GPL-2.0 OR MIT +; +; Copyright (C) 2006-2020 Andy Polyakov <appro@cryptogams.org>. +; Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. +; +; This is a modified version of: +; https://github.com/dot-asm/cryptogams/blob/master/x86_64/chacha-x86_64.pl +; https://github.com/dot-asm/cryptogams/blob/master/x86_64/poly1305-x86_64.pl +; + +OPTION DOTNAME +.text$ SEGMENT ALIGN(256) 'CODE' +EXTERN __imp_RtlVirtualUnwind:NEAR + +ALIGN 64 +$L$zero:: + DD 0,0,0,0 +$L$one:: + DD 1,0,0,0 +$L$inc:: + DD 0,1,2,3 +$L$four:: + DD 4,4,4,4 +$L$incy:: + DD 0,2,4,6,1,3,5,7 +$L$eight:: + DD 8,8,8,8,8,8,8,8 +$L$rot16:: +DB 02h,03h,00h,01h,06h,07h,04h,05h,0ah,0bh,08h,09h,0eh,0fh,0ch,0dh +$L$rot24:: +DB 03h,00h,01h,02h,07h,04h,05h,06h,0bh,08h,09h,0ah,0fh,0ch,0dh,0eh +$L$twoy:: + DD 2,0,0,0,2,0,0,0 +ALIGN 64 +$L$zeroz:: + DD 0,0,0,0,1,0,0,0,2,0,0,0,3,0,0,0 +$L$fourz:: + DD 4,0,0,0,4,0,0,0,4,0,0,0,4,0,0,0 +$L$incz:: + DD 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 +$L$sixteen:: + DD 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16 +$L$sigma:: +DB 101,120,112,97,110,100,32,51,50,45,98,121,116,101,32,107 +DB 0 +PUBLIC ChaCha20ALU + +ALIGN 64 +ChaCha20ALU PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_chacha20_ctr32:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD PTR[40+rsp] + cmp rdx,0 + je $L$chacha20_no_data + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + sub rsp,64+24 + +$L$ctr32_body:: + mov rbp,rdx + mov r12,QWORD PTR[rcx] + mov r13,QWORD PTR[8+rcx] + mov r14,QWORD PTR[16+rcx] + mov r15,QWORD PTR[24+rcx] + mov rax,QWORD PTR[r8] + mov rdx,QWORD PTR[8+r8] + mov QWORD PTR[16+rsp],r12 + mov QWORD PTR[24+rsp],r13 + mov QWORD PTR[rsp],r14 + mov QWORD PTR[8+rsp],r15 + mov QWORD PTR[48+rsp],rax + mov QWORD PTR[56+rsp],rdx + jmp $L$chacha20_loop_outer + +ALIGN 32 +$L$chacha20_loop_outer:: + mov eax,061707865h + mov ebx,03320646eh + mov ecx,079622d32h + mov edx,06b206574h + mov r8d,DWORD PTR[16+rsp] + mov r9d,DWORD PTR[20+rsp] + mov r10d,DWORD PTR[24+rsp] + mov r11d,DWORD PTR[28+rsp] + mov r12d,DWORD PTR[48+rsp] + mov r13d,DWORD PTR[52+rsp] + mov r14d,DWORD PTR[56+rsp] + mov QWORD PTR[40+rsp],r15 + mov r15d,DWORD PTR[60+rsp] + mov QWORD PTR[((64+0))+rsp],rbp + mov QWORD PTR[((64+8))+rsp],rsi + mov esi,DWORD PTR[rsp] + mov QWORD PTR[((64+16))+rsp],rdi + mov edi,DWORD PTR[4+rsp] + mov ebp,10 + jmp $L$chacha20_loop + +ALIGN 32 +$L$chacha20_loop:: + add eax,r8d + xor r12d,eax + rol r12d,16 + add ebx,r9d + xor r13d,ebx + rol r13d,16 + add esi,r12d + xor r8d,esi + rol r8d,12 + add edi,r13d + xor r9d,edi + rol r9d,12 + add eax,r8d + xor r12d,eax + rol r12d,8 + add ebx,r9d + xor r13d,ebx + rol r13d,8 + add esi,r12d + xor r8d,esi + rol r8d,7 + add edi,r13d + xor r9d,edi + rol r9d,7 + mov DWORD PTR[32+rsp],esi + mov DWORD PTR[36+rsp],edi + mov esi,DWORD PTR[40+rsp] + mov edi,DWORD PTR[44+rsp] + add ecx,r10d + xor r14d,ecx + rol r14d,16 + add edx,r11d + xor r15d,edx + rol r15d,16 + add esi,r14d + xor r10d,esi + rol r10d,12 + add edi,r15d + xor r11d,edi + rol r11d,12 + add ecx,r10d + xor r14d,ecx + rol r14d,8 + add edx,r11d + xor r15d,edx + rol r15d,8 + add esi,r14d + xor r10d,esi + rol r10d,7 + add edi,r15d + xor r11d,edi + rol r11d,7 + add eax,r9d + xor r15d,eax + rol r15d,16 + add ebx,r10d + xor r12d,ebx + rol r12d,16 + add esi,r15d + xor r9d,esi + rol r9d,12 + add edi,r12d + xor r10d,edi + rol r10d,12 + add eax,r9d + xor r15d,eax + rol r15d,8 + add ebx,r10d + xor r12d,ebx + rol r12d,8 + add esi,r15d + xor r9d,esi + rol r9d,7 + add edi,r12d + xor r10d,edi + rol r10d,7 + mov DWORD PTR[40+rsp],esi + mov DWORD PTR[44+rsp],edi + mov esi,DWORD PTR[32+rsp] + mov edi,DWORD PTR[36+rsp] + add ecx,r11d + xor r13d,ecx + rol r13d,16 + add edx,r8d + xor r14d,edx + rol r14d,16 + add esi,r13d + xor r11d,esi + rol r11d,12 + add edi,r14d + xor r8d,edi + rol r8d,12 + add ecx,r11d + xor r13d,ecx + rol r13d,8 + add edx,r8d + xor r14d,edx + rol r14d,8 + add esi,r13d + xor r11d,esi + rol r11d,7 + add edi,r14d + xor r8d,edi + rol r8d,7 + dec ebp + jnz $L$chacha20_loop + add esi,DWORD PTR[rsp] + add edi,DWORD PTR[4+rsp] + mov rbp,QWORD PTR[64+rsp] + mov DWORD PTR[32+rsp],esi + mov rsi,QWORD PTR[((64+8))+rsp] + mov DWORD PTR[36+rsp],edi + mov rdi,QWORD PTR[((64+16))+rsp] + add eax,061707865h + add ebx,03320646eh + add ecx,079622d32h + add edx,06b206574h + add r8d,DWORD PTR[16+rsp] + add r9d,DWORD PTR[20+rsp] + add r10d,DWORD PTR[24+rsp] + add r11d,DWORD PTR[28+rsp] + add r12d,DWORD PTR[48+rsp] + add r13d,DWORD PTR[52+rsp] + add r14d,DWORD PTR[56+rsp] + add r15d,DWORD PTR[60+rsp] + cmp rbp,64 + jb $L$tail + xor eax,DWORD PTR[rsi] + xor ebx,DWORD PTR[4+rsi] + xor ecx,DWORD PTR[8+rsi] + xor edx,DWORD PTR[12+rsi] + mov DWORD PTR[rdi],eax + mov eax,DWORD PTR[32+rsp] + mov DWORD PTR[4+rdi],ebx + mov ebx,DWORD PTR[36+rsp] + mov DWORD PTR[8+rdi],ecx + mov ecx,DWORD PTR[40+rsp] + mov DWORD PTR[12+rdi],edx + mov edx,DWORD PTR[44+rsp] + xor r8d,DWORD PTR[16+rsi] + add ecx,DWORD PTR[8+rsp] + xor r9d,DWORD PTR[20+rsi] + add edx,DWORD PTR[12+rsp] + xor r10d,DWORD PTR[24+rsi] + xor r11d,DWORD PTR[28+rsi] + xor eax,DWORD PTR[32+rsi] + xor ebx,DWORD PTR[36+rsi] + xor ecx,DWORD PTR[40+rsi] + xor edx,DWORD PTR[44+rsi] + xor r12d,DWORD PTR[48+rsi] + xor r13d,DWORD PTR[52+rsi] + xor r14d,DWORD PTR[56+rsi] + xor r15d,DWORD PTR[60+rsi] + lea rsi,QWORD PTR[64+rsi] + add DWORD PTR[48+rsp],1 + mov DWORD PTR[16+rdi],r8d + mov DWORD PTR[20+rdi],r9d + mov DWORD PTR[24+rdi],r10d + mov DWORD PTR[28+rdi],r11d + mov DWORD PTR[32+rdi],eax + mov DWORD PTR[36+rdi],ebx + mov DWORD PTR[40+rdi],ecx + mov DWORD PTR[44+rdi],edx + mov DWORD PTR[48+rdi],r12d + mov DWORD PTR[52+rdi],r13d + mov DWORD PTR[56+rdi],r14d + mov DWORD PTR[60+rdi],r15d + lea rdi,QWORD PTR[64+rdi] + mov r15,QWORD PTR[8+rsp] + sub rbp,64 + jnz $L$chacha20_loop_outer + jmp $L$done + +ALIGN 16 +$L$tail:: + mov DWORD PTR[rsp],eax + mov eax,DWORD PTR[8+rsp] + mov DWORD PTR[4+rsp],ebx + mov ebx,DWORD PTR[12+rsp] + mov DWORD PTR[8+rsp],ecx + add eax,DWORD PTR[40+rsp] + mov DWORD PTR[12+rsp],edx + add ebx,DWORD PTR[44+rsp] + mov DWORD PTR[16+rsp],r8d + mov DWORD PTR[20+rsp],r9d + mov DWORD PTR[24+rsp],r10d + mov DWORD PTR[28+rsp],r11d + mov DWORD PTR[40+rsp],eax + mov DWORD PTR[44+rsp],ebx + xor rbx,rbx + mov DWORD PTR[48+rsp],r12d + mov DWORD PTR[52+rsp],r13d + mov DWORD PTR[56+rsp],r14d + mov DWORD PTR[60+rsp],r15d + +$L$chacha20_loop_tail:: + movzx eax,BYTE PTR[rbx*1+rsi] + movzx edx,BYTE PTR[rbx*1+rsp] + lea rbx,QWORD PTR[1+rbx] + xor eax,edx + mov BYTE PTR[((-1))+rbx*1+rdi],al + dec rbp + jnz $L$chacha20_loop_tail + +$L$done:: + lea rsi,QWORD PTR[((64+24+48))+rsp] + mov r15,QWORD PTR[((-48))+rsi] + mov r14,QWORD PTR[((-40))+rsi] + mov r13,QWORD PTR[((-32))+rsi] + mov r12,QWORD PTR[((-24))+rsi] + mov rbp,QWORD PTR[((-16))+rsi] + mov rbx,QWORD PTR[((-8))+rsi] + lea rsp,QWORD PTR[rsi] + +$L$chacha20_no_data:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret + +$L$SEH_end_chacha20_ctr32:: +ChaCha20ALU ENDP +PUBLIC ChaCha20SSSE3 + +ALIGN 32 +ChaCha20SSSE3 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_chacha20_ssse3:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD PTR[40+rsp] + +$L$chacha20_ssse3:: + mov r10,rsp + cmp rdx,128 + je $L$chacha20_128 + ja $L$chacha20_4x + +$L$do_sse3_after_all:: + sub rsp,64+40 + and rsp,-16 + movaps XMMWORD PTR[(-40)+r10],xmm6 + movaps XMMWORD PTR[(-24)+r10],xmm7 +$L$ssse3_body:: + movdqa xmm0,XMMWORD PTR[$L$sigma] + movdqu xmm1,XMMWORD PTR[rcx] + movdqu xmm2,XMMWORD PTR[16+rcx] + movdqu xmm3,XMMWORD PTR[r8] + movdqa xmm6,XMMWORD PTR[$L$rot16] + movdqa xmm7,XMMWORD PTR[$L$rot24] + movdqa XMMWORD PTR[rsp],xmm0 + movdqa XMMWORD PTR[16+rsp],xmm1 + movdqa XMMWORD PTR[32+rsp],xmm2 + movdqa XMMWORD PTR[48+rsp],xmm3 + mov r8,10 + jmp $L$chacha20_loop_ssse3 + +ALIGN 32 +$L$chacha20_loop_outer_ssse3:: + movdqa xmm3,XMMWORD PTR[$L$one] + movdqa xmm0,XMMWORD PTR[rsp] + movdqa xmm1,XMMWORD PTR[16+rsp] + movdqa xmm2,XMMWORD PTR[32+rsp] + paddd xmm3,XMMWORD PTR[48+rsp] + mov r8,10 + movdqa XMMWORD PTR[48+rsp],xmm3 + jmp $L$chacha20_loop_ssse3 + +ALIGN 32 +$L$chacha20_loop_ssse3:: + paddd xmm0,xmm1 + pxor xmm3,xmm0 +DB 102,15,56,0,222 + paddd xmm2,xmm3 + pxor xmm1,xmm2 + movdqa xmm4,xmm1 + psrld xmm1,20 + pslld xmm4,12 + por xmm1,xmm4 + paddd xmm0,xmm1 + pxor xmm3,xmm0 +DB 102,15,56,0,223 + paddd xmm2,xmm3 + pxor xmm1,xmm2 + movdqa xmm4,xmm1 + psrld xmm1,25 + pslld xmm4,7 + por xmm1,xmm4 + pshufd xmm2,xmm2,78 + pshufd xmm1,xmm1,57 + pshufd xmm3,xmm3,147 + nop + paddd xmm0,xmm1 + pxor xmm3,xmm0 +DB 102,15,56,0,222 + paddd xmm2,xmm3 + pxor xmm1,xmm2 + movdqa xmm4,xmm1 + psrld xmm1,20 + pslld xmm4,12 + por xmm1,xmm4 + paddd xmm0,xmm1 + pxor xmm3,xmm0 +DB 102,15,56,0,223 + paddd xmm2,xmm3 + pxor xmm1,xmm2 + movdqa xmm4,xmm1 + psrld xmm1,25 + pslld xmm4,7 + por xmm1,xmm4 + pshufd xmm2,xmm2,78 + pshufd xmm1,xmm1,147 + pshufd xmm3,xmm3,57 + dec r8 + jnz $L$chacha20_loop_ssse3 + paddd xmm0,XMMWORD PTR[rsp] + paddd xmm1,XMMWORD PTR[16+rsp] + paddd xmm2,XMMWORD PTR[32+rsp] + paddd xmm3,XMMWORD PTR[48+rsp] + cmp rdx,64 + jb $L$tail_ssse3 + movdqu xmm4,XMMWORD PTR[rsi] + movdqu xmm5,XMMWORD PTR[16+rsi] + pxor xmm0,xmm4 + movdqu xmm4,XMMWORD PTR[32+rsi] + pxor xmm1,xmm5 + movdqu xmm5,XMMWORD PTR[48+rsi] + lea rsi,QWORD PTR[64+rsi] + pxor xmm2,xmm4 + pxor xmm3,xmm5 + movdqu XMMWORD PTR[rdi],xmm0 + movdqu XMMWORD PTR[16+rdi],xmm1 + movdqu XMMWORD PTR[32+rdi],xmm2 + movdqu XMMWORD PTR[48+rdi],xmm3 + lea rdi,QWORD PTR[64+rdi] + sub rdx,64 + jnz $L$chacha20_loop_outer_ssse3 + jmp $L$done_ssse3 + +ALIGN 16 +$L$tail_ssse3:: + movdqa XMMWORD PTR[rsp],xmm0 + movdqa XMMWORD PTR[16+rsp],xmm1 + movdqa XMMWORD PTR[32+rsp],xmm2 + movdqa XMMWORD PTR[48+rsp],xmm3 + xor r8,r8 + +$L$chacha20_loop_tail_ssse3:: + movzx eax,BYTE PTR[r8*1+rsi] + movzx ecx,BYTE PTR[r8*1+rsp] + lea r8,QWORD PTR[1+r8] + xor eax,ecx + mov BYTE PTR[((-1))+r8*1+rdi],al + dec rdx + jnz $L$chacha20_loop_tail_ssse3 + +$L$done_ssse3:: + movaps xmm6,XMMWORD PTR[((-40))+r10] + movaps xmm7,XMMWORD PTR[((-24))+r10] + lea rsp,QWORD PTR[r10] + +$L$ssse3_epilogue:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret + +$L$SEH_end_chacha20_ssse3:: +ChaCha20SSSE3 ENDP + +ALIGN 32 +chacha20_128 PROC PRIVATE + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_chacha20_128:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD PTR[40+rsp] + +$L$chacha20_128:: + mov r10,rsp + sub rsp,64+104 + and rsp,-16 + movaps XMMWORD PTR[(-104)+r10],xmm6 + movaps XMMWORD PTR[(-88)+r10],xmm7 + movaps XMMWORD PTR[(-72)+r10],xmm8 + movaps XMMWORD PTR[(-56)+r10],xmm9 + movaps XMMWORD PTR[(-40)+r10],xmm10 + movaps XMMWORD PTR[(-24)+r10],xmm11 +$L$128_body:: + movdqa xmm8,XMMWORD PTR[$L$sigma] + movdqu xmm9,XMMWORD PTR[rcx] + movdqu xmm2,XMMWORD PTR[16+rcx] + movdqu xmm3,XMMWORD PTR[r8] + movdqa xmm1,XMMWORD PTR[$L$one] + movdqa xmm6,XMMWORD PTR[$L$rot16] + movdqa xmm7,XMMWORD PTR[$L$rot24] + movdqa xmm10,xmm8 + movdqa XMMWORD PTR[rsp],xmm8 + movdqa xmm11,xmm9 + movdqa XMMWORD PTR[16+rsp],xmm9 + movdqa xmm0,xmm2 + movdqa XMMWORD PTR[32+rsp],xmm2 + paddd xmm1,xmm3 + movdqa XMMWORD PTR[48+rsp],xmm3 + mov r8,10 + jmp $L$chacha20_loop_128 + +ALIGN 32 +$L$chacha20_loop_128:: + paddd xmm8,xmm9 + pxor xmm3,xmm8 + paddd xmm10,xmm11 + pxor xmm1,xmm10 +DB 102,15,56,0,222 +DB 102,15,56,0,206 + paddd xmm2,xmm3 + paddd xmm0,xmm1 + pxor xmm9,xmm2 + pxor xmm11,xmm0 + movdqa xmm4,xmm9 + psrld xmm9,20 + movdqa xmm5,xmm11 + pslld xmm4,12 + psrld xmm11,20 + por xmm9,xmm4 + pslld xmm5,12 + por xmm11,xmm5 + paddd xmm8,xmm9 + pxor xmm3,xmm8 + paddd xmm10,xmm11 + pxor xmm1,xmm10 +DB 102,15,56,0,223 +DB 102,15,56,0,207 + paddd xmm2,xmm3 + paddd xmm0,xmm1 + pxor xmm9,xmm2 + pxor xmm11,xmm0 + movdqa xmm4,xmm9 + psrld xmm9,25 + movdqa xmm5,xmm11 + pslld xmm4,7 + psrld xmm11,25 + por xmm9,xmm4 + pslld xmm5,7 + por xmm11,xmm5 + pshufd xmm2,xmm2,78 + pshufd xmm9,xmm9,57 + pshufd xmm3,xmm3,147 + pshufd xmm0,xmm0,78 + pshufd xmm11,xmm11,57 + pshufd xmm1,xmm1,147 + paddd xmm8,xmm9 + pxor xmm3,xmm8 + paddd xmm10,xmm11 + pxor xmm1,xmm10 +DB 102,15,56,0,222 +DB 102,15,56,0,206 + paddd xmm2,xmm3 + paddd xmm0,xmm1 + pxor xmm9,xmm2 + pxor xmm11,xmm0 + movdqa xmm4,xmm9 + psrld xmm9,20 + movdqa xmm5,xmm11 + pslld xmm4,12 + psrld xmm11,20 + por xmm9,xmm4 + pslld xmm5,12 + por xmm11,xmm5 + paddd xmm8,xmm9 + pxor xmm3,xmm8 + paddd xmm10,xmm11 + pxor xmm1,xmm10 +DB 102,15,56,0,223 +DB 102,15,56,0,207 + paddd xmm2,xmm3 + paddd xmm0,xmm1 + pxor xmm9,xmm2 + pxor xmm11,xmm0 + movdqa xmm4,xmm9 + psrld xmm9,25 + movdqa xmm5,xmm11 + pslld xmm4,7 + psrld xmm11,25 + por xmm9,xmm4 + pslld xmm5,7 + por xmm11,xmm5 + pshufd xmm2,xmm2,78 + pshufd xmm9,xmm9,147 + pshufd xmm3,xmm3,57 + pshufd xmm0,xmm0,78 + pshufd xmm11,xmm11,147 + pshufd xmm1,xmm1,57 + dec r8 + jnz $L$chacha20_loop_128 + paddd xmm8,XMMWORD PTR[rsp] + paddd xmm9,XMMWORD PTR[16+rsp] + paddd xmm2,XMMWORD PTR[32+rsp] + paddd xmm3,XMMWORD PTR[48+rsp] + paddd xmm1,XMMWORD PTR[$L$one] + paddd xmm10,XMMWORD PTR[rsp] + paddd xmm11,XMMWORD PTR[16+rsp] + paddd xmm0,XMMWORD PTR[32+rsp] + paddd xmm1,XMMWORD PTR[48+rsp] + movdqu xmm4,XMMWORD PTR[rsi] + movdqu xmm5,XMMWORD PTR[16+rsi] + pxor xmm8,xmm4 + movdqu xmm4,XMMWORD PTR[32+rsi] + pxor xmm9,xmm5 + movdqu xmm5,XMMWORD PTR[48+rsi] + pxor xmm2,xmm4 + movdqu xmm4,XMMWORD PTR[64+rsi] + pxor xmm3,xmm5 + movdqu xmm5,XMMWORD PTR[80+rsi] + pxor xmm10,xmm4 + movdqu xmm4,XMMWORD PTR[96+rsi] + pxor xmm11,xmm5 + movdqu xmm5,XMMWORD PTR[112+rsi] + pxor xmm0,xmm4 + pxor xmm1,xmm5 + movdqu XMMWORD PTR[rdi],xmm8 + movdqu XMMWORD PTR[16+rdi],xmm9 + movdqu XMMWORD PTR[32+rdi],xmm2 + movdqu XMMWORD PTR[48+rdi],xmm3 + movdqu XMMWORD PTR[64+rdi],xmm10 + movdqu XMMWORD PTR[80+rdi],xmm11 + movdqu XMMWORD PTR[96+rdi],xmm0 + movdqu XMMWORD PTR[112+rdi],xmm1 + movaps xmm6,XMMWORD PTR[((-104))+r10] + movaps xmm7,XMMWORD PTR[((-88))+r10] + movaps xmm8,XMMWORD PTR[((-72))+r10] + movaps xmm9,XMMWORD PTR[((-56))+r10] + movaps xmm10,XMMWORD PTR[((-40))+r10] + movaps xmm11,XMMWORD PTR[((-24))+r10] + lea rsp,QWORD PTR[r10] + +$L$128_epilogue:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret + +$L$SEH_end_chacha20_128:: +chacha20_128 ENDP + +ALIGN 32 +chacha20_4x PROC PRIVATE + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_chacha20_4x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD PTR[40+rsp] + +$L$chacha20_4x:: + mov r10,rsp + mov r11,r9 + cmp rdx,192 + ja $L$proceed4x + and r11,71303168 + cmp r11,4194304 + je $L$do_sse3_after_all + +$L$proceed4x:: + sub rsp,0140h+168 + and rsp,-16 + movaps XMMWORD PTR[(-168)+r10],xmm6 + movaps XMMWORD PTR[(-152)+r10],xmm7 + movaps XMMWORD PTR[(-136)+r10],xmm8 + movaps XMMWORD PTR[(-120)+r10],xmm9 + movaps XMMWORD PTR[(-104)+r10],xmm10 + movaps XMMWORD PTR[(-88)+r10],xmm11 + movaps XMMWORD PTR[(-72)+r10],xmm12 + movaps XMMWORD PTR[(-56)+r10],xmm13 + movaps XMMWORD PTR[(-40)+r10],xmm14 + movaps XMMWORD PTR[(-24)+r10],xmm15 +$L$4x_body:: + movdqa xmm11,XMMWORD PTR[$L$sigma] + movdqu xmm15,XMMWORD PTR[rcx] + movdqu xmm7,XMMWORD PTR[16+rcx] + movdqu xmm3,XMMWORD PTR[r8] + lea rcx,QWORD PTR[256+rsp] + lea r9,QWORD PTR[$L$rot16] + lea r11,QWORD PTR[$L$rot24] + pshufd xmm8,xmm11,000h + pshufd xmm9,xmm11,055h + movdqa XMMWORD PTR[64+rsp],xmm8 + pshufd xmm10,xmm11,0aah + movdqa XMMWORD PTR[80+rsp],xmm9 + pshufd xmm11,xmm11,0ffh + movdqa XMMWORD PTR[96+rsp],xmm10 + movdqa XMMWORD PTR[112+rsp],xmm11 + pshufd xmm12,xmm15,000h + pshufd xmm13,xmm15,055h + movdqa XMMWORD PTR[(128-256)+rcx],xmm12 + pshufd xmm14,xmm15,0aah + movdqa XMMWORD PTR[(144-256)+rcx],xmm13 + pshufd xmm15,xmm15,0ffh + movdqa XMMWORD PTR[(160-256)+rcx],xmm14 + movdqa XMMWORD PTR[(176-256)+rcx],xmm15 + pshufd xmm4,xmm7,000h + pshufd xmm5,xmm7,055h + movdqa XMMWORD PTR[(192-256)+rcx],xmm4 + pshufd xmm6,xmm7,0aah + movdqa XMMWORD PTR[(208-256)+rcx],xmm5 + pshufd xmm7,xmm7,0ffh + movdqa XMMWORD PTR[(224-256)+rcx],xmm6 + movdqa XMMWORD PTR[(240-256)+rcx],xmm7 + pshufd xmm0,xmm3,000h + pshufd xmm1,xmm3,055h + paddd xmm0,XMMWORD PTR[$L$inc] + pshufd xmm2,xmm3,0aah + movdqa XMMWORD PTR[(272-256)+rcx],xmm1 + pshufd xmm3,xmm3,0ffh + movdqa XMMWORD PTR[(288-256)+rcx],xmm2 + movdqa XMMWORD PTR[(304-256)+rcx],xmm3 + jmp $L$chacha20_loop_enter4x + +ALIGN 32 +$L$chacha20_loop_outer4x:: + movdqa xmm8,XMMWORD PTR[64+rsp] + movdqa xmm9,XMMWORD PTR[80+rsp] + movdqa xmm10,XMMWORD PTR[96+rsp] + movdqa xmm11,XMMWORD PTR[112+rsp] + movdqa xmm12,XMMWORD PTR[((128-256))+rcx] + movdqa xmm13,XMMWORD PTR[((144-256))+rcx] + movdqa xmm14,XMMWORD PTR[((160-256))+rcx] + movdqa xmm15,XMMWORD PTR[((176-256))+rcx] + movdqa xmm4,XMMWORD PTR[((192-256))+rcx] + movdqa xmm5,XMMWORD PTR[((208-256))+rcx] + movdqa xmm6,XMMWORD PTR[((224-256))+rcx] + movdqa xmm7,XMMWORD PTR[((240-256))+rcx] + movdqa xmm0,XMMWORD PTR[((256-256))+rcx] + movdqa xmm1,XMMWORD PTR[((272-256))+rcx] + movdqa xmm2,XMMWORD PTR[((288-256))+rcx] + movdqa xmm3,XMMWORD PTR[((304-256))+rcx] + paddd xmm0,XMMWORD PTR[$L$four] + +$L$chacha20_loop_enter4x:: + movdqa XMMWORD PTR[32+rsp],xmm6 + movdqa XMMWORD PTR[48+rsp],xmm7 + movdqa xmm7,XMMWORD PTR[r9] + mov eax,10 + movdqa XMMWORD PTR[(256-256)+rcx],xmm0 + jmp $L$chacha20_loop4x + +ALIGN 32 +$L$chacha20_loop4x:: + paddd xmm8,xmm12 + paddd xmm9,xmm13 + pxor xmm0,xmm8 + pxor xmm1,xmm9 +DB 102,15,56,0,199 +DB 102,15,56,0,207 + paddd xmm4,xmm0 + paddd xmm5,xmm1 + pxor xmm12,xmm4 + pxor xmm13,xmm5 + movdqa xmm6,xmm12 + pslld xmm12,12 + psrld xmm6,20 + movdqa xmm7,xmm13 + pslld xmm13,12 + por xmm12,xmm6 + psrld xmm7,20 + movdqa xmm6,XMMWORD PTR[r11] + por xmm13,xmm7 + paddd xmm8,xmm12 + paddd xmm9,xmm13 + pxor xmm0,xmm8 + pxor xmm1,xmm9 +DB 102,15,56,0,198 +DB 102,15,56,0,206 + paddd xmm4,xmm0 + paddd xmm5,xmm1 + pxor xmm12,xmm4 + pxor xmm13,xmm5 + movdqa xmm7,xmm12 + pslld xmm12,7 + psrld xmm7,25 + movdqa xmm6,xmm13 + pslld xmm13,7 + por xmm12,xmm7 + psrld xmm6,25 + movdqa xmm7,XMMWORD PTR[r9] + por xmm13,xmm6 + movdqa XMMWORD PTR[rsp],xmm4 + movdqa XMMWORD PTR[16+rsp],xmm5 + movdqa xmm4,XMMWORD PTR[32+rsp] + movdqa xmm5,XMMWORD PTR[48+rsp] + paddd xmm10,xmm14 + paddd xmm11,xmm15 + pxor xmm2,xmm10 + pxor xmm3,xmm11 +DB 102,15,56,0,215 +DB 102,15,56,0,223 + paddd xmm4,xmm2 + paddd xmm5,xmm3 + pxor xmm14,xmm4 + pxor xmm15,xmm5 + movdqa xmm6,xmm14 + pslld xmm14,12 + psrld xmm6,20 + movdqa xmm7,xmm15 + pslld xmm15,12 + por xmm14,xmm6 + psrld xmm7,20 + movdqa xmm6,XMMWORD PTR[r11] + por xmm15,xmm7 + paddd xmm10,xmm14 + paddd xmm11,xmm15 + pxor xmm2,xmm10 + pxor xmm3,xmm11 +DB 102,15,56,0,214 +DB 102,15,56,0,222 + paddd xmm4,xmm2 + paddd xmm5,xmm3 + pxor xmm14,xmm4 + pxor xmm15,xmm5 + movdqa xmm7,xmm14 + pslld xmm14,7 + psrld xmm7,25 + movdqa xmm6,xmm15 + pslld xmm15,7 + por xmm14,xmm7 + psrld xmm6,25 + movdqa xmm7,XMMWORD PTR[r9] + por xmm15,xmm6 + paddd xmm8,xmm13 + paddd xmm9,xmm14 + pxor xmm3,xmm8 + pxor xmm0,xmm9 +DB 102,15,56,0,223 +DB 102,15,56,0,199 + paddd xmm4,xmm3 + paddd xmm5,xmm0 + pxor xmm13,xmm4 + pxor xmm14,xmm5 + movdqa xmm6,xmm13 + pslld xmm13,12 + psrld xmm6,20 + movdqa xmm7,xmm14 + pslld xmm14,12 + por xmm13,xmm6 + psrld xmm7,20 + movdqa xmm6,XMMWORD PTR[r11] + por xmm14,xmm7 + paddd xmm8,xmm13 + paddd xmm9,xmm14 + pxor xmm3,xmm8 + pxor xmm0,xmm9 +DB 102,15,56,0,222 +DB 102,15,56,0,198 + paddd xmm4,xmm3 + paddd xmm5,xmm0 + pxor xmm13,xmm4 + pxor xmm14,xmm5 + movdqa xmm7,xmm13 + pslld xmm13,7 + psrld xmm7,25 + movdqa xmm6,xmm14 + pslld xmm14,7 + por xmm13,xmm7 + psrld xmm6,25 + movdqa xmm7,XMMWORD PTR[r9] + por xmm14,xmm6 + movdqa XMMWORD PTR[32+rsp],xmm4 + movdqa XMMWORD PTR[48+rsp],xmm5 + movdqa xmm4,XMMWORD PTR[rsp] + movdqa xmm5,XMMWORD PTR[16+rsp] + paddd xmm10,xmm15 + paddd xmm11,xmm12 + pxor xmm1,xmm10 + pxor xmm2,xmm11 +DB 102,15,56,0,207 +DB 102,15,56,0,215 + paddd xmm4,xmm1 + paddd xmm5,xmm2 + pxor xmm15,xmm4 + pxor xmm12,xmm5 + movdqa xmm6,xmm15 + pslld xmm15,12 + psrld xmm6,20 + movdqa xmm7,xmm12 + pslld xmm12,12 + por xmm15,xmm6 + psrld xmm7,20 + movdqa xmm6,XMMWORD PTR[r11] + por xmm12,xmm7 + paddd xmm10,xmm15 + paddd xmm11,xmm12 + pxor xmm1,xmm10 + pxor xmm2,xmm11 +DB 102,15,56,0,206 +DB 102,15,56,0,214 + paddd xmm4,xmm1 + paddd xmm5,xmm2 + pxor xmm15,xmm4 + pxor xmm12,xmm5 + movdqa xmm7,xmm15 + pslld xmm15,7 + psrld xmm7,25 + movdqa xmm6,xmm12 + pslld xmm12,7 + por xmm15,xmm7 + psrld xmm6,25 + movdqa xmm7,XMMWORD PTR[r9] + por xmm12,xmm6 + dec eax + jnz $L$chacha20_loop4x + paddd xmm8,XMMWORD PTR[64+rsp] + paddd xmm9,XMMWORD PTR[80+rsp] + paddd xmm10,XMMWORD PTR[96+rsp] + paddd xmm11,XMMWORD PTR[112+rsp] + movdqa xmm6,xmm8 + punpckldq xmm8,xmm9 + movdqa xmm7,xmm10 + punpckldq xmm10,xmm11 + punpckhdq xmm6,xmm9 + punpckhdq xmm7,xmm11 + movdqa xmm9,xmm8 + punpcklqdq xmm8,xmm10 + movdqa xmm11,xmm6 + punpcklqdq xmm6,xmm7 + punpckhqdq xmm9,xmm10 + punpckhqdq xmm11,xmm7 + paddd xmm12,XMMWORD PTR[((128-256))+rcx] + paddd xmm13,XMMWORD PTR[((144-256))+rcx] + paddd xmm14,XMMWORD PTR[((160-256))+rcx] + paddd xmm15,XMMWORD PTR[((176-256))+rcx] + movdqa XMMWORD PTR[rsp],xmm8 + movdqa XMMWORD PTR[16+rsp],xmm9 + movdqa xmm8,XMMWORD PTR[32+rsp] + movdqa xmm9,XMMWORD PTR[48+rsp] + movdqa xmm10,xmm12 + punpckldq xmm12,xmm13 + movdqa xmm7,xmm14 + punpckldq xmm14,xmm15 + punpckhdq xmm10,xmm13 + punpckhdq xmm7,xmm15 + movdqa xmm13,xmm12 + punpcklqdq xmm12,xmm14 + movdqa xmm15,xmm10 + punpcklqdq xmm10,xmm7 + punpckhqdq xmm13,xmm14 + punpckhqdq xmm15,xmm7 + paddd xmm4,XMMWORD PTR[((192-256))+rcx] + paddd xmm5,XMMWORD PTR[((208-256))+rcx] + paddd xmm8,XMMWORD PTR[((224-256))+rcx] + paddd xmm9,XMMWORD PTR[((240-256))+rcx] + movdqa XMMWORD PTR[32+rsp],xmm6 + movdqa XMMWORD PTR[48+rsp],xmm11 + movdqa xmm14,xmm4 + punpckldq xmm4,xmm5 + movdqa xmm7,xmm8 + punpckldq xmm8,xmm9 + punpckhdq xmm14,xmm5 + punpckhdq xmm7,xmm9 + movdqa xmm5,xmm4 + punpcklqdq xmm4,xmm8 + movdqa xmm9,xmm14 + punpcklqdq xmm14,xmm7 + punpckhqdq xmm5,xmm8 + punpckhqdq xmm9,xmm7 + paddd xmm0,XMMWORD PTR[((256-256))+rcx] + paddd xmm1,XMMWORD PTR[((272-256))+rcx] + paddd xmm2,XMMWORD PTR[((288-256))+rcx] + paddd xmm3,XMMWORD PTR[((304-256))+rcx] + movdqa xmm8,xmm0 + punpckldq xmm0,xmm1 + movdqa xmm7,xmm2 + punpckldq xmm2,xmm3 + punpckhdq xmm8,xmm1 + punpckhdq xmm7,xmm3 + movdqa xmm1,xmm0 + punpcklqdq xmm0,xmm2 + movdqa xmm3,xmm8 + punpcklqdq xmm8,xmm7 + punpckhqdq xmm1,xmm2 + punpckhqdq xmm3,xmm7 + cmp rdx,64*4 + jb $L$tail4x + movdqu xmm6,XMMWORD PTR[rsi] + movdqu xmm11,XMMWORD PTR[16+rsi] + movdqu xmm2,XMMWORD PTR[32+rsi] + movdqu xmm7,XMMWORD PTR[48+rsi] + pxor xmm6,XMMWORD PTR[rsp] + pxor xmm11,xmm12 + pxor xmm2,xmm4 + pxor xmm7,xmm0 + movdqu XMMWORD PTR[rdi],xmm6 + movdqu xmm6,XMMWORD PTR[64+rsi] + movdqu XMMWORD PTR[16+rdi],xmm11 + movdqu xmm11,XMMWORD PTR[80+rsi] + movdqu XMMWORD PTR[32+rdi],xmm2 + movdqu xmm2,XMMWORD PTR[96+rsi] + movdqu XMMWORD PTR[48+rdi],xmm7 + movdqu xmm7,XMMWORD PTR[112+rsi] + lea rsi,QWORD PTR[128+rsi] + pxor xmm6,XMMWORD PTR[16+rsp] + pxor xmm11,xmm13 + pxor xmm2,xmm5 + pxor xmm7,xmm1 + movdqu XMMWORD PTR[64+rdi],xmm6 + movdqu xmm6,XMMWORD PTR[rsi] + movdqu XMMWORD PTR[80+rdi],xmm11 + movdqu xmm11,XMMWORD PTR[16+rsi] + movdqu XMMWORD PTR[96+rdi],xmm2 + movdqu xmm2,XMMWORD PTR[32+rsi] + movdqu XMMWORD PTR[112+rdi],xmm7 + lea rdi,QWORD PTR[128+rdi] + movdqu xmm7,XMMWORD PTR[48+rsi] + pxor xmm6,XMMWORD PTR[32+rsp] + pxor xmm11,xmm10 + pxor xmm2,xmm14 + pxor xmm7,xmm8 + movdqu XMMWORD PTR[rdi],xmm6 + movdqu xmm6,XMMWORD PTR[64+rsi] + movdqu XMMWORD PTR[16+rdi],xmm11 + movdqu xmm11,XMMWORD PTR[80+rsi] + movdqu XMMWORD PTR[32+rdi],xmm2 + movdqu xmm2,XMMWORD PTR[96+rsi] + movdqu XMMWORD PTR[48+rdi],xmm7 + movdqu xmm7,XMMWORD PTR[112+rsi] + lea rsi,QWORD PTR[128+rsi] + pxor xmm6,XMMWORD PTR[48+rsp] + pxor xmm11,xmm15 + pxor xmm2,xmm9 + pxor xmm7,xmm3 + movdqu XMMWORD PTR[64+rdi],xmm6 + movdqu XMMWORD PTR[80+rdi],xmm11 + movdqu XMMWORD PTR[96+rdi],xmm2 + movdqu XMMWORD PTR[112+rdi],xmm7 + lea rdi,QWORD PTR[128+rdi] + sub rdx,64*4 + jnz $L$chacha20_loop_outer4x + jmp $L$done4x + +$L$tail4x:: + cmp rdx,192 + jae $L$192_or_more4x + cmp rdx,128 + jae $L$128_or_more4x + cmp rdx,64 + jae $L$64_or_more4x + xor r9,r9 + movdqa XMMWORD PTR[16+rsp],xmm12 + movdqa XMMWORD PTR[32+rsp],xmm4 + movdqa XMMWORD PTR[48+rsp],xmm0 + jmp $L$chacha20_loop_tail4x + +ALIGN 32 +$L$64_or_more4x:: + movdqu xmm6,XMMWORD PTR[rsi] + movdqu xmm11,XMMWORD PTR[16+rsi] + movdqu xmm2,XMMWORD PTR[32+rsi] + movdqu xmm7,XMMWORD PTR[48+rsi] + pxor xmm6,XMMWORD PTR[rsp] + pxor xmm11,xmm12 + pxor xmm2,xmm4 + pxor xmm7,xmm0 + movdqu XMMWORD PTR[rdi],xmm6 + movdqu XMMWORD PTR[16+rdi],xmm11 + movdqu XMMWORD PTR[32+rdi],xmm2 + movdqu XMMWORD PTR[48+rdi],xmm7 + je $L$done4x + movdqa xmm6,XMMWORD PTR[16+rsp] + lea rsi,QWORD PTR[64+rsi] + xor r9,r9 + movdqa XMMWORD PTR[rsp],xmm6 + movdqa XMMWORD PTR[16+rsp],xmm13 + lea rdi,QWORD PTR[64+rdi] + movdqa XMMWORD PTR[32+rsp],xmm5 + sub rdx,64 + movdqa XMMWORD PTR[48+rsp],xmm1 + jmp $L$chacha20_loop_tail4x + +ALIGN 32 +$L$128_or_more4x:: + movdqu xmm6,XMMWORD PTR[rsi] + movdqu xmm11,XMMWORD PTR[16+rsi] + movdqu xmm2,XMMWORD PTR[32+rsi] + movdqu xmm7,XMMWORD PTR[48+rsi] + pxor xmm6,XMMWORD PTR[rsp] + pxor xmm11,xmm12 + pxor xmm2,xmm4 + pxor xmm7,xmm0 + movdqu XMMWORD PTR[rdi],xmm6 + movdqu xmm6,XMMWORD PTR[64+rsi] + movdqu XMMWORD PTR[16+rdi],xmm11 + movdqu xmm11,XMMWORD PTR[80+rsi] + movdqu XMMWORD PTR[32+rdi],xmm2 + movdqu xmm2,XMMWORD PTR[96+rsi] + movdqu XMMWORD PTR[48+rdi],xmm7 + movdqu xmm7,XMMWORD PTR[112+rsi] + pxor xmm6,XMMWORD PTR[16+rsp] + pxor xmm11,xmm13 + pxor xmm2,xmm5 + pxor xmm7,xmm1 + movdqu XMMWORD PTR[64+rdi],xmm6 + movdqu XMMWORD PTR[80+rdi],xmm11 + movdqu XMMWORD PTR[96+rdi],xmm2 + movdqu XMMWORD PTR[112+rdi],xmm7 + je $L$done4x + movdqa xmm6,XMMWORD PTR[32+rsp] + lea rsi,QWORD PTR[128+rsi] + xor r9,r9 + movdqa XMMWORD PTR[rsp],xmm6 + movdqa XMMWORD PTR[16+rsp],xmm10 + lea rdi,QWORD PTR[128+rdi] + movdqa XMMWORD PTR[32+rsp],xmm14 + sub rdx,128 + movdqa XMMWORD PTR[48+rsp],xmm8 + jmp $L$chacha20_loop_tail4x + +ALIGN 32 +$L$192_or_more4x:: + movdqu xmm6,XMMWORD PTR[rsi] + movdqu xmm11,XMMWORD PTR[16+rsi] + movdqu xmm2,XMMWORD PTR[32+rsi] + movdqu xmm7,XMMWORD PTR[48+rsi] + pxor xmm6,XMMWORD PTR[rsp] + pxor xmm11,xmm12 + pxor xmm2,xmm4 + pxor xmm7,xmm0 + movdqu XMMWORD PTR[rdi],xmm6 + movdqu xmm6,XMMWORD PTR[64+rsi] + movdqu XMMWORD PTR[16+rdi],xmm11 + movdqu xmm11,XMMWORD PTR[80+rsi] + movdqu XMMWORD PTR[32+rdi],xmm2 + movdqu xmm2,XMMWORD PTR[96+rsi] + movdqu XMMWORD PTR[48+rdi],xmm7 + movdqu xmm7,XMMWORD PTR[112+rsi] + lea rsi,QWORD PTR[128+rsi] + pxor xmm6,XMMWORD PTR[16+rsp] + pxor xmm11,xmm13 + pxor xmm2,xmm5 + pxor xmm7,xmm1 + movdqu XMMWORD PTR[64+rdi],xmm6 + movdqu xmm6,XMMWORD PTR[rsi] + movdqu XMMWORD PTR[80+rdi],xmm11 + movdqu xmm11,XMMWORD PTR[16+rsi] + movdqu XMMWORD PTR[96+rdi],xmm2 + movdqu xmm2,XMMWORD PTR[32+rsi] + movdqu XMMWORD PTR[112+rdi],xmm7 + lea rdi,QWORD PTR[128+rdi] + movdqu xmm7,XMMWORD PTR[48+rsi] + pxor xmm6,XMMWORD PTR[32+rsp] + pxor xmm11,xmm10 + pxor xmm2,xmm14 + pxor xmm7,xmm8 + movdqu XMMWORD PTR[rdi],xmm6 + movdqu XMMWORD PTR[16+rdi],xmm11 + movdqu XMMWORD PTR[32+rdi],xmm2 + movdqu XMMWORD PTR[48+rdi],xmm7 + je $L$done4x + movdqa xmm6,XMMWORD PTR[48+rsp] + lea rsi,QWORD PTR[64+rsi] + xor r9,r9 + movdqa XMMWORD PTR[rsp],xmm6 + movdqa XMMWORD PTR[16+rsp],xmm15 + lea rdi,QWORD PTR[64+rdi] + movdqa XMMWORD PTR[32+rsp],xmm9 + sub rdx,192 + movdqa XMMWORD PTR[48+rsp],xmm3 + +$L$chacha20_loop_tail4x:: + movzx eax,BYTE PTR[r9*1+rsi] + movzx ecx,BYTE PTR[r9*1+rsp] + lea r9,QWORD PTR[1+r9] + xor eax,ecx + mov BYTE PTR[((-1))+r9*1+rdi],al + dec rdx + jnz $L$chacha20_loop_tail4x + +$L$done4x:: + movaps xmm6,XMMWORD PTR[((-168))+r10] + movaps xmm7,XMMWORD PTR[((-152))+r10] + movaps xmm8,XMMWORD PTR[((-136))+r10] + movaps xmm9,XMMWORD PTR[((-120))+r10] + movaps xmm10,XMMWORD PTR[((-104))+r10] + movaps xmm11,XMMWORD PTR[((-88))+r10] + movaps xmm12,XMMWORD PTR[((-72))+r10] + movaps xmm13,XMMWORD PTR[((-56))+r10] + movaps xmm14,XMMWORD PTR[((-40))+r10] + movaps xmm15,XMMWORD PTR[((-24))+r10] + lea rsp,QWORD PTR[r10] + +$L$4x_epilogue:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret + +$L$SEH_end_chacha20_4x:: +chacha20_4x ENDP +PUBLIC ChaCha20AVX2 + +ALIGN 32 +ChaCha20AVX2 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_chacha20_avx2:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD PTR[40+rsp] + +$L$chacha20_8x:: + mov r10,rsp + sub rsp,0280h+168 + and rsp,-32 + movaps XMMWORD PTR[(-168)+r10],xmm6 + movaps XMMWORD PTR[(-152)+r10],xmm7 + movaps XMMWORD PTR[(-136)+r10],xmm8 + movaps XMMWORD PTR[(-120)+r10],xmm9 + movaps XMMWORD PTR[(-104)+r10],xmm10 + movaps XMMWORD PTR[(-88)+r10],xmm11 + movaps XMMWORD PTR[(-72)+r10],xmm12 + movaps XMMWORD PTR[(-56)+r10],xmm13 + movaps XMMWORD PTR[(-40)+r10],xmm14 + movaps XMMWORD PTR[(-24)+r10],xmm15 +$L$avx2_body:: + vzeroupper + vbroadcasti128 ymm11,XMMWORD PTR[$L$sigma] + vbroadcasti128 ymm3,XMMWORD PTR[rcx] + vbroadcasti128 ymm15,XMMWORD PTR[16+rcx] + vbroadcasti128 ymm7,XMMWORD PTR[r8] + lea rcx,QWORD PTR[256+rsp] + lea rax,QWORD PTR[512+rsp] + lea r9,QWORD PTR[$L$rot16] + lea r11,QWORD PTR[$L$rot24] + vpshufd ymm8,ymm11,000h + vpshufd ymm9,ymm11,055h + vmovdqa YMMWORD PTR[(128-256)+rcx],ymm8 + vpshufd ymm10,ymm11,0aah + vmovdqa YMMWORD PTR[(160-256)+rcx],ymm9 + vpshufd ymm11,ymm11,0ffh + vmovdqa YMMWORD PTR[(192-256)+rcx],ymm10 + vmovdqa YMMWORD PTR[(224-256)+rcx],ymm11 + vpshufd ymm0,ymm3,000h + vpshufd ymm1,ymm3,055h + vmovdqa YMMWORD PTR[(256-256)+rcx],ymm0 + vpshufd ymm2,ymm3,0aah + vmovdqa YMMWORD PTR[(288-256)+rcx],ymm1 + vpshufd ymm3,ymm3,0ffh + vmovdqa YMMWORD PTR[(320-256)+rcx],ymm2 + vmovdqa YMMWORD PTR[(352-256)+rcx],ymm3 + vpshufd ymm12,ymm15,000h + vpshufd ymm13,ymm15,055h + vmovdqa YMMWORD PTR[(384-512)+rax],ymm12 + vpshufd ymm14,ymm15,0aah + vmovdqa YMMWORD PTR[(416-512)+rax],ymm13 + vpshufd ymm15,ymm15,0ffh + vmovdqa YMMWORD PTR[(448-512)+rax],ymm14 + vmovdqa YMMWORD PTR[(480-512)+rax],ymm15 + vpshufd ymm4,ymm7,000h + vpshufd ymm5,ymm7,055h + vpaddd ymm4,ymm4,YMMWORD PTR[$L$incy] + vpshufd ymm6,ymm7,0aah + vmovdqa YMMWORD PTR[(544-512)+rax],ymm5 + vpshufd ymm7,ymm7,0ffh + vmovdqa YMMWORD PTR[(576-512)+rax],ymm6 + vmovdqa YMMWORD PTR[(608-512)+rax],ymm7 + jmp $L$chacha20_loop_enter8x + +ALIGN 32 +$L$chacha20_loop_outer8x:: + vmovdqa ymm8,YMMWORD PTR[((128-256))+rcx] + vmovdqa ymm9,YMMWORD PTR[((160-256))+rcx] + vmovdqa ymm10,YMMWORD PTR[((192-256))+rcx] + vmovdqa ymm11,YMMWORD PTR[((224-256))+rcx] + vmovdqa ymm0,YMMWORD PTR[((256-256))+rcx] + vmovdqa ymm1,YMMWORD PTR[((288-256))+rcx] + vmovdqa ymm2,YMMWORD PTR[((320-256))+rcx] + vmovdqa ymm3,YMMWORD PTR[((352-256))+rcx] + vmovdqa ymm12,YMMWORD PTR[((384-512))+rax] + vmovdqa ymm13,YMMWORD PTR[((416-512))+rax] + vmovdqa ymm14,YMMWORD PTR[((448-512))+rax] + vmovdqa ymm15,YMMWORD PTR[((480-512))+rax] + vmovdqa ymm4,YMMWORD PTR[((512-512))+rax] + vmovdqa ymm5,YMMWORD PTR[((544-512))+rax] + vmovdqa ymm6,YMMWORD PTR[((576-512))+rax] + vmovdqa ymm7,YMMWORD PTR[((608-512))+rax] + vpaddd ymm4,ymm4,YMMWORD PTR[$L$eight] + +$L$chacha20_loop_enter8x:: + vmovdqa YMMWORD PTR[64+rsp],ymm14 + vmovdqa YMMWORD PTR[96+rsp],ymm15 + vbroadcasti128 ymm15,XMMWORD PTR[r9] + vmovdqa YMMWORD PTR[(512-512)+rax],ymm4 + mov eax,10 + jmp $L$chacha20_loop8x + +ALIGN 32 +$L$chacha20_loop8x:: + vpaddd ymm8,ymm8,ymm0 + vpxor ymm4,ymm8,ymm4 + vpshufb ymm4,ymm4,ymm15 + vpaddd ymm9,ymm9,ymm1 + vpxor ymm5,ymm9,ymm5 + vpshufb ymm5,ymm5,ymm15 + vpaddd ymm12,ymm12,ymm4 + vpxor ymm0,ymm12,ymm0 + vpslld ymm14,ymm0,12 + vpsrld ymm0,ymm0,20 + vpor ymm0,ymm14,ymm0 + vbroadcasti128 ymm14,XMMWORD PTR[r11] + vpaddd ymm13,ymm13,ymm5 + vpxor ymm1,ymm13,ymm1 + vpslld ymm15,ymm1,12 + vpsrld ymm1,ymm1,20 + vpor ymm1,ymm15,ymm1 + vpaddd ymm8,ymm8,ymm0 + vpxor ymm4,ymm8,ymm4 + vpshufb ymm4,ymm4,ymm14 + vpaddd ymm9,ymm9,ymm1 + vpxor ymm5,ymm9,ymm5 + vpshufb ymm5,ymm5,ymm14 + vpaddd ymm12,ymm12,ymm4 + vpxor ymm0,ymm12,ymm0 + vpslld ymm15,ymm0,7 + vpsrld ymm0,ymm0,25 + vpor ymm0,ymm15,ymm0 + vbroadcasti128 ymm15,XMMWORD PTR[r9] + vpaddd ymm13,ymm13,ymm5 + vpxor ymm1,ymm13,ymm1 + vpslld ymm14,ymm1,7 + vpsrld ymm1,ymm1,25 + vpor ymm1,ymm14,ymm1 + vmovdqa YMMWORD PTR[rsp],ymm12 + vmovdqa YMMWORD PTR[32+rsp],ymm13 + vmovdqa ymm12,YMMWORD PTR[64+rsp] + vmovdqa ymm13,YMMWORD PTR[96+rsp] + vpaddd ymm10,ymm10,ymm2 + vpxor ymm6,ymm10,ymm6 + vpshufb ymm6,ymm6,ymm15 + vpaddd ymm11,ymm11,ymm3 + vpxor ymm7,ymm11,ymm7 + vpshufb ymm7,ymm7,ymm15 + vpaddd ymm12,ymm12,ymm6 + vpxor ymm2,ymm12,ymm2 + vpslld ymm14,ymm2,12 + vpsrld ymm2,ymm2,20 + vpor ymm2,ymm14,ymm2 + vbroadcasti128 ymm14,XMMWORD PTR[r11] + vpaddd ymm13,ymm13,ymm7 + vpxor ymm3,ymm13,ymm3 + vpslld ymm15,ymm3,12 + vpsrld ymm3,ymm3,20 + vpor ymm3,ymm15,ymm3 + vpaddd ymm10,ymm10,ymm2 + vpxor ymm6,ymm10,ymm6 + vpshufb ymm6,ymm6,ymm14 + vpaddd ymm11,ymm11,ymm3 + vpxor ymm7,ymm11,ymm7 + vpshufb ymm7,ymm7,ymm14 + vpaddd ymm12,ymm12,ymm6 + vpxor ymm2,ymm12,ymm2 + vpslld ymm15,ymm2,7 + vpsrld ymm2,ymm2,25 + vpor ymm2,ymm15,ymm2 + vbroadcasti128 ymm15,XMMWORD PTR[r9] + vpaddd ymm13,ymm13,ymm7 + vpxor ymm3,ymm13,ymm3 + vpslld ymm14,ymm3,7 + vpsrld ymm3,ymm3,25 + vpor ymm3,ymm14,ymm3 + vpaddd ymm8,ymm8,ymm1 + vpxor ymm7,ymm8,ymm7 + vpshufb ymm7,ymm7,ymm15 + vpaddd ymm9,ymm9,ymm2 + vpxor ymm4,ymm9,ymm4 + vpshufb ymm4,ymm4,ymm15 + vpaddd ymm12,ymm12,ymm7 + vpxor ymm1,ymm12,ymm1 + vpslld ymm14,ymm1,12 + vpsrld ymm1,ymm1,20 + vpor ymm1,ymm14,ymm1 + vbroadcasti128 ymm14,XMMWORD PTR[r11] + vpaddd ymm13,ymm13,ymm4 + vpxor ymm2,ymm13,ymm2 + vpslld ymm15,ymm2,12 + vpsrld ymm2,ymm2,20 + vpor ymm2,ymm15,ymm2 + vpaddd ymm8,ymm8,ymm1 + vpxor ymm7,ymm8,ymm7 + vpshufb ymm7,ymm7,ymm14 + vpaddd ymm9,ymm9,ymm2 + vpxor ymm4,ymm9,ymm4 + vpshufb ymm4,ymm4,ymm14 + vpaddd ymm12,ymm12,ymm7 + vpxor ymm1,ymm12,ymm1 + vpslld ymm15,ymm1,7 + vpsrld ymm1,ymm1,25 + vpor ymm1,ymm15,ymm1 + vbroadcasti128 ymm15,XMMWORD PTR[r9] + vpaddd ymm13,ymm13,ymm4 + vpxor ymm2,ymm13,ymm2 + vpslld ymm14,ymm2,7 + vpsrld ymm2,ymm2,25 + vpor ymm2,ymm14,ymm2 + vmovdqa YMMWORD PTR[64+rsp],ymm12 + vmovdqa YMMWORD PTR[96+rsp],ymm13 + vmovdqa ymm12,YMMWORD PTR[rsp] + vmovdqa ymm13,YMMWORD PTR[32+rsp] + vpaddd ymm10,ymm10,ymm3 + vpxor ymm5,ymm10,ymm5 + vpshufb ymm5,ymm5,ymm15 + vpaddd ymm11,ymm11,ymm0 + vpxor ymm6,ymm11,ymm6 + vpshufb ymm6,ymm6,ymm15 + vpaddd ymm12,ymm12,ymm5 + vpxor ymm3,ymm12,ymm3 + vpslld ymm14,ymm3,12 + vpsrld ymm3,ymm3,20 + vpor ymm3,ymm14,ymm3 + vbroadcasti128 ymm14,XMMWORD PTR[r11] + vpaddd ymm13,ymm13,ymm6 + vpxor ymm0,ymm13,ymm0 + vpslld ymm15,ymm0,12 + vpsrld ymm0,ymm0,20 + vpor ymm0,ymm15,ymm0 + vpaddd ymm10,ymm10,ymm3 + vpxor ymm5,ymm10,ymm5 + vpshufb ymm5,ymm5,ymm14 + vpaddd ymm11,ymm11,ymm0 + vpxor ymm6,ymm11,ymm6 + vpshufb ymm6,ymm6,ymm14 + vpaddd ymm12,ymm12,ymm5 + vpxor ymm3,ymm12,ymm3 + vpslld ymm15,ymm3,7 + vpsrld ymm3,ymm3,25 + vpor ymm3,ymm15,ymm3 + vbroadcasti128 ymm15,XMMWORD PTR[r9] + vpaddd ymm13,ymm13,ymm6 + vpxor ymm0,ymm13,ymm0 + vpslld ymm14,ymm0,7 + vpsrld ymm0,ymm0,25 + vpor ymm0,ymm14,ymm0 + dec eax + jnz $L$chacha20_loop8x + lea rax,QWORD PTR[512+rsp] + vpaddd ymm8,ymm8,YMMWORD PTR[((128-256))+rcx] + vpaddd ymm9,ymm9,YMMWORD PTR[((160-256))+rcx] + vpaddd ymm10,ymm10,YMMWORD PTR[((192-256))+rcx] + vpaddd ymm11,ymm11,YMMWORD PTR[((224-256))+rcx] + vpunpckldq ymm14,ymm8,ymm9 + vpunpckldq ymm15,ymm10,ymm11 + vpunpckhdq ymm8,ymm8,ymm9 + vpunpckhdq ymm10,ymm10,ymm11 + vpunpcklqdq ymm9,ymm14,ymm15 + vpunpckhqdq ymm14,ymm14,ymm15 + vpunpcklqdq ymm11,ymm8,ymm10 + vpunpckhqdq ymm8,ymm8,ymm10 + vpaddd ymm0,ymm0,YMMWORD PTR[((256-256))+rcx] + vpaddd ymm1,ymm1,YMMWORD PTR[((288-256))+rcx] + vpaddd ymm2,ymm2,YMMWORD PTR[((320-256))+rcx] + vpaddd ymm3,ymm3,YMMWORD PTR[((352-256))+rcx] + vpunpckldq ymm10,ymm0,ymm1 + vpunpckldq ymm15,ymm2,ymm3 + vpunpckhdq ymm0,ymm0,ymm1 + vpunpckhdq ymm2,ymm2,ymm3 + vpunpcklqdq ymm1,ymm10,ymm15 + vpunpckhqdq ymm10,ymm10,ymm15 + vpunpcklqdq ymm3,ymm0,ymm2 + vpunpckhqdq ymm0,ymm0,ymm2 + vperm2i128 ymm15,ymm9,ymm1,020h + vperm2i128 ymm1,ymm9,ymm1,031h + vperm2i128 ymm9,ymm14,ymm10,020h + vperm2i128 ymm10,ymm14,ymm10,031h + vperm2i128 ymm14,ymm11,ymm3,020h + vperm2i128 ymm3,ymm11,ymm3,031h + vperm2i128 ymm11,ymm8,ymm0,020h + vperm2i128 ymm0,ymm8,ymm0,031h + vmovdqa YMMWORD PTR[rsp],ymm15 + vmovdqa YMMWORD PTR[32+rsp],ymm9 + vmovdqa ymm15,YMMWORD PTR[64+rsp] + vmovdqa ymm9,YMMWORD PTR[96+rsp] + vpaddd ymm12,ymm12,YMMWORD PTR[((384-512))+rax] + vpaddd ymm13,ymm13,YMMWORD PTR[((416-512))+rax] + vpaddd ymm15,ymm15,YMMWORD PTR[((448-512))+rax] + vpaddd ymm9,ymm9,YMMWORD PTR[((480-512))+rax] + vpunpckldq ymm2,ymm12,ymm13 + vpunpckldq ymm8,ymm15,ymm9 + vpunpckhdq ymm12,ymm12,ymm13 + vpunpckhdq ymm15,ymm15,ymm9 + vpunpcklqdq ymm13,ymm2,ymm8 + vpunpckhqdq ymm2,ymm2,ymm8 + vpunpcklqdq ymm9,ymm12,ymm15 + vpunpckhqdq ymm12,ymm12,ymm15 + vpaddd ymm4,ymm4,YMMWORD PTR[((512-512))+rax] + vpaddd ymm5,ymm5,YMMWORD PTR[((544-512))+rax] + vpaddd ymm6,ymm6,YMMWORD PTR[((576-512))+rax] + vpaddd ymm7,ymm7,YMMWORD PTR[((608-512))+rax] + vpunpckldq ymm15,ymm4,ymm5 + vpunpckldq ymm8,ymm6,ymm7 + vpunpckhdq ymm4,ymm4,ymm5 + vpunpckhdq ymm6,ymm6,ymm7 + vpunpcklqdq ymm5,ymm15,ymm8 + vpunpckhqdq ymm15,ymm15,ymm8 + vpunpcklqdq ymm7,ymm4,ymm6 + vpunpckhqdq ymm4,ymm4,ymm6 + vperm2i128 ymm8,ymm13,ymm5,020h + vperm2i128 ymm5,ymm13,ymm5,031h + vperm2i128 ymm13,ymm2,ymm15,020h + vperm2i128 ymm15,ymm2,ymm15,031h + vperm2i128 ymm2,ymm9,ymm7,020h + vperm2i128 ymm7,ymm9,ymm7,031h + vperm2i128 ymm9,ymm12,ymm4,020h + vperm2i128 ymm4,ymm12,ymm4,031h + vmovdqa ymm6,YMMWORD PTR[rsp] + vmovdqa ymm12,YMMWORD PTR[32+rsp] + cmp rdx,64*8 + jb $L$tail8x + vpxor ymm6,ymm6,YMMWORD PTR[rsi] + vpxor ymm8,ymm8,YMMWORD PTR[32+rsi] + vpxor ymm1,ymm1,YMMWORD PTR[64+rsi] + vpxor ymm5,ymm5,YMMWORD PTR[96+rsi] + lea rsi,QWORD PTR[128+rsi] + vmovdqu YMMWORD PTR[rdi],ymm6 + vmovdqu YMMWORD PTR[32+rdi],ymm8 + vmovdqu YMMWORD PTR[64+rdi],ymm1 + vmovdqu YMMWORD PTR[96+rdi],ymm5 + lea rdi,QWORD PTR[128+rdi] + vpxor ymm12,ymm12,YMMWORD PTR[rsi] + vpxor ymm13,ymm13,YMMWORD PTR[32+rsi] + vpxor ymm10,ymm10,YMMWORD PTR[64+rsi] + vpxor ymm15,ymm15,YMMWORD PTR[96+rsi] + lea rsi,QWORD PTR[128+rsi] + vmovdqu YMMWORD PTR[rdi],ymm12 + vmovdqu YMMWORD PTR[32+rdi],ymm13 + vmovdqu YMMWORD PTR[64+rdi],ymm10 + vmovdqu YMMWORD PTR[96+rdi],ymm15 + lea rdi,QWORD PTR[128+rdi] + vpxor ymm14,ymm14,YMMWORD PTR[rsi] + vpxor ymm2,ymm2,YMMWORD PTR[32+rsi] + vpxor ymm3,ymm3,YMMWORD PTR[64+rsi] + vpxor ymm7,ymm7,YMMWORD PTR[96+rsi] + lea rsi,QWORD PTR[128+rsi] + vmovdqu YMMWORD PTR[rdi],ymm14 + vmovdqu YMMWORD PTR[32+rdi],ymm2 + vmovdqu YMMWORD PTR[64+rdi],ymm3 + vmovdqu YMMWORD PTR[96+rdi],ymm7 + lea rdi,QWORD PTR[128+rdi] + vpxor ymm11,ymm11,YMMWORD PTR[rsi] + vpxor ymm9,ymm9,YMMWORD PTR[32+rsi] + vpxor ymm0,ymm0,YMMWORD PTR[64+rsi] + vpxor ymm4,ymm4,YMMWORD PTR[96+rsi] + lea rsi,QWORD PTR[128+rsi] + vmovdqu YMMWORD PTR[rdi],ymm11 + vmovdqu YMMWORD PTR[32+rdi],ymm9 + vmovdqu YMMWORD PTR[64+rdi],ymm0 + vmovdqu YMMWORD PTR[96+rdi],ymm4 + lea rdi,QWORD PTR[128+rdi] + sub rdx,64*8 + jnz $L$chacha20_loop_outer8x + jmp $L$done8x + +$L$tail8x:: + cmp rdx,448 + jae $L$448_or_more8x + cmp rdx,384 + jae $L$384_or_more8x + cmp rdx,320 + jae $L$320_or_more8x + cmp rdx,256 + jae $L$256_or_more8x + cmp rdx,192 + jae $L$192_or_more8x + cmp rdx,128 + jae $L$128_or_more8x + cmp rdx,64 + jae $L$64_or_more8x + xor r9,r9 + vmovdqa YMMWORD PTR[rsp],ymm6 + vmovdqa YMMWORD PTR[32+rsp],ymm8 + jmp $L$chacha20_loop_tail8x + +ALIGN 32 +$L$64_or_more8x:: + vpxor ymm6,ymm6,YMMWORD PTR[rsi] + vpxor ymm8,ymm8,YMMWORD PTR[32+rsi] + vmovdqu YMMWORD PTR[rdi],ymm6 + vmovdqu YMMWORD PTR[32+rdi],ymm8 + je $L$done8x + lea rsi,QWORD PTR[64+rsi] + xor r9,r9 + vmovdqa YMMWORD PTR[rsp],ymm1 + lea rdi,QWORD PTR[64+rdi] + sub rdx,64 + vmovdqa YMMWORD PTR[32+rsp],ymm5 + jmp $L$chacha20_loop_tail8x + +ALIGN 32 +$L$128_or_more8x:: + vpxor ymm6,ymm6,YMMWORD PTR[rsi] + vpxor ymm8,ymm8,YMMWORD PTR[32+rsi] + vpxor ymm1,ymm1,YMMWORD PTR[64+rsi] + vpxor ymm5,ymm5,YMMWORD PTR[96+rsi] + vmovdqu YMMWORD PTR[rdi],ymm6 + vmovdqu YMMWORD PTR[32+rdi],ymm8 + vmovdqu YMMWORD PTR[64+rdi],ymm1 + vmovdqu YMMWORD PTR[96+rdi],ymm5 + je $L$done8x + lea rsi,QWORD PTR[128+rsi] + xor r9,r9 + vmovdqa YMMWORD PTR[rsp],ymm12 + lea rdi,QWORD PTR[128+rdi] + sub rdx,128 + vmovdqa YMMWORD PTR[32+rsp],ymm13 + jmp $L$chacha20_loop_tail8x + +ALIGN 32 +$L$192_or_more8x:: + vpxor ymm6,ymm6,YMMWORD PTR[rsi] + vpxor ymm8,ymm8,YMMWORD PTR[32+rsi] + vpxor ymm1,ymm1,YMMWORD PTR[64+rsi] + vpxor ymm5,ymm5,YMMWORD PTR[96+rsi] + vpxor ymm12,ymm12,YMMWORD PTR[128+rsi] + vpxor ymm13,ymm13,YMMWORD PTR[160+rsi] + vmovdqu YMMWORD PTR[rdi],ymm6 + vmovdqu YMMWORD PTR[32+rdi],ymm8 + vmovdqu YMMWORD PTR[64+rdi],ymm1 + vmovdqu YMMWORD PTR[96+rdi],ymm5 + vmovdqu YMMWORD PTR[128+rdi],ymm12 + vmovdqu YMMWORD PTR[160+rdi],ymm13 + je $L$done8x + lea rsi,QWORD PTR[192+rsi] + xor r9,r9 + vmovdqa YMMWORD PTR[rsp],ymm10 + lea rdi,QWORD PTR[192+rdi] + sub rdx,192 + vmovdqa YMMWORD PTR[32+rsp],ymm15 + jmp $L$chacha20_loop_tail8x + +ALIGN 32 +$L$256_or_more8x:: + vpxor ymm6,ymm6,YMMWORD PTR[rsi] + vpxor ymm8,ymm8,YMMWORD PTR[32+rsi] + vpxor ymm1,ymm1,YMMWORD PTR[64+rsi] + vpxor ymm5,ymm5,YMMWORD PTR[96+rsi] + vpxor ymm12,ymm12,YMMWORD PTR[128+rsi] + vpxor ymm13,ymm13,YMMWORD PTR[160+rsi] + vpxor ymm10,ymm10,YMMWORD PTR[192+rsi] + vpxor ymm15,ymm15,YMMWORD PTR[224+rsi] + vmovdqu YMMWORD PTR[rdi],ymm6 + vmovdqu YMMWORD PTR[32+rdi],ymm8 + vmovdqu YMMWORD PTR[64+rdi],ymm1 + vmovdqu YMMWORD PTR[96+rdi],ymm5 + vmovdqu YMMWORD PTR[128+rdi],ymm12 + vmovdqu YMMWORD PTR[160+rdi],ymm13 + vmovdqu YMMWORD PTR[192+rdi],ymm10 + vmovdqu YMMWORD PTR[224+rdi],ymm15 + je $L$done8x + lea rsi,QWORD PTR[256+rsi] + xor r9,r9 + vmovdqa YMMWORD PTR[rsp],ymm14 + lea rdi,QWORD PTR[256+rdi] + sub rdx,256 + vmovdqa YMMWORD PTR[32+rsp],ymm2 + jmp $L$chacha20_loop_tail8x + +ALIGN 32 +$L$320_or_more8x:: + vpxor ymm6,ymm6,YMMWORD PTR[rsi] + vpxor ymm8,ymm8,YMMWORD PTR[32+rsi] + vpxor ymm1,ymm1,YMMWORD PTR[64+rsi] + vpxor ymm5,ymm5,YMMWORD PTR[96+rsi] + vpxor ymm12,ymm12,YMMWORD PTR[128+rsi] + vpxor ymm13,ymm13,YMMWORD PTR[160+rsi] + vpxor ymm10,ymm10,YMMWORD PTR[192+rsi] + vpxor ymm15,ymm15,YMMWORD PTR[224+rsi] + vpxor ymm14,ymm14,YMMWORD PTR[256+rsi] + vpxor ymm2,ymm2,YMMWORD PTR[288+rsi] + vmovdqu YMMWORD PTR[rdi],ymm6 + vmovdqu YMMWORD PTR[32+rdi],ymm8 + vmovdqu YMMWORD PTR[64+rdi],ymm1 + vmovdqu YMMWORD PTR[96+rdi],ymm5 + vmovdqu YMMWORD PTR[128+rdi],ymm12 + vmovdqu YMMWORD PTR[160+rdi],ymm13 + vmovdqu YMMWORD PTR[192+rdi],ymm10 + vmovdqu YMMWORD PTR[224+rdi],ymm15 + vmovdqu YMMWORD PTR[256+rdi],ymm14 + vmovdqu YMMWORD PTR[288+rdi],ymm2 + je $L$done8x + lea rsi,QWORD PTR[320+rsi] + xor r9,r9 + vmovdqa YMMWORD PTR[rsp],ymm3 + lea rdi,QWORD PTR[320+rdi] + sub rdx,320 + vmovdqa YMMWORD PTR[32+rsp],ymm7 + jmp $L$chacha20_loop_tail8x + +ALIGN 32 +$L$384_or_more8x:: + vpxor ymm6,ymm6,YMMWORD PTR[rsi] + vpxor ymm8,ymm8,YMMWORD PTR[32+rsi] + vpxor ymm1,ymm1,YMMWORD PTR[64+rsi] + vpxor ymm5,ymm5,YMMWORD PTR[96+rsi] + vpxor ymm12,ymm12,YMMWORD PTR[128+rsi] + vpxor ymm13,ymm13,YMMWORD PTR[160+rsi] + vpxor ymm10,ymm10,YMMWORD PTR[192+rsi] + vpxor ymm15,ymm15,YMMWORD PTR[224+rsi] + vpxor ymm14,ymm14,YMMWORD PTR[256+rsi] + vpxor ymm2,ymm2,YMMWORD PTR[288+rsi] + vpxor ymm3,ymm3,YMMWORD PTR[320+rsi] + vpxor ymm7,ymm7,YMMWORD PTR[352+rsi] + vmovdqu YMMWORD PTR[rdi],ymm6 + vmovdqu YMMWORD PTR[32+rdi],ymm8 + vmovdqu YMMWORD PTR[64+rdi],ymm1 + vmovdqu YMMWORD PTR[96+rdi],ymm5 + vmovdqu YMMWORD PTR[128+rdi],ymm12 + vmovdqu YMMWORD PTR[160+rdi],ymm13 + vmovdqu YMMWORD PTR[192+rdi],ymm10 + vmovdqu YMMWORD PTR[224+rdi],ymm15 + vmovdqu YMMWORD PTR[256+rdi],ymm14 + vmovdqu YMMWORD PTR[288+rdi],ymm2 + vmovdqu YMMWORD PTR[320+rdi],ymm3 + vmovdqu YMMWORD PTR[352+rdi],ymm7 + je $L$done8x + lea rsi,QWORD PTR[384+rsi] + xor r9,r9 + vmovdqa YMMWORD PTR[rsp],ymm11 + lea rdi,QWORD PTR[384+rdi] + sub rdx,384 + vmovdqa YMMWORD PTR[32+rsp],ymm9 + jmp $L$chacha20_loop_tail8x + +ALIGN 32 +$L$448_or_more8x:: + vpxor ymm6,ymm6,YMMWORD PTR[rsi] + vpxor ymm8,ymm8,YMMWORD PTR[32+rsi] + vpxor ymm1,ymm1,YMMWORD PTR[64+rsi] + vpxor ymm5,ymm5,YMMWORD PTR[96+rsi] + vpxor ymm12,ymm12,YMMWORD PTR[128+rsi] + vpxor ymm13,ymm13,YMMWORD PTR[160+rsi] + vpxor ymm10,ymm10,YMMWORD PTR[192+rsi] + vpxor ymm15,ymm15,YMMWORD PTR[224+rsi] + vpxor ymm14,ymm14,YMMWORD PTR[256+rsi] + vpxor ymm2,ymm2,YMMWORD PTR[288+rsi] + vpxor ymm3,ymm3,YMMWORD PTR[320+rsi] + vpxor ymm7,ymm7,YMMWORD PTR[352+rsi] + vpxor ymm11,ymm11,YMMWORD PTR[384+rsi] + vpxor ymm9,ymm9,YMMWORD PTR[416+rsi] + vmovdqu YMMWORD PTR[rdi],ymm6 + vmovdqu YMMWORD PTR[32+rdi],ymm8 + vmovdqu YMMWORD PTR[64+rdi],ymm1 + vmovdqu YMMWORD PTR[96+rdi],ymm5 + vmovdqu YMMWORD PTR[128+rdi],ymm12 + vmovdqu YMMWORD PTR[160+rdi],ymm13 + vmovdqu YMMWORD PTR[192+rdi],ymm10 + vmovdqu YMMWORD PTR[224+rdi],ymm15 + vmovdqu YMMWORD PTR[256+rdi],ymm14 + vmovdqu YMMWORD PTR[288+rdi],ymm2 + vmovdqu YMMWORD PTR[320+rdi],ymm3 + vmovdqu YMMWORD PTR[352+rdi],ymm7 + vmovdqu YMMWORD PTR[384+rdi],ymm11 + vmovdqu YMMWORD PTR[416+rdi],ymm9 + je $L$done8x + lea rsi,QWORD PTR[448+rsi] + xor r9,r9 + vmovdqa YMMWORD PTR[rsp],ymm0 + lea rdi,QWORD PTR[448+rdi] + sub rdx,448 + vmovdqa YMMWORD PTR[32+rsp],ymm4 + +$L$chacha20_loop_tail8x:: + movzx eax,BYTE PTR[r9*1+rsi] + movzx ecx,BYTE PTR[r9*1+rsp] + lea r9,QWORD PTR[1+r9] + xor eax,ecx + mov BYTE PTR[((-1))+r9*1+rdi],al + dec rdx + jnz $L$chacha20_loop_tail8x + +$L$done8x:: + vzeroall + movaps xmm6,XMMWORD PTR[((-168))+r10] + movaps xmm7,XMMWORD PTR[((-152))+r10] + movaps xmm8,XMMWORD PTR[((-136))+r10] + movaps xmm9,XMMWORD PTR[((-120))+r10] + movaps xmm10,XMMWORD PTR[((-104))+r10] + movaps xmm11,XMMWORD PTR[((-88))+r10] + movaps xmm12,XMMWORD PTR[((-72))+r10] + movaps xmm13,XMMWORD PTR[((-56))+r10] + movaps xmm14,XMMWORD PTR[((-40))+r10] + movaps xmm15,XMMWORD PTR[((-24))+r10] + lea rsp,QWORD PTR[r10] + +$L$avx2_epilogue:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret + +$L$SEH_end_chacha20_avx2:: +ChaCha20AVX2 ENDP +PUBLIC ChaCha20AVX512 + +ALIGN 32 +ChaCha20AVX512 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_chacha20_avx512:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD PTR[40+rsp] + +$L$chacha20_avx512:: + mov r10,rsp + cmp rdx,512 + ja $L$chacha20_16x + sub rsp,64+40 + and rsp,-16 + movaps XMMWORD PTR[(-40)+r10],xmm6 + movaps XMMWORD PTR[(-24)+r10],xmm7 +$L$avx512_body:: + vbroadcasti32x4 zmm0,XMMWORD PTR[$L$sigma] + vbroadcasti32x4 zmm17,XMMWORD PTR[rcx] + vbroadcasti32x4 zmm18,XMMWORD PTR[16+rcx] + vbroadcasti32x4 zmm19,XMMWORD PTR[r8] + vmovdqa32 zmm16,zmm0 + vmovdqa32 zmm20,ZMMWORD PTR[$L$fourz] + vpaddd zmm3,zmm19,ZMMWORD PTR[$L$zeroz] + jmp $L$chacha20_loop_outer_avx512 + +ALIGN 32 +$L$chacha20_loop_outer_avx512:: + vmovdqa32 zmm1,zmm17 + vmovdqa32 zmm2,zmm18 + vmovdqa32 zmm19,zmm3 + mov r8,10 + jmp $L$chacha20_loop_avx512 + +ALIGN 32 +$L$chacha20_loop_avx512:: + vpaddd zmm0,zmm0,zmm1 + vpxord zmm3,zmm3,zmm0 + vprold zmm3,zmm3,16 + vpaddd zmm2,zmm2,zmm3 + vpxord zmm1,zmm1,zmm2 + vprold zmm1,zmm1,12 + vpaddd zmm0,zmm0,zmm1 + vpxord zmm3,zmm3,zmm0 + vprold zmm3,zmm3,8 + vpaddd zmm2,zmm2,zmm3 + vpxord zmm1,zmm1,zmm2 + vprold zmm1,zmm1,7 + vpshufd zmm2,zmm2,78 + vpshufd zmm1,zmm1,57 + vpshufd zmm3,zmm3,147 + vpaddd zmm0,zmm0,zmm1 + vpxord zmm3,zmm3,zmm0 + vprold zmm3,zmm3,16 + vpaddd zmm2,zmm2,zmm3 + vpxord zmm1,zmm1,zmm2 + vprold zmm1,zmm1,12 + vpaddd zmm0,zmm0,zmm1 + vpxord zmm3,zmm3,zmm0 + vprold zmm3,zmm3,8 + vpaddd zmm2,zmm2,zmm3 + vpxord zmm1,zmm1,zmm2 + vprold zmm1,zmm1,7 + vpshufd zmm2,zmm2,78 + vpshufd zmm1,zmm1,147 + vpshufd zmm3,zmm3,57 + dec r8 + jnz $L$chacha20_loop_avx512 + vpaddd zmm0,zmm0,zmm16 + vpaddd zmm1,zmm1,zmm17 + vpaddd zmm2,zmm2,zmm18 + vpaddd zmm3,zmm3,zmm19 + sub rdx,64 + jb $L$tail64_avx512 + vpxor xmm4,xmm0,XMMWORD PTR[rsi] + vpxor xmm5,xmm1,XMMWORD PTR[16+rsi] + vpxor xmm6,xmm2,XMMWORD PTR[32+rsi] + vpxor xmm7,xmm3,XMMWORD PTR[48+rsi] + lea rsi,QWORD PTR[64+rsi] + vmovdqu XMMWORD PTR[rdi],xmm4 + vmovdqu XMMWORD PTR[16+rdi],xmm5 + vmovdqu XMMWORD PTR[32+rdi],xmm6 + vmovdqu XMMWORD PTR[48+rdi],xmm7 + lea rdi,QWORD PTR[64+rdi] + jz $L$done_avx512 + vextracti32x4 xmm4,zmm0,1 + vextracti32x4 xmm5,zmm1,1 + vextracti32x4 xmm6,zmm2,1 + vextracti32x4 xmm7,zmm3,1 + sub rdx,64 + jb $L$tail_avx512 + vpxor xmm4,xmm4,XMMWORD PTR[rsi] + vpxor xmm5,xmm5,XMMWORD PTR[16+rsi] + vpxor xmm6,xmm6,XMMWORD PTR[32+rsi] + vpxor xmm7,xmm7,XMMWORD PTR[48+rsi] + lea rsi,QWORD PTR[64+rsi] + vmovdqu XMMWORD PTR[rdi],xmm4 + vmovdqu XMMWORD PTR[16+rdi],xmm5 + vmovdqu XMMWORD PTR[32+rdi],xmm6 + vmovdqu XMMWORD PTR[48+rdi],xmm7 + lea rdi,QWORD PTR[64+rdi] + jz $L$done_avx512 + vextracti32x4 xmm4,zmm0,2 + vextracti32x4 xmm5,zmm1,2 + vextracti32x4 xmm6,zmm2,2 + vextracti32x4 xmm7,zmm3,2 + sub rdx,64 + jb $L$tail_avx512 + vpxor xmm4,xmm4,XMMWORD PTR[rsi] + vpxor xmm5,xmm5,XMMWORD PTR[16+rsi] + vpxor xmm6,xmm6,XMMWORD PTR[32+rsi] + vpxor xmm7,xmm7,XMMWORD PTR[48+rsi] + lea rsi,QWORD PTR[64+rsi] + vmovdqu XMMWORD PTR[rdi],xmm4 + vmovdqu XMMWORD PTR[16+rdi],xmm5 + vmovdqu XMMWORD PTR[32+rdi],xmm6 + vmovdqu XMMWORD PTR[48+rdi],xmm7 + lea rdi,QWORD PTR[64+rdi] + jz $L$done_avx512 + vextracti32x4 xmm4,zmm0,3 + vextracti32x4 xmm5,zmm1,3 + vextracti32x4 xmm6,zmm2,3 + vextracti32x4 xmm7,zmm3,3 + sub rdx,64 + jb $L$tail_avx512 + vmovdqa32 zmm0,zmm16 + vpaddd zmm3,zmm19,zmm20 + vpxor xmm4,xmm4,XMMWORD PTR[rsi] + vpxor xmm5,xmm5,XMMWORD PTR[16+rsi] + vpxor xmm6,xmm6,XMMWORD PTR[32+rsi] + vpxor xmm7,xmm7,XMMWORD PTR[48+rsi] + lea rsi,QWORD PTR[64+rsi] + vmovdqu XMMWORD PTR[rdi],xmm4 + vmovdqu XMMWORD PTR[16+rdi],xmm5 + vmovdqu XMMWORD PTR[32+rdi],xmm6 + vmovdqu XMMWORD PTR[48+rdi],xmm7 + lea rdi,QWORD PTR[64+rdi] + jnz $L$chacha20_loop_outer_avx512 + jmp $L$done_avx512 + +ALIGN 16 +$L$tail64_avx512:: + vmovdqa XMMWORD PTR[rsp],xmm0 + vmovdqa XMMWORD PTR[16+rsp],xmm1 + vmovdqa XMMWORD PTR[32+rsp],xmm2 + vmovdqa XMMWORD PTR[48+rsp],xmm3 + add rdx,64 + jmp $L$chacha20_loop_tail_avx512 + +ALIGN 16 +$L$tail_avx512:: + vmovdqa XMMWORD PTR[rsp],xmm4 + vmovdqa XMMWORD PTR[16+rsp],xmm5 + vmovdqa XMMWORD PTR[32+rsp],xmm6 + vmovdqa XMMWORD PTR[48+rsp],xmm7 + add rdx,64 + +$L$chacha20_loop_tail_avx512:: + movzx eax,BYTE PTR[r8*1+rsi] + movzx ecx,BYTE PTR[r8*1+rsp] + lea r8,QWORD PTR[1+r8] + xor eax,ecx + mov BYTE PTR[((-1))+r8*1+rdi],al + dec rdx + jnz $L$chacha20_loop_tail_avx512 + vmovdqu32 ZMMWORD PTR[rsp],zmm16 + +$L$done_avx512:: + vzeroall + movaps xmm6,XMMWORD PTR[((-40))+r10] + movaps xmm7,XMMWORD PTR[((-24))+r10] + lea rsp,QWORD PTR[r10] + +$L$avx512_epilogue:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret + +$L$SEH_end_chacha20_avx512:: +ChaCha20AVX512 ENDP +PUBLIC ChaCha20AVX512VL + +ALIGN 32 +ChaCha20AVX512VL PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_chacha20_avx512vl:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD PTR[40+rsp] + +$L$chacha20_avx512vl:: + mov r10,rsp + cmp rdx,128 + ja $L$chacha20_8xvl + sub rsp,64+40 + movaps XMMWORD PTR[(-40)+r10],xmm6 + movaps XMMWORD PTR[(-24)+r10],xmm7 +$L$avx512vl_body:: + vbroadcasti32x4 ymm0,XMMWORD PTR[$L$sigma] + vbroadcasti32x4 ymm17,XMMWORD PTR[rcx] + vbroadcasti32x4 ymm18,XMMWORD PTR[16+rcx] + vbroadcasti32x4 ymm19,XMMWORD PTR[r8] + vmovdqa32 ymm16,ymm0 + vmovdqa32 ymm20,YMMWORD PTR[$L$twoy] + vpaddd ymm3,ymm19,YMMWORD PTR[$L$zeroz] + jmp $L$chacha20_loop_outer_avx512vl + +ALIGN 32 +$L$chacha20_loop_outer_avx512vl:: + vmovdqa32 ymm1,ymm17 + vmovdqa32 ymm2,ymm18 + vmovdqa32 ymm19,ymm3 + mov r8,10 + jmp $L$chacha20_loop_avx512vl + +ALIGN 32 +$L$chacha20_loop_avx512vl:: + vpaddd ymm0,ymm0,ymm1 + vpxor ymm3,ymm3,ymm0 + vprold ymm3,ymm3,16 + vpaddd ymm2,ymm2,ymm3 + vpxor ymm1,ymm1,ymm2 + vprold ymm1,ymm1,12 + vpaddd ymm0,ymm0,ymm1 + vpxor ymm3,ymm3,ymm0 + vprold ymm3,ymm3,8 + vpaddd ymm2,ymm2,ymm3 + vpxor ymm1,ymm1,ymm2 + vprold ymm1,ymm1,7 + vpshufd ymm2,ymm2,78 + vpshufd ymm1,ymm1,57 + vpshufd ymm3,ymm3,147 + vpaddd ymm0,ymm0,ymm1 + vpxor ymm3,ymm3,ymm0 + vprold ymm3,ymm3,16 + vpaddd ymm2,ymm2,ymm3 + vpxor ymm1,ymm1,ymm2 + vprold ymm1,ymm1,12 + vpaddd ymm0,ymm0,ymm1 + vpxor ymm3,ymm3,ymm0 + vprold ymm3,ymm3,8 + vpaddd ymm2,ymm2,ymm3 + vpxor ymm1,ymm1,ymm2 + vprold ymm1,ymm1,7 + vpshufd ymm2,ymm2,78 + vpshufd ymm1,ymm1,147 + vpshufd ymm3,ymm3,57 + sub r8,1 + jnz $L$chacha20_loop_avx512vl + vpaddd ymm0,ymm0,ymm16 + vpaddd ymm1,ymm1,ymm17 + vpaddd ymm2,ymm2,ymm18 + vpaddd ymm3,ymm3,ymm19 + sub rdx,64 + jb $L$tail64_avx512vl + vpxor xmm4,xmm0,XMMWORD PTR[rsi] + vpxor xmm5,xmm1,XMMWORD PTR[16+rsi] + vpxor xmm6,xmm2,XMMWORD PTR[32+rsi] + vpxor xmm7,xmm3,XMMWORD PTR[48+rsi] + lea rsi,QWORD PTR[64+rsi] + vmovdqu XMMWORD PTR[rdi],xmm4 + vmovdqu XMMWORD PTR[16+rdi],xmm5 + vmovdqu XMMWORD PTR[32+rdi],xmm6 + vmovdqu XMMWORD PTR[48+rdi],xmm7 + lea rdi,QWORD PTR[64+rdi] + jz $L$done_avx512vl + vextracti128 xmm4,ymm0,1 + vextracti128 xmm5,ymm1,1 + vextracti128 xmm6,ymm2,1 + vextracti128 xmm7,ymm3,1 + sub rdx,64 + jb $L$tail_avx512vl + vmovdqa32 ymm0,ymm16 + vpaddd ymm3,ymm19,ymm20 + vpxor xmm4,xmm4,XMMWORD PTR[rsi] + vpxor xmm5,xmm5,XMMWORD PTR[16+rsi] + vpxor xmm6,xmm6,XMMWORD PTR[32+rsi] + vpxor xmm7,xmm7,XMMWORD PTR[48+rsi] + lea rsi,QWORD PTR[64+rsi] + vmovdqu XMMWORD PTR[rdi],xmm4 + vmovdqu XMMWORD PTR[16+rdi],xmm5 + vmovdqu XMMWORD PTR[32+rdi],xmm6 + vmovdqu XMMWORD PTR[48+rdi],xmm7 + lea rdi,QWORD PTR[64+rdi] + jnz $L$chacha20_loop_outer_avx512vl + jmp $L$done_avx512vl + +ALIGN 16 +$L$tail64_avx512vl:: + vmovdqa XMMWORD PTR[rsp],xmm0 + vmovdqa XMMWORD PTR[16+rsp],xmm1 + vmovdqa XMMWORD PTR[32+rsp],xmm2 + vmovdqa XMMWORD PTR[48+rsp],xmm3 + add rdx,64 + jmp $L$chacha20_loop_tail_avx512vl + +ALIGN 16 +$L$tail_avx512vl:: + vmovdqa XMMWORD PTR[rsp],xmm4 + vmovdqa XMMWORD PTR[16+rsp],xmm5 + vmovdqa XMMWORD PTR[32+rsp],xmm6 + vmovdqa XMMWORD PTR[48+rsp],xmm7 + add rdx,64 + +$L$chacha20_loop_tail_avx512vl:: + movzx eax,BYTE PTR[r8*1+rsi] + movzx ecx,BYTE PTR[r8*1+rsp] + lea r8,QWORD PTR[1+r8] + xor eax,ecx + mov BYTE PTR[((-1))+r8*1+rdi],al + dec rdx + jnz $L$chacha20_loop_tail_avx512vl + vmovdqu32 YMMWORD PTR[rsp],ymm16 + vmovdqu32 YMMWORD PTR[32+rsp],ymm16 + +$L$done_avx512vl:: + vzeroall + movaps xmm6,XMMWORD PTR[((-40))+r10] + movaps xmm7,XMMWORD PTR[((-24))+r10] + lea rsp,QWORD PTR[r10] + +$L$avx512vl_epilogue:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret + +$L$SEH_end_chacha20_avx512vl:: +ChaCha20AVX512VL ENDP + +ALIGN 32 +chacha20_16x PROC PRIVATE + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_chacha20_16x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD PTR[40+rsp] + +$L$chacha20_16x:: + mov r10,rsp + sub rsp,64+168 + and rsp,-64 + movaps XMMWORD PTR[(-168)+r10],xmm6 + movaps XMMWORD PTR[(-152)+r10],xmm7 + movaps XMMWORD PTR[(-136)+r10],xmm8 + movaps XMMWORD PTR[(-120)+r10],xmm9 + movaps XMMWORD PTR[(-104)+r10],xmm10 + movaps XMMWORD PTR[(-88)+r10],xmm11 + movaps XMMWORD PTR[(-72)+r10],xmm12 + movaps XMMWORD PTR[(-56)+r10],xmm13 + movaps XMMWORD PTR[(-40)+r10],xmm14 + movaps XMMWORD PTR[(-24)+r10],xmm15 +$L$16x_body:: + vzeroupper + lea r9,QWORD PTR[$L$sigma] + vbroadcasti32x4 zmm3,XMMWORD PTR[r9] + vbroadcasti32x4 zmm7,XMMWORD PTR[rcx] + vbroadcasti32x4 zmm11,XMMWORD PTR[16+rcx] + vbroadcasti32x4 zmm15,XMMWORD PTR[r8] + vpshufd zmm0,zmm3,000h + vpshufd zmm1,zmm3,055h + vpshufd zmm2,zmm3,0aah + vpshufd zmm3,zmm3,0ffh + vmovdqa64 zmm16,zmm0 + vmovdqa64 zmm17,zmm1 + vmovdqa64 zmm18,zmm2 + vmovdqa64 zmm19,zmm3 + vpshufd zmm4,zmm7,000h + vpshufd zmm5,zmm7,055h + vpshufd zmm6,zmm7,0aah + vpshufd zmm7,zmm7,0ffh + vmovdqa64 zmm20,zmm4 + vmovdqa64 zmm21,zmm5 + vmovdqa64 zmm22,zmm6 + vmovdqa64 zmm23,zmm7 + vpshufd zmm8,zmm11,000h + vpshufd zmm9,zmm11,055h + vpshufd zmm10,zmm11,0aah + vpshufd zmm11,zmm11,0ffh + vmovdqa64 zmm24,zmm8 + vmovdqa64 zmm25,zmm9 + vmovdqa64 zmm26,zmm10 + vmovdqa64 zmm27,zmm11 + vpshufd zmm12,zmm15,000h + vpshufd zmm13,zmm15,055h + vpshufd zmm14,zmm15,0aah + vpshufd zmm15,zmm15,0ffh + vpaddd zmm12,zmm12,ZMMWORD PTR[$L$incz] + vmovdqa64 zmm28,zmm12 + vmovdqa64 zmm29,zmm13 + vmovdqa64 zmm30,zmm14 + vmovdqa64 zmm31,zmm15 + mov eax,10 + jmp $L$chacha20_loop16x + +ALIGN 32 +$L$chacha20_loop_outer16x:: + vpbroadcastd zmm0,DWORD PTR[r9] + vpbroadcastd zmm1,DWORD PTR[4+r9] + vpbroadcastd zmm2,DWORD PTR[8+r9] + vpbroadcastd zmm3,DWORD PTR[12+r9] + vpaddd zmm28,zmm28,ZMMWORD PTR[$L$sixteen] + vmovdqa64 zmm4,zmm20 + vmovdqa64 zmm5,zmm21 + vmovdqa64 zmm6,zmm22 + vmovdqa64 zmm7,zmm23 + vmovdqa64 zmm8,zmm24 + vmovdqa64 zmm9,zmm25 + vmovdqa64 zmm10,zmm26 + vmovdqa64 zmm11,zmm27 + vmovdqa64 zmm12,zmm28 + vmovdqa64 zmm13,zmm29 + vmovdqa64 zmm14,zmm30 + vmovdqa64 zmm15,zmm31 + vmovdqa64 zmm16,zmm0 + vmovdqa64 zmm17,zmm1 + vmovdqa64 zmm18,zmm2 + vmovdqa64 zmm19,zmm3 + mov eax,10 + jmp $L$chacha20_loop16x + +ALIGN 32 +$L$chacha20_loop16x:: + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprold zmm12,zmm12,16 + vprold zmm13,zmm13,16 + vprold zmm14,zmm14,16 + vprold zmm15,zmm15,16 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprold zmm4,zmm4,12 + vprold zmm5,zmm5,12 + vprold zmm6,zmm6,12 + vprold zmm7,zmm7,12 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprold zmm12,zmm12,8 + vprold zmm13,zmm13,8 + vprold zmm14,zmm14,8 + vprold zmm15,zmm15,8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprold zmm4,zmm4,7 + vprold zmm5,zmm5,7 + vprold zmm6,zmm6,7 + vprold zmm7,zmm7,7 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprold zmm15,zmm15,16 + vprold zmm12,zmm12,16 + vprold zmm13,zmm13,16 + vprold zmm14,zmm14,16 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprold zmm5,zmm5,12 + vprold zmm6,zmm6,12 + vprold zmm7,zmm7,12 + vprold zmm4,zmm4,12 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprold zmm15,zmm15,8 + vprold zmm12,zmm12,8 + vprold zmm13,zmm13,8 + vprold zmm14,zmm14,8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprold zmm5,zmm5,7 + vprold zmm6,zmm6,7 + vprold zmm7,zmm7,7 + vprold zmm4,zmm4,7 + dec eax + jnz $L$chacha20_loop16x + vpaddd zmm0,zmm0,zmm16 + vpaddd zmm1,zmm1,zmm17 + vpaddd zmm2,zmm2,zmm18 + vpaddd zmm3,zmm3,zmm19 + vpunpckldq zmm18,zmm0,zmm1 + vpunpckldq zmm19,zmm2,zmm3 + vpunpckhdq zmm0,zmm0,zmm1 + vpunpckhdq zmm2,zmm2,zmm3 + vpunpcklqdq zmm1,zmm18,zmm19 + vpunpckhqdq zmm18,zmm18,zmm19 + vpunpcklqdq zmm3,zmm0,zmm2 + vpunpckhqdq zmm0,zmm0,zmm2 + vpaddd zmm4,zmm4,zmm20 + vpaddd zmm5,zmm5,zmm21 + vpaddd zmm6,zmm6,zmm22 + vpaddd zmm7,zmm7,zmm23 + vpunpckldq zmm2,zmm4,zmm5 + vpunpckldq zmm19,zmm6,zmm7 + vpunpckhdq zmm4,zmm4,zmm5 + vpunpckhdq zmm6,zmm6,zmm7 + vpunpcklqdq zmm5,zmm2,zmm19 + vpunpckhqdq zmm2,zmm2,zmm19 + vpunpcklqdq zmm7,zmm4,zmm6 + vpunpckhqdq zmm4,zmm4,zmm6 + vshufi32x4 zmm19,zmm1,zmm5,044h + vshufi32x4 zmm5,zmm1,zmm5,0eeh + vshufi32x4 zmm1,zmm18,zmm2,044h + vshufi32x4 zmm2,zmm18,zmm2,0eeh + vshufi32x4 zmm18,zmm3,zmm7,044h + vshufi32x4 zmm7,zmm3,zmm7,0eeh + vshufi32x4 zmm3,zmm0,zmm4,044h + vshufi32x4 zmm4,zmm0,zmm4,0eeh + vpaddd zmm8,zmm8,zmm24 + vpaddd zmm9,zmm9,zmm25 + vpaddd zmm10,zmm10,zmm26 + vpaddd zmm11,zmm11,zmm27 + vpunpckldq zmm6,zmm8,zmm9 + vpunpckldq zmm0,zmm10,zmm11 + vpunpckhdq zmm8,zmm8,zmm9 + vpunpckhdq zmm10,zmm10,zmm11 + vpunpcklqdq zmm9,zmm6,zmm0 + vpunpckhqdq zmm6,zmm6,zmm0 + vpunpcklqdq zmm11,zmm8,zmm10 + vpunpckhqdq zmm8,zmm8,zmm10 + vpaddd zmm12,zmm12,zmm28 + vpaddd zmm13,zmm13,zmm29 + vpaddd zmm14,zmm14,zmm30 + vpaddd zmm15,zmm15,zmm31 + vpunpckldq zmm10,zmm12,zmm13 + vpunpckldq zmm0,zmm14,zmm15 + vpunpckhdq zmm12,zmm12,zmm13 + vpunpckhdq zmm14,zmm14,zmm15 + vpunpcklqdq zmm13,zmm10,zmm0 + vpunpckhqdq zmm10,zmm10,zmm0 + vpunpcklqdq zmm15,zmm12,zmm14 + vpunpckhqdq zmm12,zmm12,zmm14 + vshufi32x4 zmm0,zmm9,zmm13,044h + vshufi32x4 zmm13,zmm9,zmm13,0eeh + vshufi32x4 zmm9,zmm6,zmm10,044h + vshufi32x4 zmm10,zmm6,zmm10,0eeh + vshufi32x4 zmm6,zmm11,zmm15,044h + vshufi32x4 zmm15,zmm11,zmm15,0eeh + vshufi32x4 zmm11,zmm8,zmm12,044h + vshufi32x4 zmm12,zmm8,zmm12,0eeh + vshufi32x4 zmm16,zmm19,zmm0,088h + vshufi32x4 zmm19,zmm19,zmm0,0ddh + vshufi32x4 zmm0,zmm5,zmm13,088h + vshufi32x4 zmm13,zmm5,zmm13,0ddh + vshufi32x4 zmm17,zmm1,zmm9,088h + vshufi32x4 zmm1,zmm1,zmm9,0ddh + vshufi32x4 zmm9,zmm2,zmm10,088h + vshufi32x4 zmm10,zmm2,zmm10,0ddh + vshufi32x4 zmm14,zmm18,zmm6,088h + vshufi32x4 zmm18,zmm18,zmm6,0ddh + vshufi32x4 zmm6,zmm7,zmm15,088h + vshufi32x4 zmm15,zmm7,zmm15,0ddh + vshufi32x4 zmm8,zmm3,zmm11,088h + vshufi32x4 zmm3,zmm3,zmm11,0ddh + vshufi32x4 zmm11,zmm4,zmm12,088h + vshufi32x4 zmm12,zmm4,zmm12,0ddh + cmp rdx,64*16 + jb $L$tail16x + vpxord zmm16,zmm16,ZMMWORD PTR[rsi] + vpxord zmm17,zmm17,ZMMWORD PTR[64+rsi] + vpxord zmm14,zmm14,ZMMWORD PTR[128+rsi] + vpxord zmm8,zmm8,ZMMWORD PTR[192+rsi] + vmovdqu32 ZMMWORD PTR[rdi],zmm16 + vmovdqu32 ZMMWORD PTR[64+rdi],zmm17 + vmovdqu32 ZMMWORD PTR[128+rdi],zmm14 + vmovdqu32 ZMMWORD PTR[192+rdi],zmm8 + vpxord zmm19,zmm19,ZMMWORD PTR[256+rsi] + vpxord zmm1,zmm1,ZMMWORD PTR[320+rsi] + vpxord zmm18,zmm18,ZMMWORD PTR[384+rsi] + vpxord zmm3,zmm3,ZMMWORD PTR[448+rsi] + vmovdqu32 ZMMWORD PTR[256+rdi],zmm19 + vmovdqu32 ZMMWORD PTR[320+rdi],zmm1 + vmovdqu32 ZMMWORD PTR[384+rdi],zmm18 + vmovdqu32 ZMMWORD PTR[448+rdi],zmm3 + vpxord zmm0,zmm0,ZMMWORD PTR[512+rsi] + vpxord zmm9,zmm9,ZMMWORD PTR[576+rsi] + vpxord zmm6,zmm6,ZMMWORD PTR[640+rsi] + vpxord zmm11,zmm11,ZMMWORD PTR[704+rsi] + vmovdqu32 ZMMWORD PTR[512+rdi],zmm0 + vmovdqu32 ZMMWORD PTR[576+rdi],zmm9 + vmovdqu32 ZMMWORD PTR[640+rdi],zmm6 + vmovdqu32 ZMMWORD PTR[704+rdi],zmm11 + vpxord zmm13,zmm13,ZMMWORD PTR[768+rsi] + vpxord zmm10,zmm10,ZMMWORD PTR[832+rsi] + vpxord zmm15,zmm15,ZMMWORD PTR[896+rsi] + vpxord zmm12,zmm12,ZMMWORD PTR[960+rsi] + lea rsi,QWORD PTR[1024+rsi] + vmovdqu32 ZMMWORD PTR[768+rdi],zmm13 + vmovdqu32 ZMMWORD PTR[832+rdi],zmm10 + vmovdqu32 ZMMWORD PTR[896+rdi],zmm15 + vmovdqu32 ZMMWORD PTR[960+rdi],zmm12 + lea rdi,QWORD PTR[1024+rdi] + sub rdx,64*16 + jnz $L$chacha20_loop_outer16x + jmp $L$done16x + +ALIGN 32 +$L$tail16x:: + xor r9,r9 + sub rdi,rsi + cmp rdx,64*1 + jb $L$ess_than_64_16x + vpxord zmm16,zmm16,ZMMWORD PTR[rsi] + vmovdqu32 ZMMWORD PTR[rsi*1+rdi],zmm16 + je $L$done16x + vmovdqa32 zmm16,zmm17 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*2 + jb $L$ess_than_64_16x + vpxord zmm17,zmm17,ZMMWORD PTR[rsi] + vmovdqu32 ZMMWORD PTR[rsi*1+rdi],zmm17 + je $L$done16x + vmovdqa32 zmm16,zmm14 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*3 + jb $L$ess_than_64_16x + vpxord zmm14,zmm14,ZMMWORD PTR[rsi] + vmovdqu32 ZMMWORD PTR[rsi*1+rdi],zmm14 + je $L$done16x + vmovdqa32 zmm16,zmm8 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*4 + jb $L$ess_than_64_16x + vpxord zmm8,zmm8,ZMMWORD PTR[rsi] + vmovdqu32 ZMMWORD PTR[rsi*1+rdi],zmm8 + je $L$done16x + vmovdqa32 zmm16,zmm19 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*5 + jb $L$ess_than_64_16x + vpxord zmm19,zmm19,ZMMWORD PTR[rsi] + vmovdqu32 ZMMWORD PTR[rsi*1+rdi],zmm19 + je $L$done16x + vmovdqa32 zmm16,zmm1 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*6 + jb $L$ess_than_64_16x + vpxord zmm1,zmm1,ZMMWORD PTR[rsi] + vmovdqu32 ZMMWORD PTR[rsi*1+rdi],zmm1 + je $L$done16x + vmovdqa32 zmm16,zmm18 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*7 + jb $L$ess_than_64_16x + vpxord zmm18,zmm18,ZMMWORD PTR[rsi] + vmovdqu32 ZMMWORD PTR[rsi*1+rdi],zmm18 + je $L$done16x + vmovdqa32 zmm16,zmm3 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*8 + jb $L$ess_than_64_16x + vpxord zmm3,zmm3,ZMMWORD PTR[rsi] + vmovdqu32 ZMMWORD PTR[rsi*1+rdi],zmm3 + je $L$done16x + vmovdqa32 zmm16,zmm0 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*9 + jb $L$ess_than_64_16x + vpxord zmm0,zmm0,ZMMWORD PTR[rsi] + vmovdqu32 ZMMWORD PTR[rsi*1+rdi],zmm0 + je $L$done16x + vmovdqa32 zmm16,zmm9 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*10 + jb $L$ess_than_64_16x + vpxord zmm9,zmm9,ZMMWORD PTR[rsi] + vmovdqu32 ZMMWORD PTR[rsi*1+rdi],zmm9 + je $L$done16x + vmovdqa32 zmm16,zmm6 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*11 + jb $L$ess_than_64_16x + vpxord zmm6,zmm6,ZMMWORD PTR[rsi] + vmovdqu32 ZMMWORD PTR[rsi*1+rdi],zmm6 + je $L$done16x + vmovdqa32 zmm16,zmm11 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*12 + jb $L$ess_than_64_16x + vpxord zmm11,zmm11,ZMMWORD PTR[rsi] + vmovdqu32 ZMMWORD PTR[rsi*1+rdi],zmm11 + je $L$done16x + vmovdqa32 zmm16,zmm13 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*13 + jb $L$ess_than_64_16x + vpxord zmm13,zmm13,ZMMWORD PTR[rsi] + vmovdqu32 ZMMWORD PTR[rsi*1+rdi],zmm13 + je $L$done16x + vmovdqa32 zmm16,zmm10 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*14 + jb $L$ess_than_64_16x + vpxord zmm10,zmm10,ZMMWORD PTR[rsi] + vmovdqu32 ZMMWORD PTR[rsi*1+rdi],zmm10 + je $L$done16x + vmovdqa32 zmm16,zmm15 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*15 + jb $L$ess_than_64_16x + vpxord zmm15,zmm15,ZMMWORD PTR[rsi] + vmovdqu32 ZMMWORD PTR[rsi*1+rdi],zmm15 + je $L$done16x + vmovdqa32 zmm16,zmm12 + lea rsi,QWORD PTR[64+rsi] + +$L$ess_than_64_16x:: + vmovdqa32 ZMMWORD PTR[rsp],zmm16 + lea rdi,QWORD PTR[rsi*1+rdi] + and rdx,63 + +$L$chacha20_loop_tail16x:: + movzx eax,BYTE PTR[r9*1+rsi] + movzx ecx,BYTE PTR[r9*1+rsp] + lea r9,QWORD PTR[1+r9] + xor eax,ecx + mov BYTE PTR[((-1))+r9*1+rdi],al + dec rdx + jnz $L$chacha20_loop_tail16x + vpxord zmm16,zmm16,zmm16 + vmovdqa32 ZMMWORD PTR[rsp],zmm16 + +$L$done16x:: + vzeroall + movaps xmm6,XMMWORD PTR[((-168))+r10] + movaps xmm7,XMMWORD PTR[((-152))+r10] + movaps xmm8,XMMWORD PTR[((-136))+r10] + movaps xmm9,XMMWORD PTR[((-120))+r10] + movaps xmm10,XMMWORD PTR[((-104))+r10] + movaps xmm11,XMMWORD PTR[((-88))+r10] + movaps xmm12,XMMWORD PTR[((-72))+r10] + movaps xmm13,XMMWORD PTR[((-56))+r10] + movaps xmm14,XMMWORD PTR[((-40))+r10] + movaps xmm15,XMMWORD PTR[((-24))+r10] + lea rsp,QWORD PTR[r10] + +$L$16x_epilogue:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret + +$L$SEH_end_chacha20_16x:: +chacha20_16x ENDP + +ALIGN 32 +chacha20_8xvl PROC PRIVATE + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_chacha20_8xvl:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD PTR[40+rsp] + +$L$chacha20_8xvl:: + mov r10,rsp + sub rsp,64+168 + and rsp,-64 + movaps XMMWORD PTR[(-168)+r10],xmm6 + movaps XMMWORD PTR[(-152)+r10],xmm7 + movaps XMMWORD PTR[(-136)+r10],xmm8 + movaps XMMWORD PTR[(-120)+r10],xmm9 + movaps XMMWORD PTR[(-104)+r10],xmm10 + movaps XMMWORD PTR[(-88)+r10],xmm11 + movaps XMMWORD PTR[(-72)+r10],xmm12 + movaps XMMWORD PTR[(-56)+r10],xmm13 + movaps XMMWORD PTR[(-40)+r10],xmm14 + movaps XMMWORD PTR[(-24)+r10],xmm15 +$L$8xvl_body:: + vzeroupper + lea r9,QWORD PTR[$L$sigma] + vbroadcasti128 ymm3,XMMWORD PTR[r9] + vbroadcasti128 ymm7,XMMWORD PTR[rcx] + vbroadcasti128 ymm11,XMMWORD PTR[16+rcx] + vbroadcasti128 ymm15,XMMWORD PTR[r8] + vpshufd ymm0,ymm3,000h + vpshufd ymm1,ymm3,055h + vpshufd ymm2,ymm3,0aah + vpshufd ymm3,ymm3,0ffh + vmovdqa64 ymm16,ymm0 + vmovdqa64 ymm17,ymm1 + vmovdqa64 ymm18,ymm2 + vmovdqa64 ymm19,ymm3 + vpshufd ymm4,ymm7,000h + vpshufd ymm5,ymm7,055h + vpshufd ymm6,ymm7,0aah + vpshufd ymm7,ymm7,0ffh + vmovdqa64 ymm20,ymm4 + vmovdqa64 ymm21,ymm5 + vmovdqa64 ymm22,ymm6 + vmovdqa64 ymm23,ymm7 + vpshufd ymm8,ymm11,000h + vpshufd ymm9,ymm11,055h + vpshufd ymm10,ymm11,0aah + vpshufd ymm11,ymm11,0ffh + vmovdqa64 ymm24,ymm8 + vmovdqa64 ymm25,ymm9 + vmovdqa64 ymm26,ymm10 + vmovdqa64 ymm27,ymm11 + vpshufd ymm12,ymm15,000h + vpshufd ymm13,ymm15,055h + vpshufd ymm14,ymm15,0aah + vpshufd ymm15,ymm15,0ffh + vpaddd ymm12,ymm12,YMMWORD PTR[$L$incy] + vmovdqa64 ymm28,ymm12 + vmovdqa64 ymm29,ymm13 + vmovdqa64 ymm30,ymm14 + vmovdqa64 ymm31,ymm15 + mov eax,10 + jmp $L$chacha20_loop8xvl + +ALIGN 32 +$L$chacha20_loop_outer8xvl:: + vpbroadcastd ymm2,DWORD PTR[8+r9] + vpbroadcastd ymm3,DWORD PTR[12+r9] + vpaddd ymm28,ymm28,YMMWORD PTR[$L$eight] + vmovdqa64 ymm4,ymm20 + vmovdqa64 ymm5,ymm21 + vmovdqa64 ymm6,ymm22 + vmovdqa64 ymm7,ymm23 + vmovdqa64 ymm8,ymm24 + vmovdqa64 ymm9,ymm25 + vmovdqa64 ymm10,ymm26 + vmovdqa64 ymm11,ymm27 + vmovdqa64 ymm12,ymm28 + vmovdqa64 ymm13,ymm29 + vmovdqa64 ymm14,ymm30 + vmovdqa64 ymm15,ymm31 + vmovdqa64 ymm16,ymm0 + vmovdqa64 ymm17,ymm1 + vmovdqa64 ymm18,ymm2 + vmovdqa64 ymm19,ymm3 + mov eax,10 + jmp $L$chacha20_loop8xvl + +ALIGN 32 +$L$chacha20_loop8xvl:: + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxor ymm12,ymm12,ymm0 + vpxor ymm13,ymm13,ymm1 + vpxor ymm14,ymm14,ymm2 + vpxor ymm15,ymm15,ymm3 + vprold ymm12,ymm12,16 + vprold ymm13,ymm13,16 + vprold ymm14,ymm14,16 + vprold ymm15,ymm15,16 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxor ymm4,ymm4,ymm8 + vpxor ymm5,ymm5,ymm9 + vpxor ymm6,ymm6,ymm10 + vpxor ymm7,ymm7,ymm11 + vprold ymm4,ymm4,12 + vprold ymm5,ymm5,12 + vprold ymm6,ymm6,12 + vprold ymm7,ymm7,12 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxor ymm12,ymm12,ymm0 + vpxor ymm13,ymm13,ymm1 + vpxor ymm14,ymm14,ymm2 + vpxor ymm15,ymm15,ymm3 + vprold ymm12,ymm12,8 + vprold ymm13,ymm13,8 + vprold ymm14,ymm14,8 + vprold ymm15,ymm15,8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxor ymm4,ymm4,ymm8 + vpxor ymm5,ymm5,ymm9 + vpxor ymm6,ymm6,ymm10 + vpxor ymm7,ymm7,ymm11 + vprold ymm4,ymm4,7 + vprold ymm5,ymm5,7 + vprold ymm6,ymm6,7 + vprold ymm7,ymm7,7 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxor ymm15,ymm15,ymm0 + vpxor ymm12,ymm12,ymm1 + vpxor ymm13,ymm13,ymm2 + vpxor ymm14,ymm14,ymm3 + vprold ymm15,ymm15,16 + vprold ymm12,ymm12,16 + vprold ymm13,ymm13,16 + vprold ymm14,ymm14,16 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxor ymm5,ymm5,ymm10 + vpxor ymm6,ymm6,ymm11 + vpxor ymm7,ymm7,ymm8 + vpxor ymm4,ymm4,ymm9 + vprold ymm5,ymm5,12 + vprold ymm6,ymm6,12 + vprold ymm7,ymm7,12 + vprold ymm4,ymm4,12 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxor ymm15,ymm15,ymm0 + vpxor ymm12,ymm12,ymm1 + vpxor ymm13,ymm13,ymm2 + vpxor ymm14,ymm14,ymm3 + vprold ymm15,ymm15,8 + vprold ymm12,ymm12,8 + vprold ymm13,ymm13,8 + vprold ymm14,ymm14,8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxor ymm5,ymm5,ymm10 + vpxor ymm6,ymm6,ymm11 + vpxor ymm7,ymm7,ymm8 + vpxor ymm4,ymm4,ymm9 + vprold ymm5,ymm5,7 + vprold ymm6,ymm6,7 + vprold ymm7,ymm7,7 + vprold ymm4,ymm4,7 + dec eax + jnz $L$chacha20_loop8xvl + vpaddd ymm0,ymm0,ymm16 + vpaddd ymm1,ymm1,ymm17 + vpaddd ymm2,ymm2,ymm18 + vpaddd ymm3,ymm3,ymm19 + vpunpckldq ymm18,ymm0,ymm1 + vpunpckldq ymm19,ymm2,ymm3 + vpunpckhdq ymm0,ymm0,ymm1 + vpunpckhdq ymm2,ymm2,ymm3 + vpunpcklqdq ymm1,ymm18,ymm19 + vpunpckhqdq ymm18,ymm18,ymm19 + vpunpcklqdq ymm3,ymm0,ymm2 + vpunpckhqdq ymm0,ymm0,ymm2 + vpaddd ymm4,ymm4,ymm20 + vpaddd ymm5,ymm5,ymm21 + vpaddd ymm6,ymm6,ymm22 + vpaddd ymm7,ymm7,ymm23 + vpunpckldq ymm2,ymm4,ymm5 + vpunpckldq ymm19,ymm6,ymm7 + vpunpckhdq ymm4,ymm4,ymm5 + vpunpckhdq ymm6,ymm6,ymm7 + vpunpcklqdq ymm5,ymm2,ymm19 + vpunpckhqdq ymm2,ymm2,ymm19 + vpunpcklqdq ymm7,ymm4,ymm6 + vpunpckhqdq ymm4,ymm4,ymm6 + vshufi32x4 ymm19,ymm1,ymm5,0 + vshufi32x4 ymm5,ymm1,ymm5,3 + vshufi32x4 ymm1,ymm18,ymm2,0 + vshufi32x4 ymm2,ymm18,ymm2,3 + vshufi32x4 ymm18,ymm3,ymm7,0 + vshufi32x4 ymm7,ymm3,ymm7,3 + vshufi32x4 ymm3,ymm0,ymm4,0 + vshufi32x4 ymm4,ymm0,ymm4,3 + vpaddd ymm8,ymm8,ymm24 + vpaddd ymm9,ymm9,ymm25 + vpaddd ymm10,ymm10,ymm26 + vpaddd ymm11,ymm11,ymm27 + vpunpckldq ymm6,ymm8,ymm9 + vpunpckldq ymm0,ymm10,ymm11 + vpunpckhdq ymm8,ymm8,ymm9 + vpunpckhdq ymm10,ymm10,ymm11 + vpunpcklqdq ymm9,ymm6,ymm0 + vpunpckhqdq ymm6,ymm6,ymm0 + vpunpcklqdq ymm11,ymm8,ymm10 + vpunpckhqdq ymm8,ymm8,ymm10 + vpaddd ymm12,ymm12,ymm28 + vpaddd ymm13,ymm13,ymm29 + vpaddd ymm14,ymm14,ymm30 + vpaddd ymm15,ymm15,ymm31 + vpunpckldq ymm10,ymm12,ymm13 + vpunpckldq ymm0,ymm14,ymm15 + vpunpckhdq ymm12,ymm12,ymm13 + vpunpckhdq ymm14,ymm14,ymm15 + vpunpcklqdq ymm13,ymm10,ymm0 + vpunpckhqdq ymm10,ymm10,ymm0 + vpunpcklqdq ymm15,ymm12,ymm14 + vpunpckhqdq ymm12,ymm12,ymm14 + vperm2i128 ymm0,ymm9,ymm13,020h + vperm2i128 ymm13,ymm9,ymm13,031h + vperm2i128 ymm9,ymm6,ymm10,020h + vperm2i128 ymm10,ymm6,ymm10,031h + vperm2i128 ymm6,ymm11,ymm15,020h + vperm2i128 ymm15,ymm11,ymm15,031h + vperm2i128 ymm11,ymm8,ymm12,020h + vperm2i128 ymm12,ymm8,ymm12,031h + cmp rdx,64*8 + jb $L$tail8xvl + mov eax,080h + vpxord ymm19,ymm19,YMMWORD PTR[rsi] + vpxor ymm0,ymm0,YMMWORD PTR[32+rsi] + vpxor ymm5,ymm5,YMMWORD PTR[64+rsi] + vpxor ymm13,ymm13,YMMWORD PTR[96+rsi] + lea rsi,QWORD PTR[rax*1+rsi] + vmovdqu32 YMMWORD PTR[rdi],ymm19 + vmovdqu YMMWORD PTR[32+rdi],ymm0 + vmovdqu YMMWORD PTR[64+rdi],ymm5 + vmovdqu YMMWORD PTR[96+rdi],ymm13 + lea rdi,QWORD PTR[rax*1+rdi] + vpxor ymm1,ymm1,YMMWORD PTR[rsi] + vpxor ymm9,ymm9,YMMWORD PTR[32+rsi] + vpxor ymm2,ymm2,YMMWORD PTR[64+rsi] + vpxor ymm10,ymm10,YMMWORD PTR[96+rsi] + lea rsi,QWORD PTR[rax*1+rsi] + vmovdqu YMMWORD PTR[rdi],ymm1 + vmovdqu YMMWORD PTR[32+rdi],ymm9 + vmovdqu YMMWORD PTR[64+rdi],ymm2 + vmovdqu YMMWORD PTR[96+rdi],ymm10 + lea rdi,QWORD PTR[rax*1+rdi] + vpxord ymm18,ymm18,YMMWORD PTR[rsi] + vpxor ymm6,ymm6,YMMWORD PTR[32+rsi] + vpxor ymm7,ymm7,YMMWORD PTR[64+rsi] + vpxor ymm15,ymm15,YMMWORD PTR[96+rsi] + lea rsi,QWORD PTR[rax*1+rsi] + vmovdqu32 YMMWORD PTR[rdi],ymm18 + vmovdqu YMMWORD PTR[32+rdi],ymm6 + vmovdqu YMMWORD PTR[64+rdi],ymm7 + vmovdqu YMMWORD PTR[96+rdi],ymm15 + lea rdi,QWORD PTR[rax*1+rdi] + vpxor ymm3,ymm3,YMMWORD PTR[rsi] + vpxor ymm11,ymm11,YMMWORD PTR[32+rsi] + vpxor ymm4,ymm4,YMMWORD PTR[64+rsi] + vpxor ymm12,ymm12,YMMWORD PTR[96+rsi] + lea rsi,QWORD PTR[rax*1+rsi] + vmovdqu YMMWORD PTR[rdi],ymm3 + vmovdqu YMMWORD PTR[32+rdi],ymm11 + vmovdqu YMMWORD PTR[64+rdi],ymm4 + vmovdqu YMMWORD PTR[96+rdi],ymm12 + lea rdi,QWORD PTR[rax*1+rdi] + vpbroadcastd ymm0,DWORD PTR[r9] + vpbroadcastd ymm1,DWORD PTR[4+r9] + sub rdx,64*8 + jnz $L$chacha20_loop_outer8xvl + jmp $L$done8xvl + +ALIGN 32 +$L$tail8xvl:: + vmovdqa64 ymm8,ymm19 + xor r9,r9 + sub rdi,rsi + cmp rdx,64*1 + jb $L$ess_than_64_8xvl + vpxor ymm8,ymm8,YMMWORD PTR[rsi] + vpxor ymm0,ymm0,YMMWORD PTR[32+rsi] + vmovdqu YMMWORD PTR[rsi*1+rdi],ymm8 + vmovdqu YMMWORD PTR[32+rsi*1+rdi],ymm0 + je $L$done8xvl + vmovdqa ymm8,ymm5 + vmovdqa ymm0,ymm13 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*2 + jb $L$ess_than_64_8xvl + vpxor ymm5,ymm5,YMMWORD PTR[rsi] + vpxor ymm13,ymm13,YMMWORD PTR[32+rsi] + vmovdqu YMMWORD PTR[rsi*1+rdi],ymm5 + vmovdqu YMMWORD PTR[32+rsi*1+rdi],ymm13 + je $L$done8xvl + vmovdqa ymm8,ymm1 + vmovdqa ymm0,ymm9 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*3 + jb $L$ess_than_64_8xvl + vpxor ymm1,ymm1,YMMWORD PTR[rsi] + vpxor ymm9,ymm9,YMMWORD PTR[32+rsi] + vmovdqu YMMWORD PTR[rsi*1+rdi],ymm1 + vmovdqu YMMWORD PTR[32+rsi*1+rdi],ymm9 + je $L$done8xvl + vmovdqa ymm8,ymm2 + vmovdqa ymm0,ymm10 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*4 + jb $L$ess_than_64_8xvl + vpxor ymm2,ymm2,YMMWORD PTR[rsi] + vpxor ymm10,ymm10,YMMWORD PTR[32+rsi] + vmovdqu YMMWORD PTR[rsi*1+rdi],ymm2 + vmovdqu YMMWORD PTR[32+rsi*1+rdi],ymm10 + je $L$done8xvl + vmovdqa32 ymm8,ymm18 + vmovdqa ymm0,ymm6 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*5 + jb $L$ess_than_64_8xvl + vpxord ymm18,ymm18,YMMWORD PTR[rsi] + vpxor ymm6,ymm6,YMMWORD PTR[32+rsi] + vmovdqu32 YMMWORD PTR[rsi*1+rdi],ymm18 + vmovdqu YMMWORD PTR[32+rsi*1+rdi],ymm6 + je $L$done8xvl + vmovdqa ymm8,ymm7 + vmovdqa ymm0,ymm15 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*6 + jb $L$ess_than_64_8xvl + vpxor ymm7,ymm7,YMMWORD PTR[rsi] + vpxor ymm15,ymm15,YMMWORD PTR[32+rsi] + vmovdqu YMMWORD PTR[rsi*1+rdi],ymm7 + vmovdqu YMMWORD PTR[32+rsi*1+rdi],ymm15 + je $L$done8xvl + vmovdqa ymm8,ymm3 + vmovdqa ymm0,ymm11 + lea rsi,QWORD PTR[64+rsi] + cmp rdx,64*7 + jb $L$ess_than_64_8xvl + vpxor ymm3,ymm3,YMMWORD PTR[rsi] + vpxor ymm11,ymm11,YMMWORD PTR[32+rsi] + vmovdqu YMMWORD PTR[rsi*1+rdi],ymm3 + vmovdqu YMMWORD PTR[32+rsi*1+rdi],ymm11 + je $L$done8xvl + vmovdqa ymm8,ymm4 + vmovdqa ymm0,ymm12 + lea rsi,QWORD PTR[64+rsi] + +$L$ess_than_64_8xvl:: + vmovdqa YMMWORD PTR[rsp],ymm8 + vmovdqa YMMWORD PTR[32+rsp],ymm0 + lea rdi,QWORD PTR[rsi*1+rdi] + and rdx,63 + +$L$chacha20_loop_tail8xvl:: + movzx eax,BYTE PTR[r9*1+rsi] + movzx ecx,BYTE PTR[r9*1+rsp] + lea r9,QWORD PTR[1+r9] + xor eax,ecx + mov BYTE PTR[((-1))+r9*1+rdi],al + dec rdx + jnz $L$chacha20_loop_tail8xvl + vpxor ymm8,ymm8,ymm8 + vmovdqa YMMWORD PTR[rsp],ymm8 + vmovdqa YMMWORD PTR[32+rsp],ymm8 + +$L$done8xvl:: + vzeroall + movaps xmm6,XMMWORD PTR[((-168))+r10] + movaps xmm7,XMMWORD PTR[((-152))+r10] + movaps xmm8,XMMWORD PTR[((-136))+r10] + movaps xmm9,XMMWORD PTR[((-120))+r10] + movaps xmm10,XMMWORD PTR[((-104))+r10] + movaps xmm11,XMMWORD PTR[((-88))+r10] + movaps xmm12,XMMWORD PTR[((-72))+r10] + movaps xmm13,XMMWORD PTR[((-56))+r10] + movaps xmm14,XMMWORD PTR[((-40))+r10] + movaps xmm15,XMMWORD PTR[((-24))+r10] + lea rsp,QWORD PTR[r10] + +$L$8xvl_epilogue:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret + +$L$SEH_end_chacha20_8xvl:: +chacha20_8xvl ENDP + +ALIGN 16 +chacha20_se_handler PROC PRIVATE + DB 243,15,30,250 + push rsi + push rdi + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + pushfq + sub rsp,64 + mov rax,QWORD PTR[120+r8] + mov rbx,QWORD PTR[248+r8] + mov rsi,QWORD PTR[8+r9] + mov r11,QWORD PTR[56+r9] + lea r10,QWORD PTR[$L$ctr32_body] + cmp rbx,r10 + jb $L$chacha20_common_seh_tail + mov rax,QWORD PTR[152+r8] + lea r10,QWORD PTR[$L$chacha20_no_data] + cmp rbx,r10 + jae $L$chacha20_common_seh_tail + lea rax,QWORD PTR[((64+24+48))+rax] + mov rbx,QWORD PTR[((-8))+rax] + mov rbp,QWORD PTR[((-16))+rax] + mov r12,QWORD PTR[((-24))+rax] + mov r13,QWORD PTR[((-32))+rax] + mov r14,QWORD PTR[((-40))+rax] + mov r15,QWORD PTR[((-48))+rax] + mov QWORD PTR[144+r8],rbx + mov QWORD PTR[160+r8],rbp + mov QWORD PTR[216+r8],r12 + mov QWORD PTR[224+r8],r13 + mov QWORD PTR[232+r8],r14 + mov QWORD PTR[240+r8],r15 + +$L$chacha20_common_seh_tail:: + mov rdi,QWORD PTR[8+rax] + mov rsi,QWORD PTR[16+rax] + mov QWORD PTR[152+r8],rax + mov QWORD PTR[168+r8],rsi + mov QWORD PTR[176+r8],rdi + mov rdi,QWORD PTR[40+r9] + mov rsi,r8 + mov ecx,154 + DD 0a548f3fch + mov rsi,r9 + xor rcx,rcx + mov rdx,QWORD PTR[8+rsi] + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[16+rsi] + mov r10,QWORD PTR[40+rsi] + lea r11,QWORD PTR[56+rsi] + lea r12,QWORD PTR[24+rsi] + mov QWORD PTR[32+rsp],r10 + mov QWORD PTR[40+rsp],r11 + mov QWORD PTR[48+rsp],r12 + mov QWORD PTR[56+rsp],rcx + call QWORD PTR[__imp_RtlVirtualUnwind] + mov eax,1 + add rsp,64 + popfq + pop r15 + pop r14 + pop r13 + pop r12 + pop rbp + pop rbx + pop rdi + pop rsi + DB 0F3h,0C3h ;repret +chacha20_se_handler ENDP + +ALIGN 16 +chacha20_simd_handler PROC PRIVATE + DB 243,15,30,250 + push rsi + push rdi + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + pushfq + sub rsp,64 + mov rax,QWORD PTR[120+r8] + mov rbx,QWORD PTR[248+r8] + mov rsi,QWORD PTR[8+r9] + mov r11,QWORD PTR[56+r9] + mov r10d,DWORD PTR[r11] + lea r10,QWORD PTR[r10*1+rsi] + cmp rbx,r10 + jb $L$chacha20_common_seh_tail + mov rax,QWORD PTR[200+r8] + mov r10d,DWORD PTR[4+r11] + mov ecx,DWORD PTR[8+r11] + lea r10,QWORD PTR[r10*1+rsi] + cmp rbx,r10 + jae $L$chacha20_common_seh_tail + neg rcx + lea rsi,QWORD PTR[((-8))+rcx*1+rax] + lea rdi,QWORD PTR[512+r8] + neg ecx + shr ecx,3 + DD 0a548f3fch + jmp $L$chacha20_common_seh_tail +chacha20_simd_handler ENDP + +PUBLIC Poly1305InitALU +PUBLIC Poly1305InitAVX512IFMA +PUBLIC Poly1305BlocksALU +PUBLIC Poly1305BlocksAVX +PUBLIC Poly1305BlocksAVX2 +PUBLIC Poly1305BlocksAVX512IFMA +PUBLIC Poly1305EmitALU +PUBLIC Poly1305EmitAVX512IFMA + +ALIGN 32 +Poly1305InitALU PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_poly1305_init:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + xor rax,rax + mov QWORD PTR[rdi],rax + mov QWORD PTR[8+rdi],rax + mov QWORD PTR[16+rdi],rax + mov rax,00ffffffc0fffffffh + lea rcx,QWORD PTR[((-3))+rax] + and rax,QWORD PTR[rsi] + and rcx,QWORD PTR[8+rsi] + mov QWORD PTR[24+rdi],rax + mov QWORD PTR[32+rdi],rcx + mov DWORD PTR[48+rdi],-1 + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret +$L$SEH_end_poly1305_init:: +Poly1305InitALU ENDP + +ALIGN 32 +Poly1305BlocksALU PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_poly1305_blocks:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + +$L$blocks:: + shr rdx,4 + jz $L$poly1305_no_data + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + lea rsp,QWORD PTR[((-8))+rsp] + +$L$blocks_body:: + mov r15,rdx + mov r11,QWORD PTR[24+rdi] + mov r13,QWORD PTR[32+rdi] + mov r14,QWORD PTR[rdi] + mov rbx,QWORD PTR[8+rdi] + mov rbp,QWORD PTR[16+rdi] + mov eax,r14d + mov edx,DWORD PTR[4+rdi] + mov r8d,ebx + mov r10d,DWORD PTR[12+rdi] + mov r12d,ebp + shl rdx,26 + mov r9,r8 + shl r8,52 + add rax,rdx + shr r9,12 + add r8,rax + adc r9,0 + shl r10,14 + mov rax,r12 + shr r12,24 + add r9,r10 + shl rax,40 + add r9,rax + adc r12,0 + cmp rbp,4 + cmova r14,r8 + cmova rbx,r9 + cmova rbp,r12 + mov r12,r13 + shr r13,2 + mov rax,r12 + add r13,r12 + jmp $L$poly1305_loop + +ALIGN 32 +$L$poly1305_loop:: + add r14,QWORD PTR[rsi] + adc rbx,QWORD PTR[8+rsi] + lea rsi,QWORD PTR[16+rsi] + adc rbp,rcx + mul r14 + mov r9,rax + mov rax,r11 + mov r10,rdx + mul r14 + mov r14,rax + mov rax,r11 + mov r8,rdx + mul rbx + add r9,rax + mov rax,r13 + adc r10,rdx + mul rbx + mov rbx,rbp + add r14,rax + adc r8,rdx + imul rbx,r13 + add r9,rbx + mov rbx,r8 + adc r10,0 + imul rbp,r11 + add rbx,r9 + mov rax,-4 + adc r10,rbp + and rax,r10 + mov rbp,r10 + shr r10,2 + and rbp,3 + add rax,r10 + add r14,rax + adc rbx,0 + adc rbp,0 + mov rax,r12 + dec r15 + jnz $L$poly1305_loop + mov QWORD PTR[rdi],r14 + mov QWORD PTR[8+rdi],rbx + mov QWORD PTR[16+rdi],rbp + mov r15,QWORD PTR[8+rsp] + mov r14,QWORD PTR[16+rsp] + mov r13,QWORD PTR[24+rsp] + mov r12,QWORD PTR[32+rsp] + mov rbp,QWORD PTR[40+rsp] + mov rbx,QWORD PTR[48+rsp] + lea rsp,QWORD PTR[56+rsp] + +$L$poly1305_no_data:: +$L$blocks_epilogue:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret + +$L$SEH_end_poly1305_blocks:: +Poly1305BlocksALU ENDP + +ALIGN 32 +Poly1305EmitALU PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_poly1305_emit:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov eax,DWORD PTR[rdi] + mov ecx,DWORD PTR[4+rdi] + mov r8d,DWORD PTR[8+rdi] + mov r11d,DWORD PTR[12+rdi] + mov r10d,DWORD PTR[16+rdi] + shl rcx,26 + mov r9,r8 + shl r8,52 + add rax,rcx + shr r9,12 + add r8,rax + adc r9,0 + shl r11,14 + mov rax,r10 + shr r10,24 + add r9,r11 + mov rcx,QWORD PTR[rdi] + shl rax,40 + mov r11,QWORD PTR[8+rdi] + add r9,rax + mov rax,QWORD PTR[16+rdi] + adc r10,0 + cmp rax,4 + cmovbe r8,rcx + cmovbe r9,r11 + cmovbe r10,rax + mov rax,r8 + add r8,5 + mov rcx,r9 + adc r9,0 + adc r10,0 + shr r10,2 + cmovnz rax,r8 + cmovnz rcx,r9 + add rax,QWORD PTR[rdx] + adc rcx,QWORD PTR[8+rdx] + mov QWORD PTR[rsi],rax + mov QWORD PTR[8+rsi],rcx + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret +$L$SEH_end_poly1305_emit:: +Poly1305EmitALU ENDP + +ALIGN 32 +__poly1305_block PROC PRIVATE + DB 243,15,30,250 + mul r14 + mov r9,rax + mov rax,r11 + mov r10,rdx + mul r14 + mov r14,rax + mov rax,r11 + mov r8,rdx + mul rbx + add r9,rax + mov rax,r13 + adc r10,rdx + mul rbx + mov rbx,rbp + add r14,rax + adc r8,rdx + imul rbx,r13 + add r9,rbx + mov rbx,r8 + adc r10,0 + imul rbp,r11 + add rbx,r9 + mov rax,-4 + adc r10,rbp + and rax,r10 + mov rbp,r10 + shr r10,2 + and rbp,3 + add rax,r10 + add r14,rax + adc rbx,0 + adc rbp,0 + DB 0F3h,0C3h ;repret +__poly1305_block ENDP + +ALIGN 32 +__poly1305_init_avx PROC PRIVATE + DB 243,15,30,250 + cmp DWORD PTR[48+rdi],-1 + jne $L$done_init_avx + mov r14,r11 + mov rbx,r12 + xor rbp,rbp + lea rdi,QWORD PTR[((48+64))+rdi] + mov rax,r12 + call __poly1305_block + mov eax,03ffffffh + mov edx,03ffffffh + mov r8,r14 + and eax,r14d + mov r9,r11 + and edx,r11d + mov DWORD PTR[((-64))+rdi],eax + shr r8,26 + mov DWORD PTR[((-60))+rdi],edx + shr r9,26 + mov eax,03ffffffh + mov edx,03ffffffh + and eax,r8d + and edx,r9d + mov DWORD PTR[((-48))+rdi],eax + lea eax,DWORD PTR[rax*4+rax] + mov DWORD PTR[((-44))+rdi],edx + lea edx,DWORD PTR[rdx*4+rdx] + mov DWORD PTR[((-32))+rdi],eax + shr r8,26 + mov DWORD PTR[((-28))+rdi],edx + shr r9,26 + mov rax,rbx + mov rdx,r12 + shl rax,12 + shl rdx,12 + or rax,r8 + or rdx,r9 + and eax,03ffffffh + and edx,03ffffffh + mov DWORD PTR[((-16))+rdi],eax + lea eax,DWORD PTR[rax*4+rax] + mov DWORD PTR[((-12))+rdi],edx + lea edx,DWORD PTR[rdx*4+rdx] + mov DWORD PTR[rdi],eax + mov r8,rbx + mov DWORD PTR[4+rdi],edx + mov r9,r12 + mov eax,03ffffffh + mov edx,03ffffffh + shr r8,14 + shr r9,14 + and eax,r8d + and edx,r9d + mov DWORD PTR[16+rdi],eax + lea eax,DWORD PTR[rax*4+rax] + mov DWORD PTR[20+rdi],edx + lea edx,DWORD PTR[rdx*4+rdx] + mov DWORD PTR[32+rdi],eax + shr r8,26 + mov DWORD PTR[36+rdi],edx + shr r9,26 + mov rax,rbp + shl rax,24 + or r8,rax + mov DWORD PTR[48+rdi],r8d + lea r8,QWORD PTR[r8*4+r8] + mov DWORD PTR[52+rdi],r9d + lea r9,QWORD PTR[r9*4+r9] + mov DWORD PTR[64+rdi],r8d + mov DWORD PTR[68+rdi],r9d + mov rax,r12 + call __poly1305_block + mov eax,03ffffffh + mov r8,r14 + and eax,r14d + shr r8,26 + mov DWORD PTR[((-52))+rdi],eax + mov edx,03ffffffh + and edx,r8d + mov DWORD PTR[((-36))+rdi],edx + lea edx,DWORD PTR[rdx*4+rdx] + shr r8,26 + mov DWORD PTR[((-20))+rdi],edx + mov rax,rbx + shl rax,12 + or rax,r8 + and eax,03ffffffh + mov DWORD PTR[((-4))+rdi],eax + lea eax,DWORD PTR[rax*4+rax] + mov r8,rbx + mov DWORD PTR[12+rdi],eax + mov edx,03ffffffh + shr r8,14 + and edx,r8d + mov DWORD PTR[28+rdi],edx + lea edx,DWORD PTR[rdx*4+rdx] + shr r8,26 + mov DWORD PTR[44+rdi],edx + mov rax,rbp + shl rax,24 + or r8,rax + mov DWORD PTR[60+rdi],r8d + lea r8,QWORD PTR[r8*4+r8] + mov DWORD PTR[76+rdi],r8d + mov rax,r12 + call __poly1305_block + mov eax,03ffffffh + mov r8,r14 + and eax,r14d + shr r8,26 + mov DWORD PTR[((-56))+rdi],eax + mov edx,03ffffffh + and edx,r8d + mov DWORD PTR[((-40))+rdi],edx + lea edx,DWORD PTR[rdx*4+rdx] + shr r8,26 + mov DWORD PTR[((-24))+rdi],edx + mov rax,rbx + shl rax,12 + or rax,r8 + and eax,03ffffffh + mov DWORD PTR[((-8))+rdi],eax + lea eax,DWORD PTR[rax*4+rax] + mov r8,rbx + mov DWORD PTR[8+rdi],eax + mov edx,03ffffffh + shr r8,14 + and edx,r8d + mov DWORD PTR[24+rdi],edx + lea edx,DWORD PTR[rdx*4+rdx] + shr r8,26 + mov DWORD PTR[40+rdi],edx + mov rax,rbp + shl rax,24 + or r8,rax + mov DWORD PTR[56+rdi],r8d + lea r8,QWORD PTR[r8*4+r8] + mov DWORD PTR[72+rdi],r8d + lea rdi,QWORD PTR[((-48-64))+rdi] +$L$done_init_avx:: + DB 0F3h,0C3h ;repret +__poly1305_init_avx ENDP + +ALIGN 32 +Poly1305BlocksAVX PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_poly1305_blocks_avx:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8d,DWORD PTR[20+rdi] + cmp rdx,128 + jb $L$blocks + and rdx,-16 + vzeroupper + test r8d,r8d + jz $L$base2_64_avx + test rdx,31 + jz $L$even_avx + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + lea rsp,QWORD PTR[((-8))+rsp] + +$L$blocks_avx_body:: + mov r15,rdx + mov r8,QWORD PTR[rdi] + mov r9,QWORD PTR[8+rdi] + mov ebp,DWORD PTR[16+rdi] + mov r11,QWORD PTR[24+rdi] + mov r13,QWORD PTR[32+rdi] + mov r14d,r8d + and r8,-2147483648 + mov r12,r9 + mov ebx,r9d + and r9,-2147483648 + shr r8,6 + shl r12,52 + add r14,r8 + shr rbx,12 + shr r9,18 + add r14,r12 + adc rbx,r9 + mov r8,rbp + shl r8,40 + shr rbp,24 + add rbx,r8 + adc rbp,0 + mov r12,r13 + mov rax,r13 + shr r13,2 + add r13,r12 + add r14,QWORD PTR[rsi] + adc rbx,QWORD PTR[8+rsi] + lea rsi,QWORD PTR[16+rsi] + adc rbp,rcx + call __poly1305_block + mov rax,r14 + mov rdx,r14 + shr r14,52 + mov r11,rbx + mov r12,rbx + shr rdx,26 + and rax,03ffffffh + shl r11,12 + and rdx,03ffffffh + shr rbx,14 + or r14,r11 + shl rbp,24 + and r14,03ffffffh + shr r12,40 + and rbx,03ffffffh + or rbp,r12 + vmovd xmm0,eax + vmovd xmm1,edx + vmovd xmm2,r14d + vmovd xmm3,ebx + vmovd xmm4,ebp + lea rdx,QWORD PTR[((-16))+r15] + mov r15,QWORD PTR[8+rsp] + mov r14,QWORD PTR[16+rsp] + mov r13,QWORD PTR[24+rsp] + mov r12,QWORD PTR[32+rsp] + mov rbp,QWORD PTR[40+rsp] + mov rbx,QWORD PTR[48+rsp] + lea rax,QWORD PTR[56+rsp] + lea rsp,QWORD PTR[56+rsp] + +$L$blocks_avx_epilogue:: + jmp $L$do_avx + +ALIGN 32 +$L$base2_64_avx:: + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + lea rsp,QWORD PTR[((-8))+rsp] + +$L$base2_64_avx_body:: + mov r15,rdx + mov r11,QWORD PTR[24+rdi] + mov r13,QWORD PTR[32+rdi] + mov r14,QWORD PTR[rdi] + mov rbx,QWORD PTR[8+rdi] + mov ebp,DWORD PTR[16+rdi] + mov r12,r13 + mov rax,r13 + shr r13,2 + add r13,r12 + test rdx,31 + jz $L$init_avx + add r14,QWORD PTR[rsi] + adc rbx,QWORD PTR[8+rsi] + lea rsi,QWORD PTR[16+rsi] + adc rbp,rcx + sub r15,16 + call __poly1305_block + +$L$init_avx:: + mov rax,r14 + mov rdx,r14 + shr r14,52 + mov r8,rbx + mov r9,rbx + shr rdx,26 + and rax,03ffffffh + shl r8,12 + and rdx,03ffffffh + shr rbx,14 + or r14,r8 + shl rbp,24 + and r14,03ffffffh + shr r9,40 + and rbx,03ffffffh + or rbp,r9 + vmovd xmm0,eax + vmovd xmm1,edx + vmovd xmm2,r14d + vmovd xmm3,ebx + vmovd xmm4,ebp + mov DWORD PTR[20+rdi],1 + call __poly1305_init_avx + mov rdx,r15 + mov r15,QWORD PTR[8+rsp] + mov r14,QWORD PTR[16+rsp] + mov r13,QWORD PTR[24+rsp] + mov r12,QWORD PTR[32+rsp] + mov rbp,QWORD PTR[40+rsp] + mov rbx,QWORD PTR[48+rsp] + lea rax,QWORD PTR[56+rsp] + lea rsp,QWORD PTR[56+rsp] + +$L$base2_64_avx_epilogue:: + jmp $L$do_avx + +ALIGN 32 +$L$even_avx:: + vmovd xmm0,DWORD PTR[rdi] + vmovd xmm1,DWORD PTR[4+rdi] + vmovd xmm2,DWORD PTR[8+rdi] + vmovd xmm3,DWORD PTR[12+rdi] + vmovd xmm4,DWORD PTR[16+rdi] + +$L$do_avx:: + lea r11,QWORD PTR[((-248))+rsp] + sub rsp,0218h + vmovdqa XMMWORD PTR[80+r11],xmm6 + vmovdqa XMMWORD PTR[96+r11],xmm7 + vmovdqa XMMWORD PTR[112+r11],xmm8 + vmovdqa XMMWORD PTR[128+r11],xmm9 + vmovdqa XMMWORD PTR[144+r11],xmm10 + vmovdqa XMMWORD PTR[160+r11],xmm11 + vmovdqa XMMWORD PTR[176+r11],xmm12 + vmovdqa XMMWORD PTR[192+r11],xmm13 + vmovdqa XMMWORD PTR[208+r11],xmm14 + vmovdqa XMMWORD PTR[224+r11],xmm15 +$L$do_avx_body:: + sub rdx,64 + lea rax,QWORD PTR[((-32))+rsi] + cmovc rsi,rax + vmovdqu xmm14,XMMWORD PTR[48+rdi] + lea rdi,QWORD PTR[112+rdi] + lea rcx,QWORD PTR[$L$const] + vmovdqu xmm5,XMMWORD PTR[32+rsi] + vmovdqu xmm6,XMMWORD PTR[48+rsi] + vmovdqa xmm15,XMMWORD PTR[64+rcx] + vpsrldq xmm7,xmm5,6 + vpsrldq xmm8,xmm6,6 + vpunpckhqdq xmm9,xmm5,xmm6 + vpunpcklqdq xmm5,xmm5,xmm6 + vpunpcklqdq xmm8,xmm7,xmm8 + vpsrlq xmm9,xmm9,40 + vpsrlq xmm6,xmm5,26 + vpand xmm5,xmm5,xmm15 + vpsrlq xmm7,xmm8,4 + vpand xmm6,xmm6,xmm15 + vpsrlq xmm8,xmm8,30 + vpand xmm7,xmm7,xmm15 + vpand xmm8,xmm8,xmm15 + vpor xmm9,xmm9,XMMWORD PTR[32+rcx] + jbe $L$skip_loop_avx + vmovdqu xmm11,XMMWORD PTR[((-48))+rdi] + vmovdqu xmm12,XMMWORD PTR[((-32))+rdi] + vpshufd xmm13,xmm14,0EEh + vpshufd xmm10,xmm14,044h + vmovdqa XMMWORD PTR[(-144)+r11],xmm13 + vmovdqa XMMWORD PTR[rsp],xmm10 + vpshufd xmm14,xmm11,0EEh + vmovdqu xmm10,XMMWORD PTR[((-16))+rdi] + vpshufd xmm11,xmm11,044h + vmovdqa XMMWORD PTR[(-128)+r11],xmm14 + vmovdqa XMMWORD PTR[16+rsp],xmm11 + vpshufd xmm13,xmm12,0EEh + vmovdqu xmm11,XMMWORD PTR[rdi] + vpshufd xmm12,xmm12,044h + vmovdqa XMMWORD PTR[(-112)+r11],xmm13 + vmovdqa XMMWORD PTR[32+rsp],xmm12 + vpshufd xmm14,xmm10,0EEh + vmovdqu xmm12,XMMWORD PTR[16+rdi] + vpshufd xmm10,xmm10,044h + vmovdqa XMMWORD PTR[(-96)+r11],xmm14 + vmovdqa XMMWORD PTR[48+rsp],xmm10 + vpshufd xmm13,xmm11,0EEh + vmovdqu xmm10,XMMWORD PTR[32+rdi] + vpshufd xmm11,xmm11,044h + vmovdqa XMMWORD PTR[(-80)+r11],xmm13 + vmovdqa XMMWORD PTR[64+rsp],xmm11 + vpshufd xmm14,xmm12,0EEh + vmovdqu xmm11,XMMWORD PTR[48+rdi] + vpshufd xmm12,xmm12,044h + vmovdqa XMMWORD PTR[(-64)+r11],xmm14 + vmovdqa XMMWORD PTR[80+rsp],xmm12 + vpshufd xmm13,xmm10,0EEh + vmovdqu xmm12,XMMWORD PTR[64+rdi] + vpshufd xmm10,xmm10,044h + vmovdqa XMMWORD PTR[(-48)+r11],xmm13 + vmovdqa XMMWORD PTR[96+rsp],xmm10 + vpshufd xmm14,xmm11,0EEh + vpshufd xmm11,xmm11,044h + vmovdqa XMMWORD PTR[(-32)+r11],xmm14 + vmovdqa XMMWORD PTR[112+rsp],xmm11 + vpshufd xmm13,xmm12,0EEh + vmovdqa xmm14,XMMWORD PTR[rsp] + vpshufd xmm12,xmm12,044h + vmovdqa XMMWORD PTR[(-16)+r11],xmm13 + vmovdqa XMMWORD PTR[128+rsp],xmm12 + jmp $L$poly1305_loop_avx + +ALIGN 32 +$L$poly1305_loop_avx:: + vpmuludq xmm10,xmm14,xmm5 + vpmuludq xmm11,xmm14,xmm6 + vmovdqa XMMWORD PTR[32+r11],xmm2 + vpmuludq xmm12,xmm14,xmm7 + vmovdqa xmm2,XMMWORD PTR[16+rsp] + vpmuludq xmm13,xmm14,xmm8 + vpmuludq xmm14,xmm14,xmm9 + vmovdqa XMMWORD PTR[r11],xmm0 + vpmuludq xmm0,xmm9,XMMWORD PTR[32+rsp] + vmovdqa XMMWORD PTR[16+r11],xmm1 + vpmuludq xmm1,xmm2,xmm8 + vpaddq xmm10,xmm10,xmm0 + vpaddq xmm14,xmm14,xmm1 + vmovdqa XMMWORD PTR[48+r11],xmm3 + vpmuludq xmm0,xmm2,xmm7 + vpmuludq xmm1,xmm2,xmm6 + vpaddq xmm13,xmm13,xmm0 + vmovdqa xmm3,XMMWORD PTR[48+rsp] + vpaddq xmm12,xmm12,xmm1 + vmovdqa XMMWORD PTR[64+r11],xmm4 + vpmuludq xmm2,xmm2,xmm5 + vpmuludq xmm0,xmm3,xmm7 + vpaddq xmm11,xmm11,xmm2 + vmovdqa xmm4,XMMWORD PTR[64+rsp] + vpaddq xmm14,xmm14,xmm0 + vpmuludq xmm1,xmm3,xmm6 + vpmuludq xmm3,xmm3,xmm5 + vpaddq xmm13,xmm13,xmm1 + vmovdqa xmm2,XMMWORD PTR[80+rsp] + vpaddq xmm12,xmm12,xmm3 + vpmuludq xmm0,xmm4,xmm9 + vpmuludq xmm4,xmm4,xmm8 + vpaddq xmm11,xmm11,xmm0 + vmovdqa xmm3,XMMWORD PTR[96+rsp] + vpaddq xmm10,xmm10,xmm4 + vmovdqa xmm4,XMMWORD PTR[128+rsp] + vpmuludq xmm1,xmm2,xmm6 + vpmuludq xmm2,xmm2,xmm5 + vpaddq xmm14,xmm14,xmm1 + vpaddq xmm13,xmm13,xmm2 + vpmuludq xmm0,xmm3,xmm9 + vpmuludq xmm1,xmm3,xmm8 + vpaddq xmm12,xmm12,xmm0 + vmovdqu xmm0,XMMWORD PTR[rsi] + vpaddq xmm11,xmm11,xmm1 + vpmuludq xmm3,xmm3,xmm7 + vpmuludq xmm7,xmm4,xmm7 + vpaddq xmm10,xmm10,xmm3 + vmovdqu xmm1,XMMWORD PTR[16+rsi] + vpaddq xmm11,xmm11,xmm7 + vpmuludq xmm8,xmm4,xmm8 + vpmuludq xmm9,xmm4,xmm9 + vpsrldq xmm2,xmm0,6 + vpaddq xmm12,xmm12,xmm8 + vpaddq xmm13,xmm13,xmm9 + vpsrldq xmm3,xmm1,6 + vpmuludq xmm9,xmm5,XMMWORD PTR[112+rsp] + vpmuludq xmm5,xmm4,xmm6 + vpunpckhqdq xmm4,xmm0,xmm1 + vpaddq xmm14,xmm14,xmm9 + vmovdqa xmm9,XMMWORD PTR[((-144))+r11] + vpaddq xmm10,xmm10,xmm5 + vpunpcklqdq xmm0,xmm0,xmm1 + vpunpcklqdq xmm3,xmm2,xmm3 + vpsrldq xmm4,xmm4,5 + vpsrlq xmm1,xmm0,26 + vpand xmm0,xmm0,xmm15 + vpsrlq xmm2,xmm3,4 + vpand xmm1,xmm1,xmm15 + vpand xmm4,xmm4,XMMWORD PTR[rcx] + vpsrlq xmm3,xmm3,30 + vpand xmm2,xmm2,xmm15 + vpand xmm3,xmm3,xmm15 + vpor xmm4,xmm4,XMMWORD PTR[32+rcx] + vpaddq xmm0,xmm0,XMMWORD PTR[r11] + vpaddq xmm1,xmm1,XMMWORD PTR[16+r11] + vpaddq xmm2,xmm2,XMMWORD PTR[32+r11] + vpaddq xmm3,xmm3,XMMWORD PTR[48+r11] + vpaddq xmm4,xmm4,XMMWORD PTR[64+r11] + lea rax,QWORD PTR[32+rsi] + lea rsi,QWORD PTR[64+rsi] + sub rdx,64 + cmovc rsi,rax + vpmuludq xmm5,xmm9,xmm0 + vpmuludq xmm6,xmm9,xmm1 + vpaddq xmm10,xmm10,xmm5 + vpaddq xmm11,xmm11,xmm6 + vmovdqa xmm7,XMMWORD PTR[((-128))+r11] + vpmuludq xmm5,xmm9,xmm2 + vpmuludq xmm6,xmm9,xmm3 + vpaddq xmm12,xmm12,xmm5 + vpaddq xmm13,xmm13,xmm6 + vpmuludq xmm9,xmm9,xmm4 + vpmuludq xmm5,xmm4,XMMWORD PTR[((-112))+r11] + vpaddq xmm14,xmm14,xmm9 + vpaddq xmm10,xmm10,xmm5 + vpmuludq xmm6,xmm7,xmm2 + vpmuludq xmm5,xmm7,xmm3 + vpaddq xmm13,xmm13,xmm6 + vmovdqa xmm8,XMMWORD PTR[((-96))+r11] + vpaddq xmm14,xmm14,xmm5 + vpmuludq xmm6,xmm7,xmm1 + vpmuludq xmm7,xmm7,xmm0 + vpaddq xmm12,xmm12,xmm6 + vpaddq xmm11,xmm11,xmm7 + vmovdqa xmm9,XMMWORD PTR[((-80))+r11] + vpmuludq xmm5,xmm8,xmm2 + vpmuludq xmm6,xmm8,xmm1 + vpaddq xmm14,xmm14,xmm5 + vpaddq xmm13,xmm13,xmm6 + vmovdqa xmm7,XMMWORD PTR[((-64))+r11] + vpmuludq xmm8,xmm8,xmm0 + vpmuludq xmm5,xmm9,xmm4 + vpaddq xmm12,xmm12,xmm8 + vpaddq xmm11,xmm11,xmm5 + vmovdqa xmm8,XMMWORD PTR[((-48))+r11] + vpmuludq xmm9,xmm9,xmm3 + vpmuludq xmm6,xmm7,xmm1 + vpaddq xmm10,xmm10,xmm9 + vmovdqa xmm9,XMMWORD PTR[((-16))+r11] + vpaddq xmm14,xmm14,xmm6 + vpmuludq xmm7,xmm7,xmm0 + vpmuludq xmm5,xmm8,xmm4 + vpaddq xmm13,xmm13,xmm7 + vpaddq xmm12,xmm12,xmm5 + vmovdqu xmm5,XMMWORD PTR[32+rsi] + vpmuludq xmm7,xmm8,xmm3 + vpmuludq xmm8,xmm8,xmm2 + vpaddq xmm11,xmm11,xmm7 + vmovdqu xmm6,XMMWORD PTR[48+rsi] + vpaddq xmm10,xmm10,xmm8 + vpmuludq xmm2,xmm9,xmm2 + vpmuludq xmm3,xmm9,xmm3 + vpsrldq xmm7,xmm5,6 + vpaddq xmm11,xmm11,xmm2 + vpmuludq xmm4,xmm9,xmm4 + vpsrldq xmm8,xmm6,6 + vpaddq xmm2,xmm12,xmm3 + vpaddq xmm3,xmm13,xmm4 + vpmuludq xmm4,xmm0,XMMWORD PTR[((-32))+r11] + vpmuludq xmm0,xmm9,xmm1 + vpunpckhqdq xmm9,xmm5,xmm6 + vpaddq xmm4,xmm14,xmm4 + vpaddq xmm0,xmm10,xmm0 + vpunpcklqdq xmm5,xmm5,xmm6 + vpunpcklqdq xmm8,xmm7,xmm8 + vpsrldq xmm9,xmm9,5 + vpsrlq xmm6,xmm5,26 + vmovdqa xmm14,XMMWORD PTR[rsp] + vpand xmm5,xmm5,xmm15 + vpsrlq xmm7,xmm8,4 + vpand xmm6,xmm6,xmm15 + vpand xmm9,xmm9,XMMWORD PTR[rcx] + vpsrlq xmm8,xmm8,30 + vpand xmm7,xmm7,xmm15 + vpand xmm8,xmm8,xmm15 + vpor xmm9,xmm9,XMMWORD PTR[32+rcx] + vpsrlq xmm13,xmm3,26 + vpand xmm3,xmm3,xmm15 + vpaddq xmm4,xmm4,xmm13 + vpsrlq xmm10,xmm0,26 + vpand xmm0,xmm0,xmm15 + vpaddq xmm1,xmm11,xmm10 + vpsrlq xmm10,xmm4,26 + vpand xmm4,xmm4,xmm15 + vpsrlq xmm11,xmm1,26 + vpand xmm1,xmm1,xmm15 + vpaddq xmm2,xmm2,xmm11 + vpaddq xmm0,xmm0,xmm10 + vpsllq xmm10,xmm10,2 + vpaddq xmm0,xmm0,xmm10 + vpsrlq xmm12,xmm2,26 + vpand xmm2,xmm2,xmm15 + vpaddq xmm3,xmm3,xmm12 + vpsrlq xmm10,xmm0,26 + vpand xmm0,xmm0,xmm15 + vpaddq xmm1,xmm1,xmm10 + vpsrlq xmm13,xmm3,26 + vpand xmm3,xmm3,xmm15 + vpaddq xmm4,xmm4,xmm13 + ja $L$poly1305_loop_avx + +$L$skip_loop_avx:: + vpshufd xmm14,xmm14,010h + add rdx,32 + jnz $L$ong_tail_avx + vpaddq xmm7,xmm7,xmm2 + vpaddq xmm5,xmm5,xmm0 + vpaddq xmm6,xmm6,xmm1 + vpaddq xmm8,xmm8,xmm3 + vpaddq xmm9,xmm9,xmm4 + +$L$ong_tail_avx:: + vmovdqa XMMWORD PTR[32+r11],xmm2 + vmovdqa XMMWORD PTR[r11],xmm0 + vmovdqa XMMWORD PTR[16+r11],xmm1 + vmovdqa XMMWORD PTR[48+r11],xmm3 + vmovdqa XMMWORD PTR[64+r11],xmm4 + vpmuludq xmm12,xmm14,xmm7 + vpmuludq xmm10,xmm14,xmm5 + vpshufd xmm2,XMMWORD PTR[((-48))+rdi],010h + vpmuludq xmm11,xmm14,xmm6 + vpmuludq xmm13,xmm14,xmm8 + vpmuludq xmm14,xmm14,xmm9 + vpmuludq xmm0,xmm2,xmm8 + vpaddq xmm14,xmm14,xmm0 + vpshufd xmm3,XMMWORD PTR[((-32))+rdi],010h + vpmuludq xmm1,xmm2,xmm7 + vpaddq xmm13,xmm13,xmm1 + vpshufd xmm4,XMMWORD PTR[((-16))+rdi],010h + vpmuludq xmm0,xmm2,xmm6 + vpaddq xmm12,xmm12,xmm0 + vpmuludq xmm2,xmm2,xmm5 + vpaddq xmm11,xmm11,xmm2 + vpmuludq xmm3,xmm3,xmm9 + vpaddq xmm10,xmm10,xmm3 + vpshufd xmm2,XMMWORD PTR[rdi],010h + vpmuludq xmm1,xmm4,xmm7 + vpaddq xmm14,xmm14,xmm1 + vpmuludq xmm0,xmm4,xmm6 + vpaddq xmm13,xmm13,xmm0 + vpshufd xmm3,XMMWORD PTR[16+rdi],010h + vpmuludq xmm4,xmm4,xmm5 + vpaddq xmm12,xmm12,xmm4 + vpmuludq xmm1,xmm2,xmm9 + vpaddq xmm11,xmm11,xmm1 + vpshufd xmm4,XMMWORD PTR[32+rdi],010h + vpmuludq xmm2,xmm2,xmm8 + vpaddq xmm10,xmm10,xmm2 + vpmuludq xmm0,xmm3,xmm6 + vpaddq xmm14,xmm14,xmm0 + vpmuludq xmm3,xmm3,xmm5 + vpaddq xmm13,xmm13,xmm3 + vpshufd xmm2,XMMWORD PTR[48+rdi],010h + vpmuludq xmm1,xmm4,xmm9 + vpaddq xmm12,xmm12,xmm1 + vpshufd xmm3,XMMWORD PTR[64+rdi],010h + vpmuludq xmm0,xmm4,xmm8 + vpaddq xmm11,xmm11,xmm0 + vpmuludq xmm4,xmm4,xmm7 + vpaddq xmm10,xmm10,xmm4 + vpmuludq xmm2,xmm2,xmm5 + vpaddq xmm14,xmm14,xmm2 + vpmuludq xmm1,xmm3,xmm9 + vpaddq xmm13,xmm13,xmm1 + vpmuludq xmm0,xmm3,xmm8 + vpaddq xmm12,xmm12,xmm0 + vpmuludq xmm1,xmm3,xmm7 + vpaddq xmm11,xmm11,xmm1 + vpmuludq xmm3,xmm3,xmm6 + vpaddq xmm10,xmm10,xmm3 + jz $L$short_tail_avx + vmovdqu xmm0,XMMWORD PTR[rsi] + vmovdqu xmm1,XMMWORD PTR[16+rsi] + vpsrldq xmm2,xmm0,6 + vpsrldq xmm3,xmm1,6 + vpunpckhqdq xmm4,xmm0,xmm1 + vpunpcklqdq xmm0,xmm0,xmm1 + vpunpcklqdq xmm3,xmm2,xmm3 + vpsrlq xmm4,xmm4,40 + vpsrlq xmm1,xmm0,26 + vpand xmm0,xmm0,xmm15 + vpsrlq xmm2,xmm3,4 + vpand xmm1,xmm1,xmm15 + vpsrlq xmm3,xmm3,30 + vpand xmm2,xmm2,xmm15 + vpand xmm3,xmm3,xmm15 + vpor xmm4,xmm4,XMMWORD PTR[32+rcx] + vpshufd xmm9,XMMWORD PTR[((-64))+rdi],032h + vpaddq xmm0,xmm0,XMMWORD PTR[r11] + vpaddq xmm1,xmm1,XMMWORD PTR[16+r11] + vpaddq xmm2,xmm2,XMMWORD PTR[32+r11] + vpaddq xmm3,xmm3,XMMWORD PTR[48+r11] + vpaddq xmm4,xmm4,XMMWORD PTR[64+r11] + vpmuludq xmm5,xmm9,xmm0 + vpaddq xmm10,xmm10,xmm5 + vpmuludq xmm6,xmm9,xmm1 + vpaddq xmm11,xmm11,xmm6 + vpmuludq xmm5,xmm9,xmm2 + vpaddq xmm12,xmm12,xmm5 + vpshufd xmm7,XMMWORD PTR[((-48))+rdi],032h + vpmuludq xmm6,xmm9,xmm3 + vpaddq xmm13,xmm13,xmm6 + vpmuludq xmm9,xmm9,xmm4 + vpaddq xmm14,xmm14,xmm9 + vpmuludq xmm5,xmm7,xmm3 + vpaddq xmm14,xmm14,xmm5 + vpshufd xmm8,XMMWORD PTR[((-32))+rdi],032h + vpmuludq xmm6,xmm7,xmm2 + vpaddq xmm13,xmm13,xmm6 + vpshufd xmm9,XMMWORD PTR[((-16))+rdi],032h + vpmuludq xmm5,xmm7,xmm1 + vpaddq xmm12,xmm12,xmm5 + vpmuludq xmm7,xmm7,xmm0 + vpaddq xmm11,xmm11,xmm7 + vpmuludq xmm8,xmm8,xmm4 + vpaddq xmm10,xmm10,xmm8 + vpshufd xmm7,XMMWORD PTR[rdi],032h + vpmuludq xmm6,xmm9,xmm2 + vpaddq xmm14,xmm14,xmm6 + vpmuludq xmm5,xmm9,xmm1 + vpaddq xmm13,xmm13,xmm5 + vpshufd xmm8,XMMWORD PTR[16+rdi],032h + vpmuludq xmm9,xmm9,xmm0 + vpaddq xmm12,xmm12,xmm9 + vpmuludq xmm6,xmm7,xmm4 + vpaddq xmm11,xmm11,xmm6 + vpshufd xmm9,XMMWORD PTR[32+rdi],032h + vpmuludq xmm7,xmm7,xmm3 + vpaddq xmm10,xmm10,xmm7 + vpmuludq xmm5,xmm8,xmm1 + vpaddq xmm14,xmm14,xmm5 + vpmuludq xmm8,xmm8,xmm0 + vpaddq xmm13,xmm13,xmm8 + vpshufd xmm7,XMMWORD PTR[48+rdi],032h + vpmuludq xmm6,xmm9,xmm4 + vpaddq xmm12,xmm12,xmm6 + vpshufd xmm8,XMMWORD PTR[64+rdi],032h + vpmuludq xmm5,xmm9,xmm3 + vpaddq xmm11,xmm11,xmm5 + vpmuludq xmm9,xmm9,xmm2 + vpaddq xmm10,xmm10,xmm9 + vpmuludq xmm7,xmm7,xmm0 + vpaddq xmm14,xmm14,xmm7 + vpmuludq xmm6,xmm8,xmm4 + vpaddq xmm13,xmm13,xmm6 + vpmuludq xmm5,xmm8,xmm3 + vpaddq xmm12,xmm12,xmm5 + vpmuludq xmm6,xmm8,xmm2 + vpaddq xmm11,xmm11,xmm6 + vpmuludq xmm8,xmm8,xmm1 + vpaddq xmm10,xmm10,xmm8 + +$L$short_tail_avx:: + vpsrldq xmm9,xmm14,8 + vpsrldq xmm8,xmm13,8 + vpsrldq xmm6,xmm11,8 + vpsrldq xmm5,xmm10,8 + vpsrldq xmm7,xmm12,8 + vpaddq xmm13,xmm13,xmm8 + vpaddq xmm14,xmm14,xmm9 + vpaddq xmm10,xmm10,xmm5 + vpaddq xmm11,xmm11,xmm6 + vpaddq xmm12,xmm12,xmm7 + vpsrlq xmm3,xmm13,26 + vpand xmm13,xmm13,xmm15 + vpaddq xmm14,xmm14,xmm3 + vpsrlq xmm0,xmm10,26 + vpand xmm10,xmm10,xmm15 + vpaddq xmm11,xmm11,xmm0 + vpsrlq xmm4,xmm14,26 + vpand xmm14,xmm14,xmm15 + vpsrlq xmm1,xmm11,26 + vpand xmm11,xmm11,xmm15 + vpaddq xmm12,xmm12,xmm1 + vpaddq xmm10,xmm10,xmm4 + vpsllq xmm4,xmm4,2 + vpaddq xmm10,xmm10,xmm4 + vpsrlq xmm2,xmm12,26 + vpand xmm12,xmm12,xmm15 + vpaddq xmm13,xmm13,xmm2 + vpsrlq xmm0,xmm10,26 + vpand xmm10,xmm10,xmm15 + vpaddq xmm11,xmm11,xmm0 + vpsrlq xmm3,xmm13,26 + vpand xmm13,xmm13,xmm15 + vpaddq xmm14,xmm14,xmm3 + vmovd DWORD PTR[(-112)+rdi],xmm10 + vmovd DWORD PTR[(-108)+rdi],xmm11 + vmovd DWORD PTR[(-104)+rdi],xmm12 + vmovd DWORD PTR[(-100)+rdi],xmm13 + vmovd DWORD PTR[(-96)+rdi],xmm14 + vmovdqa xmm6,XMMWORD PTR[80+r11] + vmovdqa xmm7,XMMWORD PTR[96+r11] + vmovdqa xmm8,XMMWORD PTR[112+r11] + vmovdqa xmm9,XMMWORD PTR[128+r11] + vmovdqa xmm10,XMMWORD PTR[144+r11] + vmovdqa xmm11,XMMWORD PTR[160+r11] + vmovdqa xmm12,XMMWORD PTR[176+r11] + vmovdqa xmm13,XMMWORD PTR[192+r11] + vmovdqa xmm14,XMMWORD PTR[208+r11] + vmovdqa xmm15,XMMWORD PTR[224+r11] + lea rsp,QWORD PTR[248+r11] +$L$do_avx_epilogue:: + vzeroupper + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret + +$L$SEH_end_poly1305_blocks_avx:: +Poly1305BlocksAVX ENDP + +ALIGN 32 +Poly1305BlocksAVX2 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_poly1305_blocks_avx2:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8d,DWORD PTR[20+rdi] + cmp rdx,128 + jb $L$blocks + and rdx,-16 + vzeroupper + test r8d,r8d + jz $L$base2_64_avx2 + test rdx,63 + jz $L$even_avx2 + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + lea rsp,QWORD PTR[((-8))+rsp] + +$L$blocks_avx2_body:: + mov r15,rdx + mov r8,QWORD PTR[rdi] + mov r9,QWORD PTR[8+rdi] + mov ebp,DWORD PTR[16+rdi] + mov r11,QWORD PTR[24+rdi] + mov r13,QWORD PTR[32+rdi] + mov r14d,r8d + and r8,-2147483648 + mov r12,r9 + mov ebx,r9d + and r9,-2147483648 + shr r8,6 + shl r12,52 + add r14,r8 + shr rbx,12 + shr r9,18 + add r14,r12 + adc rbx,r9 + mov r8,rbp + shl r8,40 + shr rbp,24 + add rbx,r8 + adc rbp,0 + mov r12,r13 + mov rax,r13 + shr r13,2 + add r13,r12 + +$L$base2_26_pre_avx2:: + add r14,QWORD PTR[rsi] + adc rbx,QWORD PTR[8+rsi] + lea rsi,QWORD PTR[16+rsi] + adc rbp,rcx + sub r15,16 + call __poly1305_block + mov rax,r12 + test r15,63 + jnz $L$base2_26_pre_avx2 + mov rax,r14 + mov rdx,r14 + shr r14,52 + mov r11,rbx + mov r12,rbx + shr rdx,26 + and rax,03ffffffh + shl r11,12 + and rdx,03ffffffh + shr rbx,14 + or r14,r11 + shl rbp,24 + and r14,03ffffffh + shr r12,40 + and rbx,03ffffffh + or rbp,r12 + vmovd xmm0,eax + vmovd xmm1,edx + vmovd xmm2,r14d + vmovd xmm3,ebx + vmovd xmm4,ebp + mov rdx,r15 + mov r15,QWORD PTR[8+rsp] + mov r14,QWORD PTR[16+rsp] + mov r13,QWORD PTR[24+rsp] + mov r12,QWORD PTR[32+rsp] + mov rbp,QWORD PTR[40+rsp] + mov rbx,QWORD PTR[48+rsp] + lea rax,QWORD PTR[56+rsp] + lea rsp,QWORD PTR[56+rsp] + +$L$blocks_avx2_epilogue:: + jmp $L$do_avx2 + +ALIGN 32 +$L$base2_64_avx2:: + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + lea rsp,QWORD PTR[((-8))+rsp] + +$L$base2_64_avx2_body:: + mov r15,rdx + mov r11,QWORD PTR[24+rdi] + mov r13,QWORD PTR[32+rdi] + mov r14,QWORD PTR[rdi] + mov rbx,QWORD PTR[8+rdi] + mov ebp,DWORD PTR[16+rdi] + mov r12,r13 + mov rax,r13 + shr r13,2 + add r13,r12 + test rdx,63 + jz $L$init_avx2 + +$L$base2_64_pre_avx2:: + add r14,QWORD PTR[rsi] + adc rbx,QWORD PTR[8+rsi] + lea rsi,QWORD PTR[16+rsi] + adc rbp,rcx + sub r15,16 + call __poly1305_block + mov rax,r12 + test r15,63 + jnz $L$base2_64_pre_avx2 + +$L$init_avx2:: + mov rax,r14 + mov rdx,r14 + shr r14,52 + mov r8,rbx + mov r9,rbx + shr rdx,26 + and rax,03ffffffh + shl r8,12 + and rdx,03ffffffh + shr rbx,14 + or r14,r8 + shl rbp,24 + and r14,03ffffffh + shr r9,40 + and rbx,03ffffffh + or rbp,r9 + vmovd xmm0,eax + vmovd xmm1,edx + vmovd xmm2,r14d + vmovd xmm3,ebx + vmovd xmm4,ebp + mov DWORD PTR[20+rdi],1 + call __poly1305_init_avx + mov rdx,r15 + mov r15,QWORD PTR[8+rsp] + mov r14,QWORD PTR[16+rsp] + mov r13,QWORD PTR[24+rsp] + mov r12,QWORD PTR[32+rsp] + mov rbp,QWORD PTR[40+rsp] + mov rbx,QWORD PTR[48+rsp] + lea rax,QWORD PTR[56+rsp] + lea rsp,QWORD PTR[56+rsp] + +$L$base2_64_avx2_epilogue:: + jmp $L$do_avx2 + +ALIGN 32 +$L$even_avx2:: + vmovd xmm0,DWORD PTR[rdi] + vmovd xmm1,DWORD PTR[4+rdi] + vmovd xmm2,DWORD PTR[8+rdi] + vmovd xmm3,DWORD PTR[12+rdi] + vmovd xmm4,DWORD PTR[16+rdi] + +$L$do_avx2:: + lea r11,QWORD PTR[((-248))+rsp] + sub rsp,01c8h + vmovdqa XMMWORD PTR[80+r11],xmm6 + vmovdqa XMMWORD PTR[96+r11],xmm7 + vmovdqa XMMWORD PTR[112+r11],xmm8 + vmovdqa XMMWORD PTR[128+r11],xmm9 + vmovdqa XMMWORD PTR[144+r11],xmm10 + vmovdqa XMMWORD PTR[160+r11],xmm11 + vmovdqa XMMWORD PTR[176+r11],xmm12 + vmovdqa XMMWORD PTR[192+r11],xmm13 + vmovdqa XMMWORD PTR[208+r11],xmm14 + vmovdqa XMMWORD PTR[224+r11],xmm15 +$L$do_avx2_body:: + lea rcx,QWORD PTR[$L$const] + lea rdi,QWORD PTR[((48+64))+rdi] + vmovdqa ymm7,YMMWORD PTR[96+rcx] + vmovdqu xmm9,XMMWORD PTR[((-64))+rdi] + and rsp,-512 + vmovdqu xmm10,XMMWORD PTR[((-48))+rdi] + vmovdqu xmm6,XMMWORD PTR[((-32))+rdi] + vmovdqu xmm11,XMMWORD PTR[((-16))+rdi] + vmovdqu xmm12,XMMWORD PTR[rdi] + vmovdqu xmm13,XMMWORD PTR[16+rdi] + lea rax,QWORD PTR[144+rsp] + vmovdqu xmm14,XMMWORD PTR[32+rdi] + vpermd ymm9,ymm7,ymm9 + vmovdqu xmm15,XMMWORD PTR[48+rdi] + vpermd ymm10,ymm7,ymm10 + vmovdqu xmm5,XMMWORD PTR[64+rdi] + vpermd ymm6,ymm7,ymm6 + vmovdqa YMMWORD PTR[rsp],ymm9 + vpermd ymm11,ymm7,ymm11 + vmovdqa YMMWORD PTR[(32-144)+rax],ymm10 + vpermd ymm12,ymm7,ymm12 + vmovdqa YMMWORD PTR[(64-144)+rax],ymm6 + vpermd ymm13,ymm7,ymm13 + vmovdqa YMMWORD PTR[(96-144)+rax],ymm11 + vpermd ymm14,ymm7,ymm14 + vmovdqa YMMWORD PTR[(128-144)+rax],ymm12 + vpermd ymm15,ymm7,ymm15 + vmovdqa YMMWORD PTR[(160-144)+rax],ymm13 + vpermd ymm5,ymm7,ymm5 + vmovdqa YMMWORD PTR[(192-144)+rax],ymm14 + vmovdqa YMMWORD PTR[(224-144)+rax],ymm15 + vmovdqa YMMWORD PTR[(256-144)+rax],ymm5 + vmovdqa ymm5,YMMWORD PTR[64+rcx] + vmovdqu xmm7,XMMWORD PTR[rsi] + vmovdqu xmm8,XMMWORD PTR[16+rsi] + vinserti128 ymm7,ymm7,XMMWORD PTR[32+rsi],1 + vinserti128 ymm8,ymm8,XMMWORD PTR[48+rsi],1 + lea rsi,QWORD PTR[64+rsi] + vpsrldq ymm9,ymm7,6 + vpsrldq ymm10,ymm8,6 + vpunpckhqdq ymm6,ymm7,ymm8 + vpunpcklqdq ymm9,ymm9,ymm10 + vpunpcklqdq ymm7,ymm7,ymm8 + vpsrlq ymm10,ymm9,30 + vpsrlq ymm9,ymm9,4 + vpsrlq ymm8,ymm7,26 + vpsrlq ymm6,ymm6,40 + vpand ymm9,ymm9,ymm5 + vpand ymm7,ymm7,ymm5 + vpand ymm8,ymm8,ymm5 + vpand ymm10,ymm10,ymm5 + vpor ymm6,ymm6,YMMWORD PTR[32+rcx] + vpaddq ymm2,ymm9,ymm2 + sub rdx,64 + jz $L$tail_avx2 + jmp $L$poly1305_loop_avx2 + +ALIGN 32 +$L$poly1305_loop_avx2:: + vpaddq ymm0,ymm7,ymm0 + vmovdqa ymm7,YMMWORD PTR[rsp] + vpaddq ymm1,ymm8,ymm1 + vmovdqa ymm8,YMMWORD PTR[32+rsp] + vpaddq ymm3,ymm10,ymm3 + vmovdqa ymm9,YMMWORD PTR[96+rsp] + vpaddq ymm4,ymm6,ymm4 + vmovdqa ymm10,YMMWORD PTR[48+rax] + vmovdqa ymm5,YMMWORD PTR[112+rax] + vpmuludq ymm13,ymm7,ymm2 + vpmuludq ymm14,ymm8,ymm2 + vpmuludq ymm15,ymm9,ymm2 + vpmuludq ymm11,ymm10,ymm2 + vpmuludq ymm12,ymm5,ymm2 + vpmuludq ymm6,ymm8,ymm0 + vpmuludq ymm2,ymm8,ymm1 + vpaddq ymm12,ymm12,ymm6 + vpaddq ymm13,ymm13,ymm2 + vpmuludq ymm6,ymm8,ymm3 + vpmuludq ymm2,ymm4,YMMWORD PTR[64+rsp] + vpaddq ymm15,ymm15,ymm6 + vpaddq ymm11,ymm11,ymm2 + vmovdqa ymm8,YMMWORD PTR[((-16))+rax] + vpmuludq ymm6,ymm7,ymm0 + vpmuludq ymm2,ymm7,ymm1 + vpaddq ymm11,ymm11,ymm6 + vpaddq ymm12,ymm12,ymm2 + vpmuludq ymm6,ymm7,ymm3 + vpmuludq ymm2,ymm7,ymm4 + vmovdqu xmm7,XMMWORD PTR[rsi] + vpaddq ymm14,ymm14,ymm6 + vpaddq ymm15,ymm15,ymm2 + vinserti128 ymm7,ymm7,XMMWORD PTR[32+rsi],1 + vpmuludq ymm6,ymm8,ymm3 + vpmuludq ymm2,ymm8,ymm4 + vmovdqu xmm8,XMMWORD PTR[16+rsi] + vpaddq ymm11,ymm11,ymm6 + vpaddq ymm12,ymm12,ymm2 + vmovdqa ymm2,YMMWORD PTR[16+rax] + vpmuludq ymm6,ymm9,ymm1 + vpmuludq ymm9,ymm9,ymm0 + vpaddq ymm14,ymm14,ymm6 + vpaddq ymm13,ymm13,ymm9 + vinserti128 ymm8,ymm8,XMMWORD PTR[48+rsi],1 + lea rsi,QWORD PTR[64+rsi] + vpmuludq ymm6,ymm2,ymm1 + vpmuludq ymm2,ymm2,ymm0 + vpsrldq ymm9,ymm7,6 + vpaddq ymm15,ymm15,ymm6 + vpaddq ymm14,ymm14,ymm2 + vpmuludq ymm6,ymm10,ymm3 + vpmuludq ymm2,ymm10,ymm4 + vpsrldq ymm10,ymm8,6 + vpaddq ymm12,ymm12,ymm6 + vpaddq ymm13,ymm13,ymm2 + vpunpckhqdq ymm6,ymm7,ymm8 + vpmuludq ymm3,ymm5,ymm3 + vpmuludq ymm4,ymm5,ymm4 + vpunpcklqdq ymm7,ymm7,ymm8 + vpaddq ymm2,ymm13,ymm3 + vpaddq ymm3,ymm14,ymm4 + vpunpcklqdq ymm10,ymm9,ymm10 + vpmuludq ymm4,ymm0,YMMWORD PTR[80+rax] + vpmuludq ymm0,ymm5,ymm1 + vmovdqa ymm5,YMMWORD PTR[64+rcx] + vpaddq ymm4,ymm15,ymm4 + vpaddq ymm0,ymm11,ymm0 + vpsrlq ymm14,ymm3,26 + vpand ymm3,ymm3,ymm5 + vpaddq ymm4,ymm4,ymm14 + vpsrlq ymm11,ymm0,26 + vpand ymm0,ymm0,ymm5 + vpaddq ymm1,ymm12,ymm11 + vpsrlq ymm15,ymm4,26 + vpand ymm4,ymm4,ymm5 + vpsrlq ymm9,ymm10,4 + vpsrlq ymm12,ymm1,26 + vpand ymm1,ymm1,ymm5 + vpaddq ymm2,ymm2,ymm12 + vpaddq ymm0,ymm0,ymm15 + vpsllq ymm15,ymm15,2 + vpaddq ymm0,ymm0,ymm15 + vpand ymm9,ymm9,ymm5 + vpsrlq ymm8,ymm7,26 + vpsrlq ymm13,ymm2,26 + vpand ymm2,ymm2,ymm5 + vpaddq ymm3,ymm3,ymm13 + vpaddq ymm2,ymm2,ymm9 + vpsrlq ymm10,ymm10,30 + vpsrlq ymm11,ymm0,26 + vpand ymm0,ymm0,ymm5 + vpaddq ymm1,ymm1,ymm11 + vpsrlq ymm6,ymm6,40 + vpsrlq ymm14,ymm3,26 + vpand ymm3,ymm3,ymm5 + vpaddq ymm4,ymm4,ymm14 + vpand ymm7,ymm7,ymm5 + vpand ymm8,ymm8,ymm5 + vpand ymm10,ymm10,ymm5 + vpor ymm6,ymm6,YMMWORD PTR[32+rcx] + sub rdx,64 + jnz $L$poly1305_loop_avx2 + +DB 066h,090h +$L$tail_avx2:: + vpaddq ymm0,ymm7,ymm0 + vmovdqu ymm7,YMMWORD PTR[4+rsp] + vpaddq ymm1,ymm8,ymm1 + vmovdqu ymm8,YMMWORD PTR[36+rsp] + vpaddq ymm3,ymm10,ymm3 + vmovdqu ymm9,YMMWORD PTR[100+rsp] + vpaddq ymm4,ymm6,ymm4 + vmovdqu ymm10,YMMWORD PTR[52+rax] + vmovdqu ymm5,YMMWORD PTR[116+rax] + vpmuludq ymm13,ymm7,ymm2 + vpmuludq ymm14,ymm8,ymm2 + vpmuludq ymm15,ymm9,ymm2 + vpmuludq ymm11,ymm10,ymm2 + vpmuludq ymm12,ymm5,ymm2 + vpmuludq ymm6,ymm8,ymm0 + vpmuludq ymm2,ymm8,ymm1 + vpaddq ymm12,ymm12,ymm6 + vpaddq ymm13,ymm13,ymm2 + vpmuludq ymm6,ymm8,ymm3 + vpmuludq ymm2,ymm4,YMMWORD PTR[68+rsp] + vpaddq ymm15,ymm15,ymm6 + vpaddq ymm11,ymm11,ymm2 + vpmuludq ymm6,ymm7,ymm0 + vpmuludq ymm2,ymm7,ymm1 + vpaddq ymm11,ymm11,ymm6 + vmovdqu ymm8,YMMWORD PTR[((-12))+rax] + vpaddq ymm12,ymm12,ymm2 + vpmuludq ymm6,ymm7,ymm3 + vpmuludq ymm2,ymm7,ymm4 + vpaddq ymm14,ymm14,ymm6 + vpaddq ymm15,ymm15,ymm2 + vpmuludq ymm6,ymm8,ymm3 + vpmuludq ymm2,ymm8,ymm4 + vpaddq ymm11,ymm11,ymm6 + vpaddq ymm12,ymm12,ymm2 + vmovdqu ymm2,YMMWORD PTR[20+rax] + vpmuludq ymm6,ymm9,ymm1 + vpmuludq ymm9,ymm9,ymm0 + vpaddq ymm14,ymm14,ymm6 + vpaddq ymm13,ymm13,ymm9 + vpmuludq ymm6,ymm2,ymm1 + vpmuludq ymm2,ymm2,ymm0 + vpaddq ymm15,ymm15,ymm6 + vpaddq ymm14,ymm14,ymm2 + vpmuludq ymm6,ymm10,ymm3 + vpmuludq ymm2,ymm10,ymm4 + vpaddq ymm12,ymm12,ymm6 + vpaddq ymm13,ymm13,ymm2 + vpmuludq ymm3,ymm5,ymm3 + vpmuludq ymm4,ymm5,ymm4 + vpaddq ymm2,ymm13,ymm3 + vpaddq ymm3,ymm14,ymm4 + vpmuludq ymm4,ymm0,YMMWORD PTR[84+rax] + vpmuludq ymm0,ymm5,ymm1 + vmovdqa ymm5,YMMWORD PTR[64+rcx] + vpaddq ymm4,ymm15,ymm4 + vpaddq ymm0,ymm11,ymm0 + vpsrldq ymm8,ymm12,8 + vpsrldq ymm9,ymm2,8 + vpsrldq ymm10,ymm3,8 + vpsrldq ymm6,ymm4,8 + vpsrldq ymm7,ymm0,8 + vpaddq ymm12,ymm12,ymm8 + vpaddq ymm2,ymm2,ymm9 + vpaddq ymm3,ymm3,ymm10 + vpaddq ymm4,ymm4,ymm6 + vpaddq ymm0,ymm0,ymm7 + vpermq ymm10,ymm3,02h + vpermq ymm6,ymm4,02h + vpermq ymm7,ymm0,02h + vpermq ymm8,ymm12,02h + vpermq ymm9,ymm2,02h + vpaddq ymm3,ymm3,ymm10 + vpaddq ymm4,ymm4,ymm6 + vpaddq ymm0,ymm0,ymm7 + vpaddq ymm12,ymm12,ymm8 + vpaddq ymm2,ymm2,ymm9 + vpsrlq ymm14,ymm3,26 + vpand ymm3,ymm3,ymm5 + vpaddq ymm4,ymm4,ymm14 + vpsrlq ymm11,ymm0,26 + vpand ymm0,ymm0,ymm5 + vpaddq ymm1,ymm12,ymm11 + vpsrlq ymm15,ymm4,26 + vpand ymm4,ymm4,ymm5 + vpsrlq ymm12,ymm1,26 + vpand ymm1,ymm1,ymm5 + vpaddq ymm2,ymm2,ymm12 + vpaddq ymm0,ymm0,ymm15 + vpsllq ymm15,ymm15,2 + vpaddq ymm0,ymm0,ymm15 + vpsrlq ymm13,ymm2,26 + vpand ymm2,ymm2,ymm5 + vpaddq ymm3,ymm3,ymm13 + vpsrlq ymm11,ymm0,26 + vpand ymm0,ymm0,ymm5 + vpaddq ymm1,ymm1,ymm11 + vpsrlq ymm14,ymm3,26 + vpand ymm3,ymm3,ymm5 + vpaddq ymm4,ymm4,ymm14 + vmovd DWORD PTR[(-112)+rdi],xmm0 + vmovd DWORD PTR[(-108)+rdi],xmm1 + vmovd DWORD PTR[(-104)+rdi],xmm2 + vmovd DWORD PTR[(-100)+rdi],xmm3 + vmovd DWORD PTR[(-96)+rdi],xmm4 + vmovdqa xmm6,XMMWORD PTR[80+r11] + vmovdqa xmm7,XMMWORD PTR[96+r11] + vmovdqa xmm8,XMMWORD PTR[112+r11] + vmovdqa xmm9,XMMWORD PTR[128+r11] + vmovdqa xmm10,XMMWORD PTR[144+r11] + vmovdqa xmm11,XMMWORD PTR[160+r11] + vmovdqa xmm12,XMMWORD PTR[176+r11] + vmovdqa xmm13,XMMWORD PTR[192+r11] + vmovdqa xmm14,XMMWORD PTR[208+r11] + vmovdqa xmm15,XMMWORD PTR[224+r11] + lea rsp,QWORD PTR[248+r11] +$L$do_avx2_epilogue:: + vzeroupper + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret + +$L$SEH_end_poly1305_blocks_avx2:: +Poly1305BlocksAVX2 ENDP + +ALIGN 32 +Poly1305InitAVX512IFMA PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_poly1305_init_base2_44:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + xor rax,rax + mov QWORD PTR[rdi],rax + mov QWORD PTR[8+rdi],rax + mov QWORD PTR[16+rdi],rax + +$L$init_base2_44:: + mov rax,00ffffffc0fffffffh + mov rcx,00ffffffc0ffffffch + and rax,QWORD PTR[rsi] + mov r8,000000fffffffffffh + and rcx,QWORD PTR[8+rsi] + mov r9,000000fffffffffffh + and r8,rax + shrd rax,rcx,44 + mov QWORD PTR[40+rdi],r8 + and rax,r9 + shr rcx,24 + mov QWORD PTR[48+rdi],rax + lea rax,QWORD PTR[rax*4+rax] + mov QWORD PTR[56+rdi],rcx + shl rax,2 + lea rcx,QWORD PTR[rcx*4+rcx] + shl rcx,2 + mov QWORD PTR[24+rdi],rax + mov QWORD PTR[32+rdi],rcx + mov QWORD PTR[64+rdi],-1 +$L$no_key_base2_44:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret +$L$SEH_end_poly1305_init_base2_44:: +Poly1305InitAVX512IFMA ENDP + +ALIGN 32 +poly1305_blocks_base2_44 PROC PRIVATE + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_poly1305_blocks_base2_44:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + +$L$blocks_base2_44:: + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + and rdx,-16 + add rdx,rsi + shl rcx,40 + push rdx + +$L$blocks_base2_44_body:: + mov rdx,QWORD PTR[rdi] + mov r8,QWORD PTR[8+rdi] + mov r9,QWORD PTR[16+rdi] + mov r13,QWORD PTR[40+rdi] + mov r14,QWORD PTR[48+rdi] + mov r15,QWORD PTR[32+rdi] + mov rax,0fffff00000000000h + jmp $L$poly1305_loop_base2_44 + ud2 + +ALIGN 32 +$L$poly1305_loop_base2_44:: + mov r11,QWORD PTR[rsi] + mov r12,QWORD PTR[8+rsi] + lea rsi,QWORD PTR[16+rsi] + andn r10,rax,r11 + shrd r11,r12,44 + add rdx,r10 + shr r12,24 + andn r11,rax,r11 + add r9,rcx + add r8,r11 + add r9,r12 + mulx rbx,r10,r13 + mulx rcx,r11,r14 + mulx rbp,r12,QWORD PTR[56+rdi] + mov rdx,r8 + mulx r8,rax,r15 + add r10,rax + adc r8,rbx + mulx rbx,rax,r13 + add r11,rax + adc rcx,rbx + mulx rbx,rax,r14 + mov rdx,r9 + add r12,rax + adc rbp,rbx + mulx rbx,rax,QWORD PTR[24+rdi] + add r10,rax + adc r8,rbx + mulx r9,rax,r15 + add r11,rax + adc r9,rcx + mulx rbx,rax,r13 + add r12,rax + adc rbp,rbx + mov rax,0fffff00000000000h + andn rdx,rax,r10 + shrd r10,r8,44 + add r11,r10 + adc r9,0 + andn r8,rax,r11 + shrd r11,r9,44 + mov r9,003ffffffffffh + add r12,r11 + adc rbp,0 + and r9,r12 + shrd r12,rbp,42 + mov rcx,010000000000h + lea r12,QWORD PTR[r12*4+r12] + add rdx,r12 + cmp rsi,QWORD PTR[rsp] + jb $L$poly1305_loop_base2_44 + mov QWORD PTR[rdi],rdx + mov QWORD PTR[8+rdi],r8 + mov QWORD PTR[16+rdi],r9 + mov r15,QWORD PTR[8+rsp] + mov r14,QWORD PTR[16+rsp] + mov r13,QWORD PTR[24+rsp] + mov r12,QWORD PTR[32+rsp] + mov rbp,QWORD PTR[40+rsp] + mov rbx,QWORD PTR[48+rsp] + lea rsp,QWORD PTR[56+rsp] + +$L$blocks_base2_44_epilogue:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret + +$L$SEH_end_poly1305_blocks_base2_44:: +poly1305_blocks_base2_44 ENDP + +ALIGN 32 +Poly1305BlocksAVX512IFMA PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_poly1305_blocks_vpmadd52:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + and rdx,-16 + jz $L$poly1305_no_data_vpmadd52 + mov r8,QWORD PTR[64+rdi] + mov r9,030h + mov r10,010h + cmp rdx,040h + cmovae r9,r10 + test r8,r8 + cmovns r9,r10 + and r9,rdx + jz $L$blocks_vpmadd52_4x + sub rdx,r9 + cmovz rdx,r9 + jz $L$blocks_base2_44 + mov r10d,7 + mov r11d,1 + shl rcx,40 + kmovw k7,r10d + lea r10,QWORD PTR[$L$2_44_inp_permd] + kmovw k1,r11d + vmovq xmm21,rcx + shr rcx,40 + vmovdqa64 ymm19,YMMWORD PTR[r10] + vmovdqa64 ymm20,YMMWORD PTR[32+r10] + vpermq ymm21,ymm21,0cfh + vmovdqa64 ymm22,YMMWORD PTR[64+r10] + vmovdqu64 ymm16{k7}{z},[rdi] + vmovdqu64 ymm3{k7}{z},[40+rdi] + vmovdqu64 ymm4{k7}{z},[32+rdi] + vmovdqu64 ymm5{k7}{z},[24+rdi] + vmovdqa64 ymm23,YMMWORD PTR[96+r10] + vmovdqa64 ymm24,YMMWORD PTR[128+r10] + vmovdqu32 xmm18,XMMWORD PTR[rsi] + lea rsi,QWORD PTR[16+rsi] + vpermd ymm18,ymm19,ymm18 + vpsrlvq ymm18,ymm18,ymm20 + vpandq ymm18,ymm18,ymm22 + vporq ymm18,ymm18,ymm21 + vpaddq ymm16,ymm16,ymm18 + vpxord ymm27,ymm27,ymm27 + vpxord ymm28,ymm28,ymm28 + vpermq ymm0{k7}{z},ymm16,0 + vpermq ymm1{k7}{z},ymm16,85 + vpermq ymm2{k7}{z},ymm16,170 + vpxord ymm18,ymm18,ymm18 + vpxord ymm26,ymm26,ymm26 + vpmadd52luq ymm27,ymm0,ymm3 + vpmadd52huq ymm28,ymm0,ymm3 + vpxord ymm16,ymm16,ymm16 + vpxord ymm17,ymm17,ymm17 + vpmadd52luq ymm18,ymm1,ymm4 + vpmadd52huq ymm26,ymm1,ymm4 + vpmadd52luq ymm16,ymm2,ymm5 + vpmadd52huq ymm17,ymm2,ymm5 + vpaddq ymm27,ymm27,ymm18 + vpaddq ymm28,ymm28,ymm26 + vpaddq ymm16,ymm16,ymm27 + vpaddq ymm17,ymm17,ymm28 + vpsrlvq ymm18,ymm16,ymm23 + vpsllvq ymm17,ymm17,ymm24 + vpandq ymm16,ymm16,ymm22 + vpaddq ymm17,ymm17,ymm18 + vpermq ymm17,ymm17,147 + vpaddq ymm16,ymm16,ymm17 + vpsrlvq ymm18,ymm16,ymm23 + vpandq ymm16,ymm16,ymm22 + vpermq ymm18,ymm18,147 + vpaddq ymm16,ymm16,ymm18 + vpermq ymm18{k1}{z},ymm16,147 + vpaddq ymm16,ymm16,ymm18 + vpsllq ymm18,ymm18,2 + vpaddq ymm16,ymm16,ymm18 + vmovdqu64 YMMWORD PTR[rdi]{k7},ymm16 + jmp $L$blocks_vpmadd52_4x + +$L$poly1305_no_data_vpmadd52:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret +$L$SEH_end_poly1305_blocks_vpmadd52:: +Poly1305BlocksAVX512IFMA ENDP + +ALIGN 32 +poly1305_blocks_vpmadd52_4x PROC PRIVATE + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_poly1305_blocks_vpmadd52_4x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + and rdx,-16 + jz $L$poly1305_no_data_vpmadd52_4x + mov r8,QWORD PTR[64+rdi] + +$L$blocks_vpmadd52_4x:: + shl rcx,40 + shr rdx,4 + vpbroadcastq ymm31,rcx + vmovdqa64 ymm30,YMMWORD PTR[$L$x_mask44] + mov eax,5 + kmovw k1,eax + test r8,r8 + js $L$init_vpmadd52 + vmovq xmm0,QWORD PTR[rdi] + vmovq xmm1,QWORD PTR[8+rdi] + vmovq xmm2,QWORD PTR[16+rdi] + test rdx,3 + jnz $L$blocks_vpmadd52_2x_do + +$L$blocks_vpmadd52_4x_do:: + vpbroadcastq ymm3,QWORD PTR[64+rdi] + vpbroadcastq ymm4,QWORD PTR[96+rdi] + vpbroadcastq ymm5,QWORD PTR[128+rdi] + vpbroadcastq ymm16,QWORD PTR[160+rdi] + +$L$blocks_vpmadd52_4x_key_loaded:: + vpsllq ymm17,ymm5,2 + vpaddq ymm17,ymm17,ymm5 + vpsllq ymm17,ymm17,2 + vmovdqu64 ymm26,YMMWORD PTR[rsi] + vmovdqu64 ymm27,YMMWORD PTR[32+rsi] + lea rsi,QWORD PTR[64+rsi] + vpunpcklqdq ymm25,ymm26,ymm27 + vpunpckhqdq ymm27,ymm26,ymm27 + vpsrlq ymm26,ymm27,24 + vporq ymm26,ymm26,ymm31 + vpaddq ymm2,ymm2,ymm26 + vpandq ymm24,ymm25,ymm30 + vpsrlq ymm25,ymm25,44 + vpsllq ymm27,ymm27,20 + vporq ymm25,ymm25,ymm27 + vpandq ymm25,ymm25,ymm30 + sub rdx,4 + jz $L$tail_vpmadd52_4x + jmp $L$poly1305_loop_vpmadd52_4x + ud2 + +ALIGN 32 +$L$init_vpmadd52:: + vmovq xmm16,QWORD PTR[24+rdi] + vmovq xmm2,QWORD PTR[56+rdi] + vmovq xmm17,QWORD PTR[32+rdi] + vmovq xmm3,QWORD PTR[40+rdi] + vmovq xmm4,QWORD PTR[48+rdi] + vmovdqa ymm0,ymm3 + vmovdqa ymm1,ymm4 + vmovdqa ymm5,ymm2 + mov eax,2 + +$L$mul_init_vpmadd52:: + vpxorq ymm18,ymm18,ymm18 + vpxorq ymm19,ymm19,ymm19 + vpxorq ymm20,ymm20,ymm20 + vpxorq ymm21,ymm21,ymm21 + vpxorq ymm22,ymm22,ymm22 + vpxorq ymm23,ymm23,ymm23 + vpmadd52luq ymm18,ymm16,ymm2 + vpxorq ymm24,ymm24,ymm24 + vpxorq ymm25,ymm25,ymm25 + vpmadd52huq ymm19,ymm16,ymm2 + vpxorq ymm26,ymm26,ymm26 + vpxorq ymm27,ymm27,ymm27 + vpmadd52luq ymm20,ymm17,ymm2 + vpxorq ymm28,ymm28,ymm28 + vpxorq ymm29,ymm29,ymm29 + vpmadd52huq ymm21,ymm17,ymm2 + vpmadd52luq ymm22,ymm3,ymm2 + vpmadd52huq ymm23,ymm3,ymm2 + vpmadd52luq ymm24,ymm3,ymm0 + vpmadd52huq ymm25,ymm3,ymm0 + vpmadd52luq ymm26,ymm4,ymm0 + vpmadd52huq ymm27,ymm4,ymm0 + vpmadd52luq ymm28,ymm5,ymm0 + vpmadd52huq ymm29,ymm5,ymm0 + vpmadd52luq ymm18,ymm17,ymm1 + vpmadd52huq ymm19,ymm17,ymm1 + vpmadd52luq ymm20,ymm3,ymm1 + vpmadd52huq ymm21,ymm3,ymm1 + vpaddq ymm18,ymm18,ymm24 + vpaddq ymm19,ymm19,ymm25 + vpmadd52luq ymm22,ymm4,ymm1 + vpaddq ymm20,ymm20,ymm26 + vpaddq ymm21,ymm21,ymm27 + vpmadd52huq ymm23,ymm4,ymm1 + vpaddq ymm22,ymm22,ymm28 + vpaddq ymm23,ymm23,ymm29 + vpsrlq ymm29,ymm18,44 + vpsllq ymm19,ymm19,8 + vpandq ymm0,ymm18,ymm30 + vpaddq ymm19,ymm19,ymm29 + vpaddq ymm20,ymm20,ymm19 + vpsrlq ymm29,ymm20,44 + vpsllq ymm21,ymm21,8 + vpandq ymm1,ymm20,ymm30 + vpaddq ymm21,ymm21,ymm29 + vpaddq ymm22,ymm22,ymm21 + vpsrlq ymm29,ymm22,42 + vpsllq ymm23,ymm23,10 + vpandq ymm2,ymm22,YMMWORD PTR[$L$x_mask42] + vpaddq ymm23,ymm23,ymm29 + vpaddq ymm0,ymm0,ymm23 + vpsllq ymm23,ymm23,2 + vpaddq ymm0,ymm0,ymm23 + vpsrlq ymm29,ymm0,44 + vpandq ymm0,ymm0,ymm30 + vpaddq ymm1,ymm1,ymm29 + dec eax + jz $L$done_init_vpmadd52 + vpunpcklqdq ymm4,ymm1,ymm4 + vpbroadcastq xmm1,xmm1 + vpunpcklqdq ymm5,ymm2,ymm5 + vpbroadcastq xmm2,xmm2 + vpunpcklqdq ymm3,ymm0,ymm3 + vpbroadcastq xmm0,xmm0 + vpsllq ymm16,ymm4,2 + vpsllq ymm17,ymm5,2 + vpaddq ymm16,ymm16,ymm4 + vpaddq ymm17,ymm17,ymm5 + vpsllq ymm16,ymm16,2 + vpsllq ymm17,ymm17,2 + jmp $L$mul_init_vpmadd52 + ud2 + +ALIGN 32 +$L$done_init_vpmadd52:: + vinserti128 ymm4,ymm1,xmm4,1 + vinserti128 ymm5,ymm2,xmm5,1 + vinserti128 ymm3,ymm0,xmm3,1 + vpermq ymm4,ymm4,216 + vpermq ymm5,ymm5,216 + vpermq ymm3,ymm3,216 + vpsllq ymm16,ymm4,2 + vpaddq ymm16,ymm16,ymm4 + vpsllq ymm16,ymm16,2 + vmovq xmm0,QWORD PTR[rdi] + vmovq xmm1,QWORD PTR[8+rdi] + vmovq xmm2,QWORD PTR[16+rdi] + test rdx,3 + jnz $L$done_init_vpmadd52_2x + vmovdqu64 YMMWORD PTR[64+rdi],ymm3 + vpbroadcastq ymm3,xmm3 + vmovdqu64 YMMWORD PTR[96+rdi],ymm4 + vpbroadcastq ymm4,xmm4 + vmovdqu64 YMMWORD PTR[128+rdi],ymm5 + vpbroadcastq ymm5,xmm5 + vmovdqu64 YMMWORD PTR[160+rdi],ymm16 + vpbroadcastq ymm16,xmm16 + jmp $L$blocks_vpmadd52_4x_key_loaded + ud2 + +ALIGN 32 +$L$done_init_vpmadd52_2x:: + vmovdqu64 YMMWORD PTR[64+rdi],ymm3 + vpsrldq ymm3,ymm3,8 + vmovdqu64 YMMWORD PTR[96+rdi],ymm4 + vpsrldq ymm4,ymm4,8 + vmovdqu64 YMMWORD PTR[128+rdi],ymm5 + vpsrldq ymm5,ymm5,8 + vmovdqu64 YMMWORD PTR[160+rdi],ymm16 + vpsrldq ymm16,ymm16,8 + jmp $L$blocks_vpmadd52_2x_key_loaded + ud2 + +ALIGN 32 +$L$blocks_vpmadd52_2x_do:: + vmovdqu64 ymm5{k1}{z},[((128+8))+rdi] + vmovdqu64 ymm16{k1}{z},[((160+8))+rdi] + vmovdqu64 ymm3{k1}{z},[((64+8))+rdi] + vmovdqu64 ymm4{k1}{z},[((96+8))+rdi] + +$L$blocks_vpmadd52_2x_key_loaded:: + vmovdqu64 ymm26,YMMWORD PTR[rsi] + vpxorq ymm27,ymm27,ymm27 + lea rsi,QWORD PTR[32+rsi] + vpunpcklqdq ymm25,ymm26,ymm27 + vpunpckhqdq ymm27,ymm26,ymm27 + vpsrlq ymm26,ymm27,24 + vporq ymm26,ymm26,ymm31 + vpaddq ymm2,ymm2,ymm26 + vpandq ymm24,ymm25,ymm30 + vpsrlq ymm25,ymm25,44 + vpsllq ymm27,ymm27,20 + vporq ymm25,ymm25,ymm27 + vpandq ymm25,ymm25,ymm30 + jmp $L$tail_vpmadd52_2x + ud2 + +ALIGN 32 +$L$poly1305_loop_vpmadd52_4x:: + vpaddq ymm0,ymm0,ymm24 + vpaddq ymm1,ymm1,ymm25 + vpxorq ymm18,ymm18,ymm18 + vpxorq ymm19,ymm19,ymm19 + vpxorq ymm20,ymm20,ymm20 + vpxorq ymm21,ymm21,ymm21 + vpxorq ymm22,ymm22,ymm22 + vpxorq ymm23,ymm23,ymm23 + vpmadd52luq ymm18,ymm16,ymm2 + vpxorq ymm24,ymm24,ymm24 + vpxorq ymm25,ymm25,ymm25 + vpmadd52huq ymm19,ymm16,ymm2 + vpxorq ymm26,ymm26,ymm26 + vpxorq ymm27,ymm27,ymm27 + vpmadd52luq ymm20,ymm17,ymm2 + vpxorq ymm28,ymm28,ymm28 + vpxorq ymm29,ymm29,ymm29 + vpmadd52huq ymm21,ymm17,ymm2 + vpmadd52luq ymm22,ymm3,ymm2 + vpmadd52huq ymm23,ymm3,ymm2 + vpmadd52luq ymm24,ymm3,ymm0 + vpmadd52huq ymm25,ymm3,ymm0 + vpmadd52luq ymm26,ymm4,ymm0 + vpmadd52huq ymm27,ymm4,ymm0 + vpmadd52luq ymm28,ymm5,ymm0 + vpmadd52huq ymm29,ymm5,ymm0 + vpmadd52luq ymm18,ymm17,ymm1 + vpmadd52huq ymm19,ymm17,ymm1 + vpmadd52luq ymm20,ymm3,ymm1 + vpmadd52huq ymm21,ymm3,ymm1 + vpaddq ymm18,ymm18,ymm24 + vpaddq ymm19,ymm19,ymm25 + vpmadd52luq ymm22,ymm4,ymm1 + vpaddq ymm20,ymm20,ymm26 + vpaddq ymm21,ymm21,ymm27 + vpmadd52huq ymm23,ymm4,ymm1 + vpaddq ymm22,ymm22,ymm28 + vpaddq ymm23,ymm23,ymm29 + vmovdqu64 ymm26,YMMWORD PTR[rsi] + vmovdqu64 ymm27,YMMWORD PTR[32+rsi] + lea rsi,QWORD PTR[64+rsi] + vpunpcklqdq ymm25,ymm26,ymm27 + vpunpckhqdq ymm27,ymm26,ymm27 + vpsrlq ymm29,ymm18,44 + vpsllq ymm19,ymm19,8 + vpandq ymm0,ymm18,ymm30 + vpaddq ymm19,ymm19,ymm29 + vpsrlq ymm26,ymm27,24 + vporq ymm26,ymm26,ymm31 + vpaddq ymm20,ymm20,ymm19 + vpsrlq ymm29,ymm20,44 + vpsllq ymm21,ymm21,8 + vpandq ymm1,ymm20,ymm30 + vpaddq ymm21,ymm21,ymm29 + vpandq ymm24,ymm25,ymm30 + vpsrlq ymm25,ymm25,44 + vpsllq ymm27,ymm27,20 + vpaddq ymm22,ymm22,ymm21 + vpsrlq ymm29,ymm22,42 + vpsllq ymm23,ymm23,10 + vpandq ymm2,ymm22,YMMWORD PTR[$L$x_mask42] + vpaddq ymm23,ymm23,ymm29 + vpaddq ymm2,ymm2,ymm26 + vpaddq ymm0,ymm0,ymm23 + vpsllq ymm23,ymm23,2 + vpaddq ymm0,ymm0,ymm23 + vporq ymm25,ymm25,ymm27 + vpandq ymm25,ymm25,ymm30 + vpsrlq ymm29,ymm0,44 + vpandq ymm0,ymm0,ymm30 + vpaddq ymm1,ymm1,ymm29 + sub rdx,4 + jnz $L$poly1305_loop_vpmadd52_4x + +$L$tail_vpmadd52_4x:: + vmovdqu64 ymm5,YMMWORD PTR[128+rdi] + vmovdqu64 ymm16,YMMWORD PTR[160+rdi] + vmovdqu64 ymm3,YMMWORD PTR[64+rdi] + vmovdqu64 ymm4,YMMWORD PTR[96+rdi] + +$L$tail_vpmadd52_2x:: + vpsllq ymm17,ymm5,2 + vpaddq ymm17,ymm17,ymm5 + vpsllq ymm17,ymm17,2 + vpaddq ymm0,ymm0,ymm24 + vpaddq ymm1,ymm1,ymm25 + vpxorq ymm18,ymm18,ymm18 + vpxorq ymm19,ymm19,ymm19 + vpxorq ymm20,ymm20,ymm20 + vpxorq ymm21,ymm21,ymm21 + vpxorq ymm22,ymm22,ymm22 + vpxorq ymm23,ymm23,ymm23 + vpmadd52luq ymm18,ymm16,ymm2 + vpxorq ymm24,ymm24,ymm24 + vpxorq ymm25,ymm25,ymm25 + vpmadd52huq ymm19,ymm16,ymm2 + vpxorq ymm26,ymm26,ymm26 + vpxorq ymm27,ymm27,ymm27 + vpmadd52luq ymm20,ymm17,ymm2 + vpxorq ymm28,ymm28,ymm28 + vpxorq ymm29,ymm29,ymm29 + vpmadd52huq ymm21,ymm17,ymm2 + vpmadd52luq ymm22,ymm3,ymm2 + vpmadd52huq ymm23,ymm3,ymm2 + vpmadd52luq ymm24,ymm3,ymm0 + vpmadd52huq ymm25,ymm3,ymm0 + vpmadd52luq ymm26,ymm4,ymm0 + vpmadd52huq ymm27,ymm4,ymm0 + vpmadd52luq ymm28,ymm5,ymm0 + vpmadd52huq ymm29,ymm5,ymm0 + vpmadd52luq ymm18,ymm17,ymm1 + vpmadd52huq ymm19,ymm17,ymm1 + vpmadd52luq ymm20,ymm3,ymm1 + vpmadd52huq ymm21,ymm3,ymm1 + vpaddq ymm18,ymm18,ymm24 + vpaddq ymm19,ymm19,ymm25 + vpmadd52luq ymm22,ymm4,ymm1 + vpaddq ymm20,ymm20,ymm26 + vpaddq ymm21,ymm21,ymm27 + vpmadd52huq ymm23,ymm4,ymm1 + vpaddq ymm22,ymm22,ymm28 + vpaddq ymm23,ymm23,ymm29 + mov eax,1 + kmovw k1,eax + vpsrldq ymm24,ymm18,8 + vpsrldq ymm0,ymm19,8 + vpsrldq ymm25,ymm20,8 + vpsrldq ymm1,ymm21,8 + vpaddq ymm18,ymm18,ymm24 + vpaddq ymm19,ymm19,ymm0 + vpsrldq ymm26,ymm22,8 + vpsrldq ymm2,ymm23,8 + vpaddq ymm20,ymm20,ymm25 + vpaddq ymm21,ymm21,ymm1 + vpermq ymm24,ymm18,02h + vpermq ymm0,ymm19,02h + vpaddq ymm22,ymm22,ymm26 + vpaddq ymm23,ymm23,ymm2 + vpermq ymm25,ymm20,02h + vpermq ymm1,ymm21,02h + vpaddq ymm18{k1}{z},ymm18,ymm24 + vpaddq ymm19{k1}{z},ymm19,ymm0 + vpermq ymm26,ymm22,02h + vpermq ymm2,ymm23,02h + vpaddq ymm20{k1}{z},ymm20,ymm25 + vpaddq ymm21{k1}{z},ymm21,ymm1 + vpaddq ymm22{k1}{z},ymm22,ymm26 + vpaddq ymm23{k1}{z},ymm23,ymm2 + vpsrlq ymm29,ymm18,44 + vpsllq ymm19,ymm19,8 + vpandq ymm0,ymm18,ymm30 + vpaddq ymm19,ymm19,ymm29 + vpaddq ymm20,ymm20,ymm19 + vpsrlq ymm29,ymm20,44 + vpsllq ymm21,ymm21,8 + vpandq ymm1,ymm20,ymm30 + vpaddq ymm21,ymm21,ymm29 + vpaddq ymm22,ymm22,ymm21 + vpsrlq ymm29,ymm22,42 + vpsllq ymm23,ymm23,10 + vpandq ymm2,ymm22,YMMWORD PTR[$L$x_mask42] + vpaddq ymm23,ymm23,ymm29 + vpaddq ymm0,ymm0,ymm23 + vpsllq ymm23,ymm23,2 + vpaddq ymm0,ymm0,ymm23 + vpsrlq ymm29,ymm0,44 + vpandq ymm0,ymm0,ymm30 + vpaddq ymm1,ymm1,ymm29 + sub rdx,2 + ja $L$blocks_vpmadd52_4x_do + vmovq QWORD PTR[rdi],xmm0 + vmovq QWORD PTR[8+rdi],xmm1 + vmovq QWORD PTR[16+rdi],xmm2 + vzeroall + +$L$poly1305_no_data_vpmadd52_4x:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret +$L$SEH_end_poly1305_blocks_vpmadd52_4x:: +poly1305_blocks_vpmadd52_4x ENDP + +ALIGN 32 +Poly1305EmitAVX512IFMA PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_poly1305_emit_base2_44:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov r8,QWORD PTR[rdi] + mov r9,QWORD PTR[8+rdi] + mov r10,QWORD PTR[16+rdi] + mov rax,r9 + shr r9,20 + shl rax,44 + mov rcx,r10 + shr r10,40 + shl rcx,24 + add r8,rax + adc r9,rcx + adc r10,0 + mov rax,r8 + add r8,5 + mov rcx,r9 + adc r9,0 + adc r10,0 + shr r10,2 + cmovnz rax,r8 + cmovnz rcx,r9 + add rax,QWORD PTR[rdx] + adc rcx,QWORD PTR[8+rdx] + mov QWORD PTR[rsi],rax + mov QWORD PTR[8+rsi],rcx + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret +$L$SEH_end_poly1305_emit_base2_44:: +Poly1305EmitAVX512IFMA ENDP +ALIGN 64 +$L$const:: +$L$mask24:: + DD 00ffffffh,0,00ffffffh,0,00ffffffh,0,00ffffffh,0 +$L$129:: + DD 16777216,0,16777216,0,16777216,0,16777216,0 +$L$mask26:: + DD 03ffffffh,0,03ffffffh,0,03ffffffh,0,03ffffffh,0 +$L$permd_avx2:: + DD 2,2,2,3,2,0,2,1 +$L$permd_avx512:: + DD 0,0,0,1,0,2,0,3,0,4,0,5,0,6,0,7 + +$L$2_44_inp_permd:: + DD 0,1,1,2,2,3,7,7 +$L$2_44_inp_shift:: + DQ 0,12,24,64 +$L$2_44_mask:: + DQ 0fffffffffffh,0fffffffffffh,03ffffffffffh,0ffffffffffffffffh +$L$2_44_shift_rgt:: + DQ 44,44,42,64 +$L$2_44_shift_lft:: + DQ 8,8,10,64 + +ALIGN 64 +$L$x_mask44:: + DQ 0fffffffffffh,0fffffffffffh,0fffffffffffh,0fffffffffffh + DQ 0fffffffffffh,0fffffffffffh,0fffffffffffh,0fffffffffffh +$L$x_mask42:: + DQ 03ffffffffffh,03ffffffffffh,03ffffffffffh,03ffffffffffh + DQ 03ffffffffffh,03ffffffffffh,03ffffffffffh,03ffffffffffh + +ALIGN 16 +poly1305_se_handler PROC PRIVATE + DB 243,15,30,250 + push rsi + push rdi + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + pushfq + sub rsp,64 + mov rax,QWORD PTR[120+r8] + mov rbx,QWORD PTR[248+r8] + mov rsi,QWORD PTR[8+r9] + mov r11,QWORD PTR[56+r9] + mov r10d,DWORD PTR[r11] + lea r10,QWORD PTR[r10*1+rsi] + cmp rbx,r10 + jb $L$poly1305_common_seh_tail + mov rax,QWORD PTR[152+r8] + mov r10d,DWORD PTR[4+r11] + lea r10,QWORD PTR[r10*1+rsi] + cmp rbx,r10 + jae $L$poly1305_common_seh_tail + lea rax,QWORD PTR[56+rax] + mov rbx,QWORD PTR[((-8))+rax] + mov rbp,QWORD PTR[((-16))+rax] + mov r12,QWORD PTR[((-24))+rax] + mov r13,QWORD PTR[((-32))+rax] + mov r14,QWORD PTR[((-40))+rax] + mov r15,QWORD PTR[((-48))+rax] + mov QWORD PTR[144+r8],rbx + mov QWORD PTR[160+r8],rbp + mov QWORD PTR[216+r8],r12 + mov QWORD PTR[224+r8],r13 + mov QWORD PTR[232+r8],r14 + mov QWORD PTR[240+r8],r15 + jmp $L$poly1305_common_seh_tail +poly1305_se_handler ENDP + +ALIGN 16 +poly1305_avx_handler PROC PRIVATE + DB 243,15,30,250 + push rsi + push rdi + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + pushfq + sub rsp,64 + mov rax,QWORD PTR[120+r8] + mov rbx,QWORD PTR[248+r8] + mov rsi,QWORD PTR[8+r9] + mov r11,QWORD PTR[56+r9] + mov r10d,DWORD PTR[r11] + lea r10,QWORD PTR[r10*1+rsi] + cmp rbx,r10 + jb $L$poly1305_common_seh_tail + mov rax,QWORD PTR[152+r8] + mov r10d,DWORD PTR[4+r11] + lea r10,QWORD PTR[r10*1+rsi] + cmp rbx,r10 + jae $L$poly1305_common_seh_tail + mov rax,QWORD PTR[208+r8] + lea rsi,QWORD PTR[80+rax] + lea rax,QWORD PTR[248+rax] + lea rdi,QWORD PTR[512+r8] + mov ecx,20 + DD 0a548f3fch + +$L$poly1305_common_seh_tail:: + mov rdi,QWORD PTR[8+rax] + mov rsi,QWORD PTR[16+rax] + mov QWORD PTR[152+r8],rax + mov QWORD PTR[168+r8],rsi + mov QWORD PTR[176+r8],rdi + mov rdi,QWORD PTR[40+r9] + mov rsi,r8 + mov ecx,154 + DD 0a548f3fch + mov rsi,r9 + xor rcx,rcx + mov rdx,QWORD PTR[8+rsi] + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[16+rsi] + mov r10,QWORD PTR[40+rsi] + lea r11,QWORD PTR[56+rsi] + lea r12,QWORD PTR[24+rsi] + mov QWORD PTR[32+rsp],r10 + mov QWORD PTR[40+rsp],r11 + mov QWORD PTR[48+rsp],r12 + mov QWORD PTR[56+rsp],rcx + call QWORD PTR[__imp_RtlVirtualUnwind] + mov eax,1 + add rsp,64 + popfq + pop r15 + pop r14 + pop r13 + pop r12 + pop rbp + pop rbx + pop rdi + pop rsi + DB 0F3h,0C3h ;repret +poly1305_avx_handler ENDP + +.text$ ENDS +.pdata SEGMENT READONLY ALIGN(4) +ALIGN 4 + DD imagerel $L$SEH_begin_chacha20_ctr32 + DD imagerel $L$SEH_end_chacha20_ctr32 + DD imagerel $L$SEH_info_chacha20_ctr32 + DD imagerel $L$SEH_begin_chacha20_ssse3 + DD imagerel $L$SEH_end_chacha20_ssse3 + DD imagerel $L$SEH_info_chacha20_ssse3 + DD imagerel $L$SEH_begin_chacha20_128 + DD imagerel $L$SEH_end_chacha20_128 + DD imagerel $L$SEH_info_chacha20_128 + DD imagerel $L$SEH_begin_chacha20_4x + DD imagerel $L$SEH_end_chacha20_4x + DD imagerel $L$SEH_info_chacha20_4x + DD imagerel $L$SEH_begin_chacha20_avx2 + DD imagerel $L$SEH_end_chacha20_avx2 + DD imagerel $L$SEH_info_chacha20_avx2 + DD imagerel $L$SEH_begin_chacha20_avx512 + DD imagerel $L$SEH_end_chacha20_avx512 + DD imagerel $L$SEH_info_chacha20_avx512 + DD imagerel $L$SEH_begin_chacha20_avx512vl + DD imagerel $L$SEH_end_chacha20_avx512vl + DD imagerel $L$SEH_info_chacha20_avx512vl + DD imagerel $L$SEH_begin_chacha20_16x + DD imagerel $L$SEH_end_chacha20_16x + DD imagerel $L$SEH_info_chacha20_16x + DD imagerel $L$SEH_begin_chacha20_8xvl + DD imagerel $L$SEH_end_chacha20_8xvl + DD imagerel $L$SEH_info_chacha20_8xvl +.pdata ENDS +.xdata SEGMENT READONLY ALIGN(8) +ALIGN 8 +$L$SEH_info_chacha20_ctr32:: +DB 9,0,0,0 + DD imagerel chacha20_se_handler + +$L$SEH_info_chacha20_ssse3:: +DB 9,0,0,0 + DD imagerel chacha20_simd_handler + DD imagerel $L$ssse3_body,imagerel $L$ssse3_epilogue + DD 020h,0 + +$L$SEH_info_chacha20_128:: +DB 9,0,0,0 + DD imagerel chacha20_simd_handler + DD imagerel $L$128_body,imagerel $L$128_epilogue + DD 060h,0 + +$L$SEH_info_chacha20_4x:: +DB 9,0,0,0 + DD imagerel chacha20_simd_handler + DD imagerel $L$4x_body,imagerel $L$4x_epilogue + DD 0a0h,0 +$L$SEH_info_chacha20_avx2:: +DB 9,0,0,0 + DD imagerel chacha20_simd_handler + DD imagerel $L$avx2_body,imagerel $L$avx2_epilogue + DD 0a0h,0 +$L$SEH_info_chacha20_avx512:: +DB 9,0,0,0 + DD imagerel chacha20_simd_handler + DD imagerel $L$avx512_body,imagerel $L$avx512_epilogue + DD 020h,0 + +$L$SEH_info_chacha20_avx512vl:: +DB 9,0,0,0 + DD imagerel chacha20_simd_handler + DD imagerel $L$avx512vl_body,imagerel $L$avx512vl_epilogue + DD 020h,0 + +$L$SEH_info_chacha20_16x:: +DB 9,0,0,0 + DD imagerel chacha20_simd_handler + DD imagerel $L$16x_body,imagerel $L$16x_epilogue + DD 0a0h,0 + +$L$SEH_info_chacha20_8xvl:: +DB 9,0,0,0 + DD imagerel chacha20_simd_handler + DD imagerel $L$8xvl_body,imagerel $L$8xvl_epilogue + DD 0a0h,0 +.xdata ENDS +.pdata SEGMENT READONLY ALIGN(4) +ALIGN 4 +.pdata ENDS +.xdata SEGMENT READONLY ALIGN(8) +ALIGN 8 + +.xdata ENDS +END + +.pdata SEGMENT READONLY ALIGN(4) +ALIGN 4 + DD imagerel $L$SEH_begin_poly1305_init + DD imagerel $L$SEH_end_poly1305_init + DD imagerel $L$SEH_info_poly1305_init + DD imagerel $L$SEH_begin_poly1305_blocks + DD imagerel $L$SEH_end_poly1305_blocks + DD imagerel $L$SEH_info_poly1305_blocks + DD imagerel $L$SEH_begin_poly1305_emit + DD imagerel $L$SEH_end_poly1305_emit + DD imagerel $L$SEH_info_poly1305_emit + DD imagerel $L$SEH_begin_poly1305_blocks_avx + DD imagerel $L$base2_64_avx + DD imagerel $L$SEH_info_poly1305_blocks_avx_1 + DD imagerel $L$base2_64_avx + DD imagerel $L$even_avx + DD imagerel $L$SEH_info_poly1305_blocks_avx_2 + DD imagerel $L$even_avx + DD imagerel $L$SEH_end_poly1305_blocks_avx + DD imagerel $L$SEH_info_poly1305_blocks_avx_3 + DD imagerel $L$SEH_begin_poly1305_blocks_avx2 + DD imagerel $L$base2_64_avx2 + DD imagerel $L$SEH_info_poly1305_blocks_avx2_1 + DD imagerel $L$base2_64_avx2 + DD imagerel $L$even_avx2 + DD imagerel $L$SEH_info_poly1305_blocks_avx2_2 + DD imagerel $L$even_avx2 + DD imagerel $L$SEH_end_poly1305_blocks_avx2 + DD imagerel $L$SEH_info_poly1305_blocks_avx2_3 + DD imagerel $L$SEH_begin_poly1305_blocks_avx512 + DD imagerel $L$SEH_end_poly1305_blocks_avx512 + DD imagerel $L$SEH_info_poly1305_blocks_avx512 + DD imagerel $L$SEH_begin_poly1305_init_base2_44 + DD imagerel $L$SEH_end_poly1305_init_base2_44 + DD imagerel $L$SEH_info_poly1305_init_base2_44 + DD imagerel $L$SEH_begin_poly1305_blocks_base2_44 + DD imagerel $L$SEH_end_poly1305_blocks_base2_44 + DD imagerel $L$SEH_info_poly1305_blocks_base2_44 + DD imagerel $L$SEH_begin_poly1305_blocks_vpmadd52 + DD imagerel $L$SEH_end_poly1305_blocks_vpmadd52 + DD imagerel $L$SEH_info_poly1305_blocks_vpmadd52 + DD imagerel $L$SEH_begin_poly1305_blocks_vpmadd52_4x + DD imagerel $L$SEH_end_poly1305_blocks_vpmadd52_4x + DD imagerel $L$SEH_info_poly1305_blocks_vpmadd52_4x + DD imagerel $L$SEH_begin_poly1305_emit_base2_44 + DD imagerel $L$SEH_end_poly1305_emit_base2_44 + DD imagerel $L$SEH_info_poly1305_emit_base2_44 +.pdata ENDS +.xdata SEGMENT READONLY ALIGN(8) +ALIGN 8 +$L$SEH_info_poly1305_init:: +DB 9,0,0,0 + DD imagerel poly1305_se_handler + DD 0,0 + +$L$SEH_info_poly1305_blocks:: +DB 9,0,0,0 + DD imagerel poly1305_se_handler + DD imagerel $L$blocks_body,imagerel $L$blocks_epilogue + +$L$SEH_info_poly1305_emit:: +DB 9,0,0,0 + DD imagerel poly1305_se_handler + DD 0,0 +$L$SEH_info_poly1305_blocks_avx_1:: +DB 9,0,0,0 + DD imagerel poly1305_se_handler + DD imagerel $L$blocks_avx_body,imagerel $L$blocks_avx_epilogue + +$L$SEH_info_poly1305_blocks_avx_2:: +DB 9,0,0,0 + DD imagerel poly1305_se_handler + DD imagerel $L$base2_64_avx_body,imagerel $L$base2_64_avx_epilogue + +$L$SEH_info_poly1305_blocks_avx_3:: +DB 9,0,0,0 + DD imagerel poly1305_avx_handler + DD imagerel $L$do_avx_body,imagerel $L$do_avx_epilogue +$L$SEH_info_poly1305_blocks_avx2_1:: +DB 9,0,0,0 + DD imagerel poly1305_se_handler + DD imagerel $L$blocks_avx2_body,imagerel $L$blocks_avx2_epilogue + +$L$SEH_info_poly1305_blocks_avx2_2:: +DB 9,0,0,0 + DD imagerel poly1305_se_handler + DD imagerel $L$base2_64_avx2_body,imagerel $L$base2_64_avx2_epilogue + +$L$SEH_info_poly1305_blocks_avx2_3:: +DB 9,0,0,0 + DD imagerel poly1305_avx_handler + DD imagerel $L$do_avx2_body,imagerel $L$do_avx2_epilogue +$L$SEH_info_poly1305_blocks_avx512:: +DB 9,0,0,0 + DD imagerel poly1305_avx_handler + DD imagerel $L$do_avx512_body,imagerel $L$do_avx512_epilogue +$L$SEH_info_poly1305_init_base2_44:: +DB 9,0,0,0 + DD imagerel poly1305_se_handler + DD 0,0 + +$L$SEH_info_poly1305_blocks_base2_44:: +DB 9,0,0,0 + DD imagerel poly1305_se_handler + DD imagerel $L$blocks_base2_44_body,imagerel $L$blocks_base2_44_epilogue + +$L$SEH_info_poly1305_blocks_vpmadd52:: +DB 9,0,0,0 + DD imagerel poly1305_se_handler + DD 0,0 + +$L$SEH_info_poly1305_blocks_vpmadd52_4x:: +DB 9,0,0,0 + DD imagerel poly1305_se_handler + DD 0,0 + +$L$SEH_info_poly1305_emit_base2_44:: +DB 9,0,0,0 + DD imagerel poly1305_se_handler + DD 0,0 +.xdata ENDS +.pdata SEGMENT READONLY ALIGN(4) +ALIGN 4 +.pdata ENDS +.xdata SEGMENT READONLY ALIGN(8) +ALIGN 8 + +.xdata ENDS +END diff --git a/driver/crypto.c b/driver/crypto.c new file mode 100644 index 0000000..695cec0 --- /dev/null +++ b/driver/crypto.c @@ -0,0 +1,2888 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include "crypto.h" +#include "arithmetic.h" +#include "memory.h" + +#pragma warning(disable : 4244) /* '=': conversion from 'UINT32' to 'UINT8', possible loss of data */ +#pragma warning(disable : 4267) /* '=': conversion from 'SIZE_T' to 'ULONG', possible loss of data */ +#pragma warning(disable : 4242) /* '=': conversion from 'SIZE_T' to 'UINT32', possible loss of data */ +#pragma warning(disable : 6385) /* Reading invalid data from '<COMPLEX_EXPR>': the readable size is '_Old_5`32' \ + bytes, but '56' bytes may be read. */ +#pragma warning(disable : 26451) /* Arithmetic overflow: Using operator '*' on a 4 byte value and then casting the \ + result to a 8 byte value. Cast the value to the wider type before calling operator \ + '*' to avoid overflow (io.2). */ + +#ifdef ALLOC_PRAGMA +# pragma alloc_text(INIT, CryptoDriverEntry) +#endif +#if defined(_M_AMD64) +# include <intrin.h> + +static CPU_FEATURE CpuFeatures; + +# define CPUID_1_ECX_SSSE3_BIT 9 +# define CPUID_1_ECX_SSSE3_BIT 9 +# define CPUID_1_ECX_SSE3_BIT 0 +# define CPUID_1_EDX_SSE2_BIT 26 +# define CPUID_1_EDX_SSE_BIT 25 +# define CPUID_1_ECX_AVX_BIT 28 +# define CPUID_1_ECX_OSXSAVE_BIT 27 +# define CPUID_70_EBX_AVX2_BIT 5 +# define CPUID_70_EBX_AVX512F_BIT 16 +# define CPUID_70_EBX_AVX512IFMA_BIT 21 +# define CPUID_70_EBX_AVX512VL_BIT 31 +# define WORD_EAX 0 +# define WORD_EBX 1 +# define WORD_ECX 2 +# define WORD_EDX 3 + +typedef struct _CPUID_BIT_INFO +{ + BYTE Leaf; + BYTE Word; + BYTE Bitno; + CPU_FEATURE RequiredBy; +} CPUID_BIT_INFO; + +static CONST CPUID_BIT_INFO CpuidBitInfo[] = { + { 1, WORD_EDX, CPUID_1_EDX_SSE_BIT, CPU_FEATURE_SSSE3 }, + { 1, WORD_EDX, CPUID_1_EDX_SSE2_BIT, CPU_FEATURE_SSSE3 }, + { 1, WORD_ECX, CPUID_1_ECX_SSE3_BIT, CPU_FEATURE_SSSE3 }, + { 1, WORD_ECX, CPUID_1_ECX_SSSE3_BIT, CPU_FEATURE_SSSE3 }, + { 1, WORD_ECX, CPUID_1_ECX_AVX_BIT, CPU_FEATURE_AVX }, + { 7, WORD_EBX, CPUID_70_EBX_AVX2_BIT, CPU_FEATURE_AVX2 }, + { 7, WORD_EBX, CPUID_70_EBX_AVX512F_BIT, CPU_FEATURE_AVX512F }, + { 7, WORD_EBX, CPUID_70_EBX_AVX512IFMA_BIT, CPU_FEATURE_AVX512IFMA }, + { 7, WORD_EBX, CPUID_70_EBX_AVX512VL_BIT, CPU_FEATURE_AVX512VL }, +}; + +VOID CryptoDriverEntry(VOID) +{ + /* It's not like it's exactly hard or complicated to support Windows 7, 8, or 8.1 kernels here, + * but it also means more testing, and given how poorly suited those old network stacks are for + * high speed networking, it's simpler to just fall back to the slow implementations, and concern + * ourselves with Windows 10 (and later, given the recent bout of Start Menu meddling). + */ + RTL_OSVERSIONINFOW OsVersionInfo = { .dwOSVersionInfoSize = sizeof(OsVersionInfo) }; + if (!NT_SUCCESS(RtlGetVersion(&OsVersionInfo)) || OsVersionInfo.dwMajorVersion < 10) + return; + + CPU_FEATURE DisabledCpuFeatures = + ~(CPU_FEATURE_SSSE3 | CPU_FEATURE_AVX | CPU_FEATURE_AVX2 | CPU_FEATURE_AVX512F | CPU_FEATURE_AVX512VL | + CPU_FEATURE_AVX512IFMA); + int CpuInfo[4], InfoType, MaxInfoType; + BOOLEAN IsIntel, IsSkylakeX, HasOSXSAVE; + + __cpuid(CpuInfo, InfoType = 0); + MaxInfoType = CpuInfo[WORD_EAX]; + IsIntel = CpuInfo[WORD_EBX] == 0x756e6547 && CpuInfo[WORD_EDX] == 0x49656e69 && CpuInfo[WORD_ECX] == 0x6c65746e; + __cpuid(CpuInfo, InfoType = 1); + IsSkylakeX = IsIntel && (CpuInfo[WORD_EAX] & 0xf0ff0) == 0x50650; + HasOSXSAVE = !!(CpuInfo[WORD_ECX] & (1 << CPUID_1_ECX_OSXSAVE_BIT)); + + for (ULONG i = 0; i < ARRAYSIZE(CpuidBitInfo); ++i) + { + if (CpuidBitInfo[i].Leaf != InfoType) + __cpuid(CpuInfo, InfoType = CpuidBitInfo[i].Leaf); + if (CpuidBitInfo[i].Leaf > MaxInfoType || !(CpuInfo[CpuidBitInfo[i].Word] & (1UL << CpuidBitInfo[i].Bitno))) + DisabledCpuFeatures |= CpuidBitInfo[i].RequiredBy; + } + + ULONG64 FeatureMask = RtlGetEnabledExtendedFeatures((ULONG64)(-1)) & (ULONG64)(HasOSXSAVE ? _xgetbv(0) : 0); + if ((FeatureMask & (XSTATE_MASK_GSSE | XSTATE_MASK_AVX)) != (XSTATE_MASK_GSSE | XSTATE_MASK_AVX)) + DisabledCpuFeatures |= CPU_FEATURE_AVX | CPU_FEATURE_AVX2; + if ((FeatureMask & (XSTATE_MASK_GSSE | XSTATE_MASK_AVX | XSTATE_MASK_AVX512)) != + (XSTATE_MASK_GSSE | XSTATE_MASK_AVX | XSTATE_MASK_AVX512)) + DisabledCpuFeatures |= CPU_FEATURE_AVX512F | CPU_FEATURE_AVX512VL | CPU_FEATURE_AVX512IFMA; + + if (DisabledCpuFeatures & CPU_FEATURE_SSSE3) + DisabledCpuFeatures |= CPU_FEATURE_AVX; + if (DisabledCpuFeatures & CPU_FEATURE_AVX) + DisabledCpuFeatures |= CPU_FEATURE_AVX2; + if (DisabledCpuFeatures & CPU_FEATURE_AVX2) + DisabledCpuFeatures |= CPU_FEATURE_AVX512F; + if (DisabledCpuFeatures & CPU_FEATURE_AVX512F) + DisabledCpuFeatures |= CPU_FEATURE_AVX512VL; + if (DisabledCpuFeatures & CPU_FEATURE_AVX512F) + DisabledCpuFeatures |= CPU_FEATURE_AVX512IFMA; + + /* AVX512F downclocks too much on Skylake X, but VL is fine. */ + if (IsSkylakeX) + DisabledCpuFeatures |= CPU_FEATURE_AVX512F; + + CpuFeatures = ~DisabledCpuFeatures; +} + +_Use_decl_annotations_ +VOID +SimdGet(SIMD_STATE *State) +{ + State->HasSavedXState = FALSE; + State->CpuFeatures = CpuFeatures; + if (CpuFeatures & (CPU_FEATURE_AVX512F | CPU_FEATURE_AVX512VL | CPU_FEATURE_AVX512IFMA)) + { + State->HasSavedXState = + NT_SUCCESS(KeSaveExtendedProcessorState(XSTATE_MASK_AVX | XSTATE_MASK_AVX512, &State->XState)); + if (State->HasSavedXState) + return; + State->CpuFeatures &= ~(CPU_FEATURE_AVX512F | CPU_FEATURE_AVX512VL | CPU_FEATURE_AVX512IFMA); + } + + if (CpuFeatures & (CPU_FEATURE_AVX2 | CPU_FEATURE_AVX)) + { + State->HasSavedXState = NT_SUCCESS(KeSaveExtendedProcessorState(XSTATE_MASK_AVX, &State->XState)); + if (State->HasSavedXState) + return; + State->CpuFeatures &= ~(CPU_FEATURE_AVX2 | CPU_FEATURE_AVX); + } + + /* Sometimes State->XState isn't initialized, because of HaveSavedXState, but analysis doesn't know that. */ + _Analysis_assume_((RtlFillMemory(State, sizeof(*State), 'A'), TRUE)); + + /* We don't need to save the state for SSSE3 on recent Windows. */ +} + +_Use_decl_annotations_ +VOID +SimdPut(SIMD_STATE *State) +{ + if (!State->HasSavedXState) + { + State->CpuFeatures = 0; + return; + } + KeRestoreExtendedProcessorState(&State->XState); + RtlSecureZeroMemory(State, sizeof(*State)); +} +#else +VOID CryptoDriverEntry(VOID) {} +#endif + +static inline UINT32 +Rol32(_In_ UINT32 Word, _In_ LONG Shift) +{ + return (Word << (Shift & 31)) | (Word >> ((-Shift) & 31)); +} + +static inline UINT32 +Ror32(_In_ UINT32 Word, _In_ LONG Shift) +{ + return (Word >> (Shift & 31)) | (Word << ((-Shift) & 31)); +} + +static inline UINT64 +Rol64(_In_ UINT64 Word, _In_ LONG Shift) +{ + return (Word << (Shift & 63)) | (Word >> ((-Shift) & 63)); +} + +#define Le16ToCpup(X) Le16ToCpu(*(X)) +#define Le32ToCpup(X) Le32ToCpu(*(X)) +#define Le64ToCpup(X) Le64ToCpu(*(X)) + +static inline UINT32 +GetUnalignedLe32(_In_reads_bytes_(4) CONST UINT8 *A) +{ + UINT32 L; + RtlCopyMemory(&L, A, sizeof(L)); + return Le32ToCpup(&L); +} + +static inline UINT64 +GetUnalignedLe64(_In_reads_bytes_(8) CONST UINT8 *A) +{ + UINT64 L; + RtlCopyMemory(&L, A, sizeof(L)); + return Le64ToCpup(&L); +} + +static inline VOID +PutUnalignedLe32(_In_ UINT32 S, _Out_writes_bytes_all_(4) UINT8 *D) +{ + UINT32 L = CpuToLe32(S); + RtlCopyMemory(D, &L, sizeof(L)); +} + +static inline VOID +CpuToLe32Array(_Inout_updates_(Words) UINT32 *Buf, _In_ SIZE_T Words) +{ + while (Words--) + { + *Buf = CpuToLe32(*Buf); + ++Buf; + } +} + +static inline VOID +Le32ToCpuArray(_Inout_updates_(Words) UINT32 *Buf, _In_ SIZE_T Words) +{ + while (Words--) + { + *Buf = Le32ToCpup(Buf); + ++Buf; + } +} + +static VOID +XorCpy( + _Out_writes_bytes_all_(Len) UINT8 *Dst, + _In_reads_bytes_(Len) CONST UINT8 *Src1, + _In_reads_bytes_(Len) CONST UINT8 *Src2, + _In_ SIZE_T Len) +{ + SIZE_T i; + + for (i = 0; i < Len; ++i) + Dst[i] = Src1[i] ^ Src2[i]; +} + +#define QUARTER_ROUND(X, A, B, C, D) \ + (X[A] += X[B], \ + X[D] = Rol32((X[D] ^ X[A]), 16), \ + X[C] += X[D], \ + X[B] = Rol32((X[B] ^ X[C]), 12), \ + X[A] += X[B], \ + X[D] = Rol32((X[D] ^ X[A]), 8), \ + X[C] += X[D], \ + X[B] = Rol32((X[B] ^ X[C]), 7)) + +#define C(i, j) (i * 4 + j) + +#define DOUBLE_ROUND(X) \ + (/* Column Round */ \ + QUARTER_ROUND(X, C(0, 0), C(1, 0), C(2, 0), C(3, 0)), \ + QUARTER_ROUND(X, C(0, 1), C(1, 1), C(2, 1), C(3, 1)), \ + QUARTER_ROUND(X, C(0, 2), C(1, 2), C(2, 2), C(3, 2)), \ + QUARTER_ROUND(X, C(0, 3), C(1, 3), C(2, 3), C(3, 3)), /* Diagonal Round */ \ + QUARTER_ROUND(X, C(0, 0), C(1, 1), C(2, 2), C(3, 3)), \ + QUARTER_ROUND(X, C(0, 1), C(1, 2), C(2, 3), C(3, 0)), \ + QUARTER_ROUND(X, C(0, 2), C(1, 3), C(2, 0), C(3, 1)), \ + QUARTER_ROUND(X, C(0, 3), C(1, 0), C(2, 1), C(3, 2))) + +#define TWENTY_ROUNDS(X) \ + (DOUBLE_ROUND(X), \ + DOUBLE_ROUND(X), \ + DOUBLE_ROUND(X), \ + DOUBLE_ROUND(X), \ + DOUBLE_ROUND(X), \ + DOUBLE_ROUND(X), \ + DOUBLE_ROUND(X), \ + DOUBLE_ROUND(X), \ + DOUBLE_ROUND(X), \ + DOUBLE_ROUND(X)) + +enum CHACHA20_LENGTHS +{ + CHACHA20_NONCE_SIZE = 16, + CHACHA20_KEY_SIZE = 32, + CHACHA20_KEY_WORDS = CHACHA20_KEY_SIZE / sizeof(UINT32), + CHACHA20_BLOCK_SIZE = 64, + CHACHA20_BLOCK_WORDS = CHACHA20_BLOCK_SIZE / sizeof(UINT32), + HCHACHA20_NONCE_SIZE = CHACHA20_NONCE_SIZE, + HCHACHA20_KEY_SIZE = CHACHA20_KEY_SIZE +}; + +enum CHACHA20_CONSTANTS +{ + /* expand 32-byte k */ + CHACHA20_CONSTANT_EXPA = 0x61707865U, + CHACHA20_CONSTANT_ND_3 = 0x3320646eU, + CHACHA20_CONSTANT_2_BY = 0x79622d32U, + CHACHA20_CONSTANT_TE_K = 0x6b206574U +}; + +typedef struct _CHACHA20_CTX +{ + union + { + UINT32 State[16]; + struct + { + UINT32 Constant[4]; + UINT32 Key[8]; + UINT32 Counter[4]; + }; + }; +} CHACHA20_CTX; + +static VOID +ChaCha20Init(_Out_ CHACHA20_CTX *Ctx, _In_ CONST UINT8 Key[CHACHA20_KEY_SIZE], _In_ CONST UINT64 Nonce) +{ + Ctx->Constant[0] = CHACHA20_CONSTANT_EXPA; + Ctx->Constant[1] = CHACHA20_CONSTANT_ND_3; + Ctx->Constant[2] = CHACHA20_CONSTANT_2_BY; + Ctx->Constant[3] = CHACHA20_CONSTANT_TE_K; + Ctx->Key[0] = GetUnalignedLe32(Key + 0); + Ctx->Key[1] = GetUnalignedLe32(Key + 4); + Ctx->Key[2] = GetUnalignedLe32(Key + 8); + Ctx->Key[3] = GetUnalignedLe32(Key + 12); + Ctx->Key[4] = GetUnalignedLe32(Key + 16); + Ctx->Key[5] = GetUnalignedLe32(Key + 20); + Ctx->Key[6] = GetUnalignedLe32(Key + 24); + Ctx->Key[7] = GetUnalignedLe32(Key + 28); + Ctx->Counter[0] = 0; + Ctx->Counter[1] = 0; + Ctx->Counter[2] = Nonce & 0xffffffffU; + Ctx->Counter[3] = Nonce >> 32; +} + +#if defined(_M_AMD64) +VOID +ChaCha20ALU( + _Out_writes_bytes_all_(Len) UINT8 *Dst, + _In_reads_bytes_(Len) CONST UINT8 *Src, + _In_ SIZE_T Len, + _In_ CONST UINT32 Key[8], + _In_ CONST UINT32 Counter[4]); +VOID +ChaCha20SSSE3( + _Out_writes_bytes_all_(Len) UINT8 *Dst, + _In_reads_bytes_(Len) CONST UINT8 *Src, + _In_ SIZE_T Len, + _In_ CONST UINT32 Key[8], + _In_ CONST UINT32 Counter[4]); +VOID +ChaCha20AVX2( + _Out_writes_bytes_all_(Len) UINT8 *Dst, + _In_reads_bytes_(Len) CONST UINT8 *Src, + _In_ SIZE_T Len, + _In_ CONST UINT32 Key[8], + _In_ CONST UINT32 Counter[4]); +VOID +ChaCha20AVX512( + _Out_writes_bytes_all_(Len) UINT8 *Dst, + _In_reads_bytes_(Len) CONST UINT8 *Src, + _In_ SIZE_T Len, + _In_ CONST UINT32 Key[8], + _In_ CONST UINT32 Counter[4]); +VOID +ChaCha20AVX512VL( + _Out_writes_bytes_all_(Len) UINT8 *Dst, + _In_reads_bytes_(Len) CONST UINT8 *Src, + _In_ SIZE_T Len, + _In_ CONST UINT32 Key[8], + _In_ CONST UINT32 Counter[4]); + +static VOID +ChaCha20( + _Inout_ CHACHA20_CTX *Ctx, + _Out_writes_bytes_all_(Len) UINT8 *Out, + _In_reads_bytes_(Len) CONST UINT8 *In, + _In_ UINT32 Len, + _In_opt_ CONST SIMD_STATE *Simd) +{ + if (!Len) + return; + if (Simd && (Simd->CpuFeatures & CPU_FEATURE_AVX512F)) + ChaCha20AVX512(Out, In, Len, Ctx->Key, Ctx->Counter); + else if (Simd && (Simd->CpuFeatures & CPU_FEATURE_AVX512VL)) + ChaCha20AVX512VL(Out, In, Len, Ctx->Key, Ctx->Counter); + else if (Simd && (Simd->CpuFeatures & CPU_FEATURE_AVX2)) + ChaCha20AVX2(Out, In, Len, Ctx->Key, Ctx->Counter); + else if ((Simd && (Simd->CpuFeatures & CPU_FEATURE_SSSE3)) || (!Simd && (CpuFeatures & CPU_FEATURE_SSSE3))) + ChaCha20SSSE3(Out, In, Len, Ctx->Key, Ctx->Counter); + else + ChaCha20ALU(Out, In, Len, Ctx->Key, Ctx->Counter); + Ctx->Counter[0] += (Len + 63) / 64; +} + +static VOID +ChaCha20Block( + _Inout_ CHACHA20_CTX *Ctx, + _Out_writes_all_(CHACHA20_BLOCK_WORDS) UINT32 Stream[CHACHA20_BLOCK_WORDS], + _In_opt_ CONST SIMD_STATE *Simd) +{ + static CONST UINT32 ZeroInput[CHACHA20_BLOCK_WORDS] = { 0 }; + ChaCha20(Ctx, (UINT8 *)Stream, (CONST UINT8 *)ZeroInput, sizeof(ZeroInput), Simd); +} +#else +static VOID +ChaCha20Block( + _Inout_ CHACHA20_CTX *Ctx, + _Out_writes_all_(CHACHA20_BLOCK_WORDS) UINT32 Stream[CHACHA20_BLOCK_WORDS], + _In_opt_ CONST SIMD_STATE *Simd) +{ + UINT32 X[CHACHA20_BLOCK_WORDS]; + LONG i; + + for (i = 0; i < ARRAYSIZE(X); ++i) + X[i] = Ctx->State[i]; + + TWENTY_ROUNDS(X); + + for (i = 0; i < ARRAYSIZE(X); ++i) + Stream[i] = CpuToLe32(X[i] + Ctx->State[i]); + + Ctx->Counter[0] += 1; +} + +static VOID +ChaCha20( + _Inout_ CHACHA20_CTX *Ctx, + _Out_writes_bytes_all_(Len) UINT8 *Out, + _In_reads_bytes_(Len) CONST UINT8 *In, + _In_ UINT32 Len, + _In_opt_ CONST SIMD_STATE *Simd) +{ + UINT32 Buf[CHACHA20_BLOCK_WORDS]; + + while (Len >= CHACHA20_BLOCK_SIZE) + { + ChaCha20Block(Ctx, Buf, Simd); + XorCpy(Out, In, (UINT8 *)Buf, CHACHA20_BLOCK_SIZE); + Len -= CHACHA20_BLOCK_SIZE; + Out += CHACHA20_BLOCK_SIZE; + In += CHACHA20_BLOCK_SIZE; + } + if (Len) + { + ChaCha20Block(Ctx, Buf, Simd); + XorCpy(Out, In, (UINT8 *)Buf, Len); + } +} +#endif + +static VOID +HChaCha20( + _Out_writes_all_(CHACHA20_KEY_WORDS) UINT32 DerivedKey[CHACHA20_KEY_WORDS], + _In_ CONST UINT8 Nonce[HCHACHA20_NONCE_SIZE], + _In_ CONST UINT8 Key[HCHACHA20_KEY_SIZE]) +{ + UINT32 X[] = { CHACHA20_CONSTANT_EXPA, CHACHA20_CONSTANT_ND_3, CHACHA20_CONSTANT_2_BY, + CHACHA20_CONSTANT_TE_K, GetUnalignedLe32(Key + 0), GetUnalignedLe32(Key + 4), + GetUnalignedLe32(Key + 8), GetUnalignedLe32(Key + 12), GetUnalignedLe32(Key + 16), + GetUnalignedLe32(Key + 20), GetUnalignedLe32(Key + 24), GetUnalignedLe32(Key + 28), + GetUnalignedLe32(Nonce + 0), GetUnalignedLe32(Nonce + 4), GetUnalignedLe32(Nonce + 8), + GetUnalignedLe32(Nonce + 12) }; + + TWENTY_ROUNDS(X); + + RtlCopyMemory(DerivedKey + 0, X + 0, sizeof(UINT32) * 4); + RtlCopyMemory(DerivedKey + 4, X + 12, sizeof(UINT32) * 4); +} + +enum POLY1305_LENGTHS +{ + POLY1305_BLOCK_SIZE = 16, + POLY1305_KEY_SIZE = 32, + POLY1305_MAC_SIZE = 16 +}; + +#if defined(_M_AMD64) +typedef union _POLY1305_INTERNAL +{ + struct + { + UINT64 H[3]; + UINT64 R[2]; + } Base264; + struct + { + UINT32 H[5]; + UINT32 IsBase226; + UINT64 R[2]; + UINT64 Pad; + struct + { + UINT32 R2, R1, R4, R3; + } RP[9]; + } Base226; + struct + { + UINT64 H[3]; + UINT64 S[2]; + UINT64 R[3]; + struct + { + UINT32 R1, R3, R2, R4; + } RP[4]; + } Base244; +} POLY1305_INTERNAL; + +VOID +Poly1305InitALU(_Out_ POLY1305_INTERNAL *Ctx, _In_ CONST UINT8 Key[POLY1305_BLOCK_SIZE]); +VOID +Poly1305InitAVX512IFMA(_Out_ POLY1305_INTERNAL *Ctx, _In_ CONST UINT8 Key[POLY1305_BLOCK_SIZE]); +VOID +Poly1305BlocksALU( + _Inout_ POLY1305_INTERNAL *Ctx, + _In_reads_bytes_(Len) CONST UINT8 *In, + _In_ CONST SIZE_T Len, + _In_ CONST UINT32 PadBit); +VOID +Poly1305BlocksAVX( + _Inout_ POLY1305_INTERNAL *Ctx, + _In_reads_bytes_(Len) CONST UINT8 *In, + _In_ CONST SIZE_T Len, + _In_ CONST UINT32 PadBit); +VOID +Poly1305BlocksAVX2( + _Inout_ POLY1305_INTERNAL *Ctx, + _In_reads_bytes_(Len) CONST UINT8 *In, + _In_ CONST SIZE_T Len, + _In_ CONST UINT32 PadBit); +VOID +Poly1305BlocksAVX512IFMA( + _Inout_ POLY1305_INTERNAL *Ctx, + _In_reads_bytes_(Len) CONST UINT8 *In, + _In_ CONST SIZE_T Len, + _In_ CONST UINT32 PadBit); +VOID +Poly1305EmitALU( + _In_ CONST POLY1305_INTERNAL *Ctx, + _Out_writes_bytes_all_(POLY1305_MAC_SIZE) UINT8 Mac[POLY1305_MAC_SIZE], + _In_ CONST UINT32 Nonce[4]); +VOID +Poly1305EmitAVX512IFMA( + _In_ CONST POLY1305_INTERNAL *Ctx, + _Out_writes_bytes_all_(POLY1305_MAC_SIZE) UINT8 Mac[POLY1305_MAC_SIZE], + _In_ CONST UINT32 Nonce[4]); + +static VOID +Poly1305InitCore(_Out_ POLY1305_INTERNAL *St, _In_ CONST UINT8 Key[16], _In_opt_ CONST SIMD_STATE *Simd) +{ + if (Simd && (Simd->CpuFeatures & CPU_FEATURE_AVX512IFMA)) + Poly1305InitAVX512IFMA(St, Key); + else + Poly1305InitALU(St, Key); +} + +static VOID +Poly1305BlocksCore( + _Inout_ POLY1305_INTERNAL *St, + _In_reads_bytes_(Len) CONST UINT8 *Input, + _In_ SIZE_T Len, + _In_ CONST UINT32 PadBit, + _In_opt_ CONST SIMD_STATE *Simd) +{ + if (Simd && (Simd->CpuFeatures & CPU_FEATURE_AVX512IFMA)) + Poly1305BlocksAVX512IFMA(St, Input, Len, PadBit); + else if (Simd && (Simd->CpuFeatures & CPU_FEATURE_AVX2)) + Poly1305BlocksAVX2(St, Input, Len, PadBit); + else if (Simd && (Simd->CpuFeatures & CPU_FEATURE_AVX)) + Poly1305BlocksAVX(St, Input, Len, PadBit); + else + Poly1305BlocksALU(St, Input, Len, PadBit); +} + +static VOID +Poly1305EmitCore( + _In_ CONST POLY1305_INTERNAL *St, + _Out_writes_bytes_all_(16) UINT8 Mac[16], + _In_ CONST UINT32 Nonce[4], + _In_opt_ CONST SIMD_STATE *Simd) +{ + if (Simd && (Simd->CpuFeatures & CPU_FEATURE_AVX512IFMA)) + Poly1305EmitAVX512IFMA(St, Mac, Nonce); + else + Poly1305EmitALU(St, Mac, Nonce); +} +#else +typedef struct _POLY1305_INTERNAL +{ + UINT32 H[5]; + UINT32 R[5]; + UINT32 S[4]; +} POLY1305_INTERNAL; + +static VOID +Poly1305InitCore(_Out_ POLY1305_INTERNAL *St, _In_ CONST UINT8 Key[16], _In_opt_ CONST SIMD_STATE *Simd) +{ + /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ + St->R[0] = (GetUnalignedLe32(&Key[0])) & 0x3ffffff; + St->R[1] = (GetUnalignedLe32(&Key[3]) >> 2) & 0x3ffff03; + St->R[2] = (GetUnalignedLe32(&Key[6]) >> 4) & 0x3ffc0ff; + St->R[3] = (GetUnalignedLe32(&Key[9]) >> 6) & 0x3f03fff; + St->R[4] = (GetUnalignedLe32(&Key[12]) >> 8) & 0x00fffff; + + /* s = 5*r */ + St->S[0] = St->R[1] * 5; + St->S[1] = St->R[2] * 5; + St->S[2] = St->R[3] * 5; + St->S[3] = St->R[4] * 5; + + /* h = 0 */ + St->H[0] = 0; + St->H[1] = 0; + St->H[2] = 0; + St->H[3] = 0; + St->H[4] = 0; +} + +static VOID +Poly1305BlocksCore( + _Inout_ POLY1305_INTERNAL *St, + _In_reads_bytes_(Len) CONST UINT8 *Input, + _In_ SIZE_T Len, + _In_ CONST UINT32 PadBit, + _In_opt_ CONST SIMD_STATE *Simd) +{ + CONST UINT32 Hibit = PadBit << 24; + UINT32 R0, R1, R2, R3, R4; + UINT32 S1, S2, S3, S4; + UINT32 H0, H1, H2, H3, H4; + UINT64 D0, D1, D2, D3, D4; + UINT32 C; + + R0 = St->R[0]; + R1 = St->R[1]; + R2 = St->R[2]; + R3 = St->R[3]; + R4 = St->R[4]; + + S1 = St->S[0]; + S2 = St->S[1]; + S3 = St->S[2]; + S4 = St->S[3]; + + H0 = St->H[0]; + H1 = St->H[1]; + H2 = St->H[2]; + H3 = St->H[3]; + H4 = St->H[4]; + + while (Len >= POLY1305_BLOCK_SIZE) + { + /* h += m[i] */ + H0 += (GetUnalignedLe32(&Input[0])) & 0x3ffffff; + H1 += (GetUnalignedLe32(&Input[3]) >> 2) & 0x3ffffff; + H2 += (GetUnalignedLe32(&Input[6]) >> 4) & 0x3ffffff; + H3 += (GetUnalignedLe32(&Input[9]) >> 6) & 0x3ffffff; + H4 += (GetUnalignedLe32(&Input[12]) >> 8) | Hibit; + + /* h *= r */ + D0 = ((UINT64)H0 * R0) + ((UINT64)H1 * S4) + ((UINT64)H2 * S3) + ((UINT64)H3 * S2) + ((UINT64)H4 * S1); + D1 = ((UINT64)H0 * R1) + ((UINT64)H1 * R0) + ((UINT64)H2 * S4) + ((UINT64)H3 * S3) + ((UINT64)H4 * S2); + D2 = ((UINT64)H0 * R2) + ((UINT64)H1 * R1) + ((UINT64)H2 * R0) + ((UINT64)H3 * S4) + ((UINT64)H4 * S3); + D3 = ((UINT64)H0 * R3) + ((UINT64)H1 * R2) + ((UINT64)H2 * R1) + ((UINT64)H3 * R0) + ((UINT64)H4 * S4); + D4 = ((UINT64)H0 * R4) + ((UINT64)H1 * R3) + ((UINT64)H2 * R2) + ((UINT64)H3 * R1) + ((UINT64)H4 * R0); + + /* (partial) h %= p */ + C = (UINT32)(D0 >> 26); + H0 = (UINT32)D0 & 0x3ffffff; + D1 += C; + C = (UINT32)(D1 >> 26); + H1 = (UINT32)D1 & 0x3ffffff; + D2 += C; + C = (UINT32)(D2 >> 26); + H2 = (UINT32)D2 & 0x3ffffff; + D3 += C; + C = (UINT32)(D3 >> 26); + H3 = (UINT32)D3 & 0x3ffffff; + D4 += C; + C = (UINT32)(D4 >> 26); + H4 = (UINT32)D4 & 0x3ffffff; + H0 += C * 5; + C = (H0 >> 26); + H0 = H0 & 0x3ffffff; + H1 += C; + + Input += POLY1305_BLOCK_SIZE; + Len -= POLY1305_BLOCK_SIZE; + } + + St->H[0] = H0; + St->H[1] = H1; + St->H[2] = H2; + St->H[3] = H3; + St->H[4] = H4; +} + +static VOID +Poly1305EmitCore( + _In_ CONST POLY1305_INTERNAL *St, + _Out_writes_bytes_all_(16) UINT8 Mac[16], + _In_ CONST UINT32 Nonce[4], + _In_opt_ CONST SIMD_STATE *Simd) +{ + UINT32 H0, H1, H2, H3, H4, C; + UINT32 G0, G1, G2, G3, G4; + UINT64 F; + UINT32 Mask; + + /* fully carry h */ + H0 = St->H[0]; + H1 = St->H[1]; + H2 = St->H[2]; + H3 = St->H[3]; + H4 = St->H[4]; + + C = H1 >> 26; + H1 = H1 & 0x3ffffff; + H2 += C; + C = H2 >> 26; + H2 = H2 & 0x3ffffff; + H3 += C; + C = H3 >> 26; + H3 = H3 & 0x3ffffff; + H4 += C; + C = H4 >> 26; + H4 = H4 & 0x3ffffff; + H0 += C * 5; + C = H0 >> 26; + H0 = H0 & 0x3ffffff; + H1 += C; + + /* compute h + -p */ + G0 = H0 + 5; + C = G0 >> 26; + G0 &= 0x3ffffff; + G1 = H1 + C; + C = G1 >> 26; + G1 &= 0x3ffffff; + G2 = H2 + C; + C = G2 >> 26; + G2 &= 0x3ffffff; + G3 = H3 + C; + C = G3 >> 26; + G3 &= 0x3ffffff; + G4 = H4 + C - (1UL << 26); + + /* select h if h < p, or h + -p if h >= p */ + Mask = (G4 >> ((sizeof(UINT32) * 8) - 1)) - 1; + G0 &= Mask; + G1 &= Mask; + G2 &= Mask; + G3 &= Mask; + G4 &= Mask; + Mask = ~Mask; + + H0 = (H0 & Mask) | G0; + H1 = (H1 & Mask) | G1; + H2 = (H2 & Mask) | G2; + H3 = (H3 & Mask) | G3; + H4 = (H4 & Mask) | G4; + + /* h = h % (2^128) */ + H0 = ((H0) | (H1 << 26)) & 0xffffffff; + H1 = ((H1 >> 6) | (H2 << 20)) & 0xffffffff; + H2 = ((H2 >> 12) | (H3 << 14)) & 0xffffffff; + H3 = ((H3 >> 18) | (H4 << 8)) & 0xffffffff; + + /* mac = (h + nonce) % (2^128) */ + F = (UINT64)H0 + Nonce[0]; + H0 = (UINT32)F; + F = (UINT64)H1 + Nonce[1] + (F >> 32); + H1 = (UINT32)F; + F = (UINT64)H2 + Nonce[2] + (F >> 32); + H2 = (UINT32)F; + F = (UINT64)H3 + Nonce[3] + (F >> 32); + H3 = (UINT32)F; + + PutUnalignedLe32(H0, &Mac[0]); + PutUnalignedLe32(H1, &Mac[4]); + PutUnalignedLe32(H2, &Mac[8]); + PutUnalignedLe32(H3, &Mac[12]); +} +#endif + +typedef struct _POLY1305_CTX +{ + POLY1305_INTERNAL State; + UINT32 Nonce[4]; + UINT8 Data[POLY1305_BLOCK_SIZE]; + SIZE_T Num; + CONST SIMD_STATE *Simd; +} POLY1305_CTX; + +static VOID +Poly1305Init(_Out_ POLY1305_CTX *Ctx, _In_ CONST UINT8 Key[POLY1305_KEY_SIZE], _In_opt_ CONST SIMD_STATE *Simd) +{ + Ctx->Nonce[0] = GetUnalignedLe32(&Key[16]); + Ctx->Nonce[1] = GetUnalignedLe32(&Key[20]); + Ctx->Nonce[2] = GetUnalignedLe32(&Key[24]); + Ctx->Nonce[3] = GetUnalignedLe32(&Key[28]); + Ctx->Simd = Simd; + + Poly1305InitCore(&Ctx->State, Key, Ctx->Simd); + + Ctx->Num = 0; +} + +static VOID +Poly1305Update(_Inout_ POLY1305_CTX *Ctx, _In_reads_bytes_(Len) CONST UINT8 *Input, _In_ SIZE_T Len) +{ + CONST SIZE_T Num = Ctx->Num; + SIZE_T Rem; + + if (Num) + { + Rem = POLY1305_BLOCK_SIZE - Num; + if (Len < Rem) + { + RtlCopyMemory(Ctx->Data + Num, Input, Len); + Ctx->Num = Num + Len; + return; + } + RtlCopyMemory(Ctx->Data + Num, Input, Rem); + Poly1305BlocksCore(&Ctx->State, Ctx->Data, POLY1305_BLOCK_SIZE, 1, Ctx->Simd); + Input += Rem; + Len -= Rem; + } + + Rem = Len % POLY1305_BLOCK_SIZE; + Len -= Rem; + + if (Len >= POLY1305_BLOCK_SIZE) + { + Poly1305BlocksCore(&Ctx->State, Input, Len, 1, Ctx->Simd); + Input += Len; + } + + if (Rem) + RtlCopyMemory(Ctx->Data, Input, Rem); + + Ctx->Num = Rem; +} + +static VOID +Poly1305Final(_Inout_ POLY1305_CTX *Ctx, _Out_writes_bytes_all_(16) UINT8 Mac[POLY1305_MAC_SIZE]) +{ + SIZE_T Num = Ctx->Num; + + if (Num) + { + Ctx->Data[Num++] = 1; + while (Num < POLY1305_BLOCK_SIZE) + Ctx->Data[Num++] = 0; + Poly1305BlocksCore(&Ctx->State, Ctx->Data, POLY1305_BLOCK_SIZE, 0, Ctx->Simd); + } + + Poly1305EmitCore(&Ctx->State, Mac, Ctx->Nonce, Ctx->Simd); + + RtlSecureZeroMemory(Ctx, sizeof(*Ctx)); +} + +static CONST UINT8 Pad0[16] = { 0 }; + +_Use_decl_annotations_ +VOID +ChaCha20Poly1305Encrypt( + UINT8 *Dst, + CONST UINT8 *Src, + CONST SIZE_T SrcLen, + CONST UINT8 *Ad, + CONST SIZE_T AdLen, + CONST UINT64 Nonce, + CONST UINT8 Key[CHACHA20POLY1305_KEY_SIZE]) +{ + POLY1305_CTX Poly1305State; + CHACHA20_CTX ChaCha20State; + union + { + UINT8 Block0[POLY1305_KEY_SIZE]; + UINT64 Lens[2]; + } B = { { 0 } }; + + ChaCha20Init(&ChaCha20State, Key, Nonce); + ChaCha20(&ChaCha20State, B.Block0, B.Block0, sizeof(B.Block0), NULL); + Poly1305Init(&Poly1305State, B.Block0, NULL); + + Poly1305Update(&Poly1305State, Ad, AdLen); + Poly1305Update(&Poly1305State, Pad0, (0x10 - AdLen) & 0xf); + + ChaCha20(&ChaCha20State, Dst, Src, SrcLen, NULL); + + Poly1305Update(&Poly1305State, Dst, SrcLen); + Poly1305Update(&Poly1305State, Pad0, (0x10 - SrcLen) & 0xf); + + B.Lens[0] = CpuToLe64(AdLen); + B.Lens[1] = CpuToLe64(SrcLen); + Poly1305Update(&Poly1305State, (UINT8 *)B.Lens, sizeof(B.Lens)); + + Poly1305Final(&Poly1305State, Dst + SrcLen); + + RtlSecureZeroMemory(&ChaCha20State, sizeof(ChaCha20State)); + RtlSecureZeroMemory(&B, sizeof(B)); +} + +_Use_decl_annotations_ +BOOLEAN +ChaCha20Poly1305Decrypt( + UINT8 *Dst, + CONST UINT8 *Src, + CONST SIZE_T SrcLen, + CONST UINT8 *Ad, + CONST SIZE_T AdLen, + CONST UINT64 Nonce, + CONST UINT8 Key[CHACHA20POLY1305_KEY_SIZE]) +{ + POLY1305_CTX Poly1305State; + CHACHA20_CTX ChaCha20State; + BOOLEAN Ret; + SIZE_T DstLen; + union + { + UINT8 Block0[POLY1305_KEY_SIZE]; + UINT8 Mac[POLY1305_MAC_SIZE]; + UINT64 Lens[2]; + } B = { { 0 } }; + + if (SrcLen < POLY1305_MAC_SIZE) + return FALSE; + + ChaCha20Init(&ChaCha20State, Key, Nonce); + ChaCha20(&ChaCha20State, B.Block0, B.Block0, sizeof(B.Block0), NULL); + Poly1305Init(&Poly1305State, B.Block0, NULL); + + Poly1305Update(&Poly1305State, Ad, AdLen); + Poly1305Update(&Poly1305State, Pad0, (0x10 - AdLen) & 0xf); + + DstLen = SrcLen - POLY1305_MAC_SIZE; + Poly1305Update(&Poly1305State, Src, DstLen); + Poly1305Update(&Poly1305State, Pad0, (0x10 - DstLen) & 0xf); + + B.Lens[0] = CpuToLe64(AdLen); + B.Lens[1] = CpuToLe64(DstLen); + Poly1305Update(&Poly1305State, (UINT8 *)B.Lens, sizeof(B.Lens)); + + Poly1305Final(&Poly1305State, B.Mac); + + Ret = CryptoEqualMemory16(B.Mac, Src + DstLen); + if (Ret) + ChaCha20(&ChaCha20State, Dst, Src, DstLen, NULL); + + RtlSecureZeroMemory(&ChaCha20State, sizeof(ChaCha20State)); + RtlSecureZeroMemory(&B, sizeof(B)); + + return Ret; +} + +_Use_decl_annotations_ +BOOLEAN +ChaCha20Poly1305DecryptMdl( + UINT8 *Dst, + MDL *Src, + CONST ULONG SrcLen, + CONST ULONG SrcOffset, + CONST UINT8 *Ad, + CONST SIZE_T AdLen, + CONST UINT64 Nonce, + CONST UINT8 Key[CHACHA20POLY1305_KEY_SIZE], + CONST SIMD_STATE *Simd) +{ + POLY1305_CTX Poly1305State; + CHACHA20_CTX ChaCha20State; + UINT8 *SrcBuf; + ULONG Len, LenMdl, OffsetMdl = SrcOffset, Leftover = 0, Total = SrcLen - POLY1305_MAC_SIZE, Remaining = Total; + MDL *Mdl = Src; + BOOLEAN Ret = FALSE; + union + { + UINT32 Stream[CHACHA20_BLOCK_WORDS]; + UINT8 Block0[POLY1305_KEY_SIZE]; + UINT8 Mac[POLY1305_MAC_SIZE * 2]; + UINT64 Lens[2]; + } B = { { 0 } }; + + if (SrcLen < POLY1305_MAC_SIZE) + return FALSE; + + ChaCha20Init(&ChaCha20State, Key, Nonce); + ChaCha20(&ChaCha20State, B.Block0, B.Block0, sizeof(B.Block0), Simd); + Poly1305Init(&Poly1305State, B.Block0, Simd); + + if (AdLen) + { + Poly1305Update(&Poly1305State, Ad, AdLen); + if (AdLen & 0xf) + Poly1305Update(&Poly1305State, Pad0, 0x10 - (AdLen & 0xf)); + } + + while (OffsetMdl >= MmGetMdlByteCount(Mdl)) + { + OffsetMdl -= MmGetMdlByteCount(Mdl); + Mdl = Mdl->Next; + } + for (;;) + { + if (!Mdl) + goto out; + Len = LenMdl = min(MmGetMdlByteCount(Mdl) - OffsetMdl, Remaining); + SrcBuf = MmGetSystemAddressForMdlSafe(Mdl, NormalPagePriority | MdlMappingNoExecute | MdlMappingNoWrite); + if (!SrcBuf) + goto out; + SrcBuf += OffsetMdl; + + /* Potential TOCTOU? We read the bytes from SrcBuf for Poly1305 here, and later below + * we decrypt those bytes with ChaCha20. If a user on the same physical machine can + * access these pages, I fear it might be possible sneak in a buffer that isn't + * actually authenticated. + */ + Poly1305Update(&Poly1305State, SrcBuf, LenMdl); + + if (Leftover != 0) + { + ULONG l = min(Len, Leftover); + XorCpy(Dst, SrcBuf, ((UINT8 *)B.Stream) + (CHACHA20_BLOCK_SIZE - Leftover), l); + Leftover -= l; + SrcBuf += l; + Dst += l; + Len -= l; + } + + if (Len >= CHACHA20_BLOCK_SIZE) + { + ULONG l = ALIGN_DOWN_BY_T(ULONG, Len, CHACHA20_BLOCK_SIZE); + ChaCha20(&ChaCha20State, Dst, SrcBuf, l, Simd); + SrcBuf += l; + Dst += l; + Len -= l; + } + + if (Len) + { + ChaCha20Block(&ChaCha20State, B.Stream, Simd); + XorCpy(Dst, SrcBuf, (UINT8 *)B.Stream, Len); + Leftover = CHACHA20_BLOCK_SIZE - Len; + Dst += Len; + } + + Remaining -= LenMdl; + if (!Remaining) + { + OffsetMdl += LenMdl; + break; + } + Mdl = Mdl->Next; + OffsetMdl = 0; + } + Poly1305Update(&Poly1305State, Pad0, (0x10 - Total) & 0xf); + B.Lens[0] = CpuToLe64(AdLen); + B.Lens[1] = CpuToLe64(Total); + Poly1305Update(&Poly1305State, (UINT8 *)B.Lens, sizeof(B.Lens)); + Poly1305Final(&Poly1305State, B.Mac); + if (!NT_SUCCESS(MemCopyFromMdl(B.Mac + POLY1305_MAC_SIZE, Mdl, OffsetMdl, POLY1305_MAC_SIZE))) + goto out; + Ret = CryptoEqualMemory16(B.Mac, B.Mac + POLY1305_MAC_SIZE); +out: + RtlSecureZeroMemory(&ChaCha20State, sizeof(ChaCha20State)); + RtlSecureZeroMemory(&B, sizeof(B)); + return Ret; +} + +_Use_decl_annotations_ +BOOLEAN +ChaCha20Poly1305EncryptMdl( + UINT8 *Dst, + MDL *Src, + CONST ULONG SrcLen, + CONST ULONG SrcOffset, + CONST UINT8 *Ad, + CONST SIZE_T AdLen, + CONST UINT64 Nonce, + CONST UINT8 Key[CHACHA20POLY1305_KEY_SIZE], + CONST SIMD_STATE *Simd) +{ + POLY1305_CTX Poly1305State; + CHACHA20_CTX ChaCha20State; + UINT8 *SrcBuf; + MDL *Mdl = Src; + ULONG Len, LenMdl, OffsetMdl = SrcOffset, Leftover = 0; + union + { + UINT32 Stream[CHACHA20_BLOCK_WORDS]; + UINT8 Block0[POLY1305_KEY_SIZE]; + UINT64 Lens[2]; + } B = { { 0 } }; + + ChaCha20Init(&ChaCha20State, Key, Nonce); + ChaCha20(&ChaCha20State, B.Block0, B.Block0, sizeof(B.Block0), Simd); + Poly1305Init(&Poly1305State, B.Block0, Simd); + + if (AdLen) + { + Poly1305Update(&Poly1305State, Ad, AdLen); + if (AdLen & 0xf) + Poly1305Update(&Poly1305State, Pad0, 0x10 - (AdLen & 0xf)); + } + + while (OffsetMdl >= MmGetMdlByteCount(Mdl)) + { + OffsetMdl -= MmGetMdlByteCount(Mdl); + Mdl = Mdl->Next; + } + for (ULONG Remaining = SrcLen; Remaining; Remaining -= LenMdl) + { + if (!Mdl) + return FALSE; + Len = LenMdl = min(MmGetMdlByteCount(Mdl) - OffsetMdl, Remaining); + SrcBuf = MmGetSystemAddressForMdlSafe(Mdl, NormalPagePriority | MdlMappingNoExecute | MdlMappingNoWrite); + if (!SrcBuf) + return FALSE; + SrcBuf += OffsetMdl; + + if (Leftover != 0) + { + ULONG l = min(Len, Leftover); + XorCpy(Dst, SrcBuf, ((UINT8 *)B.Stream) + (CHACHA20_BLOCK_SIZE - Leftover), l); + Leftover -= l; + SrcBuf += l; + Dst += l; + Len -= l; + } + + if (Len >= CHACHA20_BLOCK_SIZE) + { + ULONG l = ALIGN_DOWN_BY_T(ULONG, Len, CHACHA20_BLOCK_SIZE); + ChaCha20(&ChaCha20State, Dst, SrcBuf, l, Simd); + SrcBuf += l; + Dst += l; + Len -= l; + } + + if (Len) + { + ChaCha20Block(&ChaCha20State, B.Stream, Simd); + XorCpy(Dst, SrcBuf, (UINT8 *)B.Stream, Len); + Leftover = CHACHA20_BLOCK_SIZE - Len; + Dst += Len; + } + + _Analysis_assume_((RtlFillMemory(Dst - LenMdl, LenMdl, 'A'), TRUE)); + Poly1305Update(&Poly1305State, Dst - LenMdl, LenMdl); + + Mdl = Mdl->Next; + OffsetMdl = 0; + } + Poly1305Update(&Poly1305State, Pad0, (0x10 - SrcLen) & 0xf); + B.Lens[0] = CpuToLe64(AdLen); + B.Lens[1] = CpuToLe64(SrcLen); + Poly1305Update(&Poly1305State, (UINT8 *)B.Lens, sizeof(B.Lens)); + Poly1305Final(&Poly1305State, Dst); + + RtlSecureZeroMemory(&ChaCha20State, sizeof(ChaCha20State)); + RtlSecureZeroMemory(&B, sizeof(B)); + + return TRUE; +} + +_Use_decl_annotations_ +VOID +XChaCha20Poly1305Encrypt( + UINT8 *Dst, + CONST UINT8 *Src, + CONST SIZE_T SrcLen, + CONST UINT8 *Ad, + CONST SIZE_T AdLen, + CONST UINT8 Nonce[XCHACHA20POLY1305_NONCE_SIZE], + CONST UINT8 Key[CHACHA20POLY1305_KEY_SIZE]) +{ + UINT32 DerivedKey[CHACHA20_KEY_WORDS]; + + HChaCha20(DerivedKey, Nonce, Key); + CpuToLe32Array(DerivedKey, ARRAYSIZE(DerivedKey)); + ChaCha20Poly1305Encrypt(Dst, Src, SrcLen, Ad, AdLen, GetUnalignedLe64(Nonce + 16), (UINT8 *)DerivedKey); + RtlSecureZeroMemory(DerivedKey, CHACHA20POLY1305_KEY_SIZE); +} + +_Use_decl_annotations_ +BOOLEAN +XChaCha20Poly1305Decrypt( + UINT8 *Dst, + CONST UINT8 *Src, + CONST SIZE_T SrcLen, + CONST UINT8 *Ad, + CONST SIZE_T AdLen, + CONST UINT8 Nonce[XCHACHA20POLY1305_NONCE_SIZE], + CONST UINT8 Key[CHACHA20POLY1305_KEY_SIZE]) +{ + BOOLEAN Ret; + UINT32 DerivedKey[CHACHA20_KEY_WORDS]; + + HChaCha20(DerivedKey, Nonce, Key); + CpuToLe32Array(DerivedKey, ARRAYSIZE(DerivedKey)); + Ret = ChaCha20Poly1305Decrypt(Dst, Src, SrcLen, Ad, AdLen, GetUnalignedLe64(Nonce + 16), (UINT8 *)DerivedKey); + RtlSecureZeroMemory(DerivedKey, CHACHA20POLY1305_KEY_SIZE); + return Ret; +} + +static CONST UINT32 Blake2sIv[8] = { 0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, 0xA54FF53AUL, + 0x510E527FUL, 0x9B05688CUL, 0x1F83D9ABUL, 0x5BE0CD19UL }; + +static CONST UINT8 Blake2sSigma[10][16] = { + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 }, { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 }, + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 }, { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 }, + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 }, { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 }, + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 }, { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 }, +}; + +static inline VOID +Blake2sSetLastblock(_Out_ BLAKE2S_STATE *State) +{ + State->F[0] = (UINT32)-1; +} + +static inline VOID +Blake2sIncrementCounter(_Inout_ BLAKE2S_STATE *State, _In_ CONST UINT32 Inc) +{ + State->T[0] += Inc; + State->T[1] += (State->T[0] < Inc); +} + +static inline VOID +Blake2sInitParam(_Out_ BLAKE2S_STATE *State, _In_ CONST UINT32 Param) +{ + LONG i; + + RtlZeroMemory(State, sizeof(*State)); + for (i = 0; i < 8; ++i) + State->H[i] = Blake2sIv[i]; + State->H[0] ^= Param; +} + +_Use_decl_annotations_ +VOID +Blake2sInit(BLAKE2S_STATE *State, CONST SIZE_T OutLen) +{ + Blake2sInitParam(State, 0x01010000 | OutLen); + State->OutLen = OutLen; +} + +_Use_decl_annotations_ +VOID +Blake2sInitKey(BLAKE2S_STATE *State, CONST SIZE_T OutLen, CONST UINT8 *Key, CONST SIZE_T KeyLen) +{ + UINT8 Block[BLAKE2S_BLOCK_SIZE] = { 0 }; + + Blake2sInitParam(State, 0x01010000 | KeyLen << 8 | OutLen); + State->OutLen = OutLen; + RtlCopyMemory(Block, Key, KeyLen); + Blake2sUpdate(State, Block, BLAKE2S_BLOCK_SIZE); + RtlSecureZeroMemory(Block, BLAKE2S_BLOCK_SIZE); +} + +static inline VOID +Blake2sCompress( + _Inout_ BLAKE2S_STATE *State, + _In_reads_bytes_(BLAKE2S_BLOCK_SIZE *Nblocks) CONST UINT8 *Block, + _In_ SIZE_T Nblocks, + _In_ CONST UINT32 Inc) +{ + UINT32 M[16]; + UINT32 V[16]; + LONG i; + + while (Nblocks > 0) + { + Blake2sIncrementCounter(State, Inc); + RtlCopyMemory(M, Block, BLAKE2S_BLOCK_SIZE); + Le32ToCpuArray(M, ARRAYSIZE(M)); + RtlCopyMemory(V, State->H, 32); + V[8] = Blake2sIv[0]; + V[9] = Blake2sIv[1]; + V[10] = Blake2sIv[2]; + V[11] = Blake2sIv[3]; + V[12] = Blake2sIv[4] ^ State->T[0]; + V[13] = Blake2sIv[5] ^ State->T[1]; + V[14] = Blake2sIv[6] ^ State->F[0]; + V[15] = Blake2sIv[7] ^ State->F[1]; + +#define G(R, i, A, B, C, D) \ + do \ + { \ + A += B + M[Blake2sSigma[R][2 * i + 0]]; \ + D = Ror32(D ^ A, 16); \ + C += D; \ + B = Ror32(B ^ C, 12); \ + A += B + M[Blake2sSigma[R][2 * i + 1]]; \ + D = Ror32(D ^ A, 8); \ + C += D; \ + B = Ror32(B ^ C, 7); \ + } while (0) + +#define ROUND(R) \ + do \ + { \ + G(R, 0, V[0], V[4], V[8], V[12]); \ + G(R, 1, V[1], V[5], V[9], V[13]); \ + G(R, 2, V[2], V[6], V[10], V[14]); \ + G(R, 3, V[3], V[7], V[11], V[15]); \ + G(R, 4, V[0], V[5], V[10], V[15]); \ + G(R, 5, V[1], V[6], V[11], V[12]); \ + G(R, 6, V[2], V[7], V[8], V[13]); \ + G(R, 7, V[3], V[4], V[9], V[14]); \ + } while (0) + ROUND(0); + ROUND(1); + ROUND(2); + ROUND(3); + ROUND(4); + ROUND(5); + ROUND(6); + ROUND(7); + ROUND(8); + ROUND(9); + +#undef G +#undef ROUND + + for (i = 0; i < 8; ++i) + State->H[i] ^= V[i] ^ V[i + 8]; + + Block += BLAKE2S_BLOCK_SIZE; + --Nblocks; + } +} + +_Use_decl_annotations_ +VOID +Blake2sUpdate(BLAKE2S_STATE *State, CONST UINT8 *In, SIZE_T InLen) +{ + CONST SIZE_T Fill = BLAKE2S_BLOCK_SIZE - State->BufLen; + + if (!InLen) + return; + if (InLen > Fill) + { + RtlCopyMemory(State->Buf + State->BufLen, In, Fill); + Blake2sCompress(State, State->Buf, 1, BLAKE2S_BLOCK_SIZE); + State->BufLen = 0; + In += Fill; + InLen -= Fill; + } + if (InLen > BLAKE2S_BLOCK_SIZE) + { + CONST SIZE_T Nblocks = DIV_ROUND_UP(InLen, BLAKE2S_BLOCK_SIZE); + /* Hash one less (full) block than strictly possible */ + Blake2sCompress(State, In, Nblocks - 1, BLAKE2S_BLOCK_SIZE); + In += BLAKE2S_BLOCK_SIZE * (Nblocks - 1); + InLen -= BLAKE2S_BLOCK_SIZE * (Nblocks - 1); + } + RtlCopyMemory(State->Buf + State->BufLen, In, InLen); + State->BufLen += InLen; +} + +_Use_decl_annotations_ +VOID +Blake2sFinal(BLAKE2S_STATE *State, UINT8 *Out) +{ + Blake2sSetLastblock(State); + RtlZeroMemory(State->Buf + State->BufLen, BLAKE2S_BLOCK_SIZE - State->BufLen); /* Padding */ + Blake2sCompress(State, State->Buf, 1, State->BufLen); + CpuToLe32Array(State->H, ARRAYSIZE(State->H)); + RtlCopyMemory(Out, State->H, State->OutLen); + RtlSecureZeroMemory(State, sizeof(*State)); +} + +_Use_decl_annotations_ +VOID +Blake2s(UINT8 *Out, CONST UINT8 *In, CONST UINT8 *Key, CONST SIZE_T OutLen, CONST SIZE_T InLen, CONST SIZE_T KeyLen) +{ + BLAKE2S_STATE State; + + if (KeyLen) + Blake2sInitKey(&State, OutLen, Key, KeyLen); + else + Blake2sInit(&State, OutLen); + + Blake2sUpdate(&State, In, InLen); + Blake2sFinal(&State, Out); +} + +_Use_decl_annotations_ +VOID +Blake2s256Hmac(UINT8 *Out, CONST UINT8 *In, CONST UINT8 *Key, CONST SIZE_T InLen, CONST SIZE_T KeyLen) +{ + BLAKE2S_STATE State; + __declspec(align(4)) UINT8 XKey[BLAKE2S_BLOCK_SIZE] = { 0 }; + __declspec(align(4)) UINT8 IHash[BLAKE2S_HASH_SIZE]; + LONG i; + + if (KeyLen > BLAKE2S_BLOCK_SIZE) + { + Blake2sInit(&State, BLAKE2S_HASH_SIZE); + Blake2sUpdate(&State, Key, KeyLen); + Blake2sFinal(&State, XKey); + } + else + RtlCopyMemory(XKey, Key, KeyLen); + + for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i) + XKey[i] ^= 0x36; + + Blake2sInit(&State, BLAKE2S_HASH_SIZE); + Blake2sUpdate(&State, XKey, BLAKE2S_BLOCK_SIZE); + Blake2sUpdate(&State, In, InLen); + Blake2sFinal(&State, IHash); + + for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i) + XKey[i] ^= 0x5c ^ 0x36; + + Blake2sInit(&State, BLAKE2S_HASH_SIZE); + Blake2sUpdate(&State, XKey, BLAKE2S_BLOCK_SIZE); + Blake2sUpdate(&State, IHash, BLAKE2S_HASH_SIZE); + Blake2sFinal(&State, IHash); + + RtlCopyMemory(Out, IHash, BLAKE2S_HASH_SIZE); + RtlSecureZeroMemory(XKey, BLAKE2S_BLOCK_SIZE); + RtlSecureZeroMemory(IHash, BLAKE2S_HASH_SIZE); +} + +#define SIPROUND \ + do \ + { \ + V0 += V1; \ + V1 = Rol64(V1, 13); \ + V1 ^= V0; \ + V0 = Rol64(V0, 32); \ + V2 += V3; \ + V3 = Rol64(V3, 16); \ + V3 ^= V2; \ + V0 += V3; \ + V3 = Rol64(V3, 21); \ + V3 ^= V0; \ + V2 += V1; \ + V1 = Rol64(V1, 17); \ + V1 ^= V2; \ + V2 = Rol64(V2, 32); \ + } while (0) + +#define PREAMBLE(Len) \ + UINT64 V0 = 0x736f6d6570736575ULL; \ + UINT64 V1 = 0x646f72616e646f6dULL; \ + UINT64 V2 = 0x6c7967656e657261ULL; \ + UINT64 V3 = 0x7465646279746573ULL; \ + UINT64 B = ((UINT64)(Len)) << 56; \ + V3 ^= Key->Key[1]; \ + V2 ^= Key->Key[0]; \ + V1 ^= Key->Key[1]; \ + V0 ^= Key->Key[0]; + +#define POSTAMBLE \ + V3 ^= B; \ + SIPROUND; \ + SIPROUND; \ + V0 ^= B; \ + V2 ^= 0xff; \ + SIPROUND; \ + SIPROUND; \ + SIPROUND; \ + SIPROUND; \ + return (V0 ^ V1) ^ (V2 ^ V3); + +_Use_decl_annotations_ +UINT64 +Siphash(CONST VOID *Data, SIZE_T Len, CONST SIPHASH_KEY *Key) +{ + CONST UINT8 *End = (CONST UINT8 *)Data + Len - (Len % sizeof(UINT64)); + CONST UINT8 Left = Len & (sizeof(UINT64) - 1); + UINT64 M; + PREAMBLE(Len) + for (; Data != End; Data = (CONST UINT8 *)Data + sizeof(UINT64)) + { + M = Le64ToCpup((CONST UINT64_LE *)Data); + V3 ^= M; + SIPROUND; + SIPROUND; + V0 ^= M; + } + switch (Left) + { + case 7: + B |= ((UINT64)End[6]) << 48; + /* fallthrough */; + case 6: + B |= ((UINT64)End[5]) << 40; + /* fallthrough */; + case 5: + B |= ((UINT64)End[4]) << 32; + /* fallthrough */; + case 4: + B |= Le32ToCpup((CONST UINT32_LE *)Data); + break; + case 3: + B |= ((UINT64)End[2]) << 16; + /* fallthrough */; + case 2: + B |= Le16ToCpup((CONST UINT16_LE *)Data); + break; + case 1: + B |= End[0]; + } + POSTAMBLE +} + +_Use_decl_annotations_ +UINT64 +Siphash1u64(CONST UINT64 First, CONST SIPHASH_KEY *Key) +{ + PREAMBLE(8) + V3 ^= First; + SIPROUND; + SIPROUND; + V0 ^= First; + POSTAMBLE +} + +_Use_decl_annotations_ +UINT64 +Siphash2u64(CONST UINT64 First, CONST UINT64 Second, CONST SIPHASH_KEY *Key) +{ + PREAMBLE(16) + V3 ^= First; + SIPROUND; + SIPROUND; + V0 ^= First; + V3 ^= Second; + SIPROUND; + SIPROUND; + V0 ^= Second; + POSTAMBLE +} + +_Use_decl_annotations_ +UINT64 +Siphash3u64(CONST UINT64 First, CONST UINT64 Second, CONST UINT64 Third, CONST SIPHASH_KEY *Key) +{ + PREAMBLE(24) + V3 ^= First; + SIPROUND; + SIPROUND; + V0 ^= First; + V3 ^= Second; + SIPROUND; + SIPROUND; + V0 ^= Second; + V3 ^= Third; + SIPROUND; + SIPROUND; + V0 ^= Third; + POSTAMBLE +} + +_Use_decl_annotations_ +UINT64 +Siphash4u64(CONST UINT64 First, CONST UINT64 Second, CONST UINT64 Third, CONST UINT64 Forth, CONST SIPHASH_KEY *Key) +{ + PREAMBLE(32) + V3 ^= First; + SIPROUND; + SIPROUND; + V0 ^= First; + V3 ^= Second; + SIPROUND; + SIPROUND; + V0 ^= Second; + V3 ^= Third; + SIPROUND; + SIPROUND; + V0 ^= Third; + V3 ^= Forth; + SIPROUND; + SIPROUND; + V0 ^= Forth; + POSTAMBLE +} + +_Use_decl_annotations_ +UINT64 +Siphash1u32(CONST UINT32 First, CONST SIPHASH_KEY *Key) +{ + PREAMBLE(4) + B |= First; + POSTAMBLE +} + +_Use_decl_annotations_ +UINT64 +Siphash3u32(CONST UINT32 First, CONST UINT32 Second, CONST UINT32 Third, CONST SIPHASH_KEY *Key) +{ + UINT64 Combined = (UINT64)Second << 32 | First; + PREAMBLE(12) + V3 ^= Combined; + SIPROUND; + SIPROUND; + V0 ^= Combined; + B |= Third; + POSTAMBLE +} + +#if BITS_PER_POINTER == 64 +/* Note that on 64-bit, we make HalfSiphash1-3 actually be Siphash1-3, for + * performance reasons. On 32-bit, below, we actually implement HalfSiphash1-3. + */ + +# define HSIPROUND SIPROUND +# define HPREAMBLE(Len) PREAMBLE(Len) +# define HPOSTAMBLE \ + V3 ^= B; \ + HSIPROUND; \ + V0 ^= B; \ + V2 ^= 0xff; \ + HSIPROUND; \ + HSIPROUND; \ + HSIPROUND; \ + return (UINT32)((V0 ^ V1) ^ (V2 ^ V3)); + +_Use_decl_annotations_ +UINT32 +Hsiphash(CONST VOID *Data, SIZE_T Len, CONST HSIPHASH_KEY *Key) +{ + CONST UINT8 *End = (CONST UINT8 *)Data + Len - (Len % sizeof(UINT64)); + CONST UINT8 Left = Len & (sizeof(UINT64) - 1); + UINT64 M; + HPREAMBLE(Len) + for (; Data != End; Data = (CONST UINT8 *)Data + sizeof(UINT64)) + { + M = Le64ToCpup((CONST UINT64_LE *)Data); + V3 ^= M; + HSIPROUND; + V0 ^= M; + } + switch (Left) + { + case 7: + B |= ((UINT64)End[6]) << 48; + /* fallthrough */; + case 6: + B |= ((UINT64)End[5]) << 40; + /* fallthrough */; + case 5: + B |= ((UINT64)End[4]) << 32; + /* fallthrough */; + case 4: + B |= Le32ToCpup((CONST UINT32_LE *)Data); + break; + case 3: + B |= ((UINT64)End[2]) << 16; + /* fallthrough */; + case 2: + B |= Le16ToCpup((CONST UINT16_LE *)Data); + break; + case 1: + B |= End[0]; + } + HPOSTAMBLE +} + +_Use_decl_annotations_ +UINT32 +Hsiphash1u32(CONST UINT32 First, CONST HSIPHASH_KEY *Key) +{ + HPREAMBLE(4) + B |= First; + HPOSTAMBLE +} + +_Use_decl_annotations_ +UINT32 +Hsiphash2u32(CONST UINT32 First, CONST UINT32 Second, CONST HSIPHASH_KEY *Key) +{ + UINT64 Combined = (UINT64)Second << 32 | First; + HPREAMBLE(8) + V3 ^= Combined; + HSIPROUND; + V0 ^= Combined; + HPOSTAMBLE +} + +_Use_decl_annotations_ +UINT32 +Hsiphash3u32(CONST UINT32 First, CONST UINT32 Second, CONST UINT32 Third, CONST HSIPHASH_KEY *Key) +{ + UINT64 Combined = (UINT64)Second << 32 | First; + HPREAMBLE(12) + V3 ^= Combined; + HSIPROUND; + V0 ^= Combined; + B |= Third; + HPOSTAMBLE +} + +_Use_decl_annotations_ +UINT32 +Hsiphash4u32(CONST UINT32 First, CONST UINT32 Second, CONST UINT32 Third, CONST UINT32 Forth, CONST HSIPHASH_KEY *Key) +{ + UINT64 Combined = (UINT64)Second << 32 | First; + HPREAMBLE(16) + V3 ^= Combined; + HSIPROUND; + V0 ^= Combined; + Combined = (UINT64)Forth << 32 | Third; + V3 ^= Combined; + HSIPROUND; + V0 ^= Combined; + HPOSTAMBLE +} +#else +# define HSIPROUND \ + do \ + { \ + V0 += V1; \ + V1 = Rol32(V1, 5); \ + V1 ^= V0; \ + V0 = Rol32(V0, 16); \ + V2 += V3; \ + V3 = Rol32(V3, 8); \ + V3 ^= V2; \ + V0 += V3; \ + V3 = Rol32(V3, 7); \ + V3 ^= V0; \ + V2 += V1; \ + V1 = Rol32(V1, 13); \ + V1 ^= V2; \ + V2 = Rol32(V2, 16); \ + } while (0) + +# define HPREAMBLE(Len) \ + UINT32 V0 = 0; \ + UINT32 V1 = 0; \ + UINT32 V2 = 0x6c796765U; \ + UINT32 V3 = 0x74656462U; \ + UINT32 B = ((UINT32)(Len)) << 24; \ + V3 ^= Key->Key[1]; \ + V2 ^= Key->Key[0]; \ + V1 ^= Key->Key[1]; \ + V0 ^= Key->Key[0]; + +# define HPOSTAMBLE \ + V3 ^= B; \ + HSIPROUND; \ + V0 ^= B; \ + V2 ^= 0xff; \ + HSIPROUND; \ + HSIPROUND; \ + HSIPROUND; \ + return V1 ^ V3; + +_Use_decl_annotations_ +UINT32 +Hsiphash(CONST VOID *Data, SIZE_T Len, CONST HSIPHASH_KEY *Key) +{ + CONST UINT8 *End = (CONST UINT8 *)Data + Len - (Len % sizeof(UINT32)); + CONST UINT8 Left = Len & (sizeof(UINT32) - 1); + UINT32 M; + HPREAMBLE(Len) + for (; Data != End; Data = (CONST UINT8 *)Data + sizeof(UINT32)) + { + M = Le32ToCpup((CONST UINT32_LE *)Data); + V3 ^= M; + HSIPROUND; + V0 ^= M; + } + switch (Left) + { + case 3: + B |= ((UINT32)End[2]) << 16; + /* fallthrough */; + case 2: + B |= Le16ToCpup((CONST UINT16_LE *)Data); + break; + case 1: + B |= End[0]; + } + HPOSTAMBLE +} + +_Use_decl_annotations_ +UINT32 +Hsiphash1u32(CONST UINT32 First, CONST HSIPHASH_KEY *Key) +{ + HPREAMBLE(4) + V3 ^= First; + HSIPROUND; + V0 ^= First; + HPOSTAMBLE +} + +_Use_decl_annotations_ +UINT32 +Hsiphash2u32(CONST UINT32 First, CONST UINT32 Second, CONST HSIPHASH_KEY *Key) +{ + HPREAMBLE(8) + V3 ^= First; + HSIPROUND; + V0 ^= First; + V3 ^= Second; + HSIPROUND; + V0 ^= Second; + HPOSTAMBLE +} + +_Use_decl_annotations_ +UINT32 +Hsiphash3u32(CONST UINT32 First, CONST UINT32 Second, CONST UINT32 Third, CONST HSIPHASH_KEY *Key) +{ + HPREAMBLE(12) + V3 ^= First; + HSIPROUND; + V0 ^= First; + V3 ^= Second; + HSIPROUND; + V0 ^= Second; + V3 ^= Third; + HSIPROUND; + V0 ^= Third; + HPOSTAMBLE +} + +_Use_decl_annotations_ +UINT32 +Hsiphash4u32(CONST UINT32 First, CONST UINT32 Second, CONST UINT32 Third, CONST UINT32 Forth, CONST HSIPHASH_KEY *Key) +{ + HPREAMBLE(16) + V3 ^= First; + HSIPROUND; + V0 ^= First; + V3 ^= Second; + HSIPROUND; + V0 ^= Second; + V3 ^= Third; + HSIPROUND; + V0 ^= Third; + V3 ^= Forth; + HSIPROUND; + V0 ^= Forth; + HPOSTAMBLE +} +#endif + +/* Below here is fiat's implementation of x25519. + * + * Copyright (C) 2015-2016 The fiat-crypto Authors. + * Copyright (C) 2018-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + * + * This is a machine-generated formally verified implementation of Curve25519 + * ECDH from: <https://github.com/mit-plv/fiat-crypto>. Though originally + * machine generated, it has been tweaked to be suitable for use in the kernel. + * It is optimized for 32-bit machines and machines that cannot work efficiently + * with 128-bit integer types. + */ + +/* Fe means field element. Here the field is \Z/(2^255-19). An element t, + * entries t[0]...t[9], represents the integer t[0]+2^26 t[1]+2^51 t[2]+2^77 + * t[3]+2^102 t[4]+...+2^230 t[9]. + * Fe limbs are bounded by 1.125*2^26,1.125*2^25,1.125*2^26,1.125*2^25,etc. + * Multiplication and carrying produce Fe from FeLoose. + */ +typedef struct Fe +{ + UINT32 V[10]; +} Fe; + +/* FeLoose limbs are bounded by 3.375*2^26,3.375*2^25,3.375*2^26,3.375*2^25,etc + * Addition and subtraction produce FeLoose from (Fe, Fe). + */ +typedef struct FeLoose +{ + UINT32 V[10]; +} FeLoose; + +static inline VOID +FeFrombytesImpl(_Out_writes_all_(10) UINT32 H[10], _In_reads_bytes_(32) CONST UINT8 *S) +{ + /* Ignores top bit of s. */ + UINT32 A0 = GetUnalignedLe32(S); + UINT32 A1 = GetUnalignedLe32(S + 4); + UINT32 A2 = GetUnalignedLe32(S + 8); + UINT32 A3 = GetUnalignedLe32(S + 12); + UINT32 A4 = GetUnalignedLe32(S + 16); + UINT32 A5 = GetUnalignedLe32(S + 20); + UINT32 A6 = GetUnalignedLe32(S + 24); + UINT32 A7 = GetUnalignedLe32(S + 28); + H[0] = A0 & ((1 << 26) - 1); /* 26 used, 32-26 left. 26 */ + H[1] = (A0 >> 26) | ((A1 & ((1 << 19) - 1)) << 6); /* (32-26) + 19 = 6+19 = 25 */ + H[2] = (A1 >> 19) | ((A2 & ((1 << 13) - 1)) << 13); /* (32-19) + 13 = 13+13 = 26 */ + H[3] = (A2 >> 13) | ((A3 & ((1 << 6) - 1)) << 19); /* (32-13) + 6 = 19+ 6 = 25 */ + H[4] = (A3 >> 6); /* (32- 6) = 26 */ + H[5] = A4 & ((1 << 25) - 1); /* 25 */ + H[6] = (A4 >> 25) | ((A5 & ((1 << 19) - 1)) << 7); /* (32-25) + 19 = 7+19 = 26 */ + H[7] = (A5 >> 19) | ((A6 & ((1 << 12) - 1)) << 13); /* (32-19) + 12 = 13+12 = 25 */ + H[8] = (A6 >> 12) | ((A7 & ((1 << 6) - 1)) << 20); /* (32-12) + 6 = 20+ 6 = 26 */ + H[9] = (A7 >> 6) & ((1 << 25) - 1); /* 25 */ +} + +static inline VOID +FeFrombytes(_Out_ Fe *H, _In_reads_bytes_(32) CONST UINT8 *S) +{ + FeFrombytesImpl(H->V, S); +} + +static inline UINT8 /*bool*/ +AddcarryxU25(_In_ CONST UINT8 /*bool*/ C, _In_ CONST UINT32 A, _In_ CONST UINT32 B, _Out_ UINT32 *Low) +{ + /* This function extracts 25 bits of result and 1 bit of carry + * (26 total), so a 32-bit intermediate is sufficient. + */ + UINT32 X = A + B + C; + *Low = X & ((1 << 25) - 1); + return (X >> 25) & 1; +} + +static inline UINT8 /*bool*/ +AddcarryxU26(_In_ CONST UINT8 /*bool*/ C, _In_ CONST UINT32 A, _In_ CONST UINT32 B, _Out_ UINT32 *Low) +{ + /* This function extracts 26 bits of result and 1 bit of carry + * (27 total), so a 32-bit intermediate is sufficient. + */ + UINT32 X = A + B + C; + *Low = X & ((1 << 26) - 1); + return (X >> 26) & 1; +} + +static inline UINT8 /*bool*/ +SubborrowU25(_In_ CONST UINT8 /*bool*/ C, _In_ CONST UINT32 A, _In_ CONST UINT32 B, _Out_ UINT32 *Low) +{ + /* This function extracts 25 bits of result and 1 bit of borrow + * (26 total), so a 32-bit intermediate is sufficient. + */ + UINT32 X = A - B - C; + *Low = X & ((1 << 25) - 1); + return X >> 31; +} + +static inline UINT8 /*bool*/ +SubborrowU26(_In_ CONST UINT8 /*bool*/ C, _In_ CONST UINT32 A, _In_ CONST UINT32 B, _Out_ UINT32 *Low) +{ + /* This function extracts 26 bits of result and 1 bit of borrow + *(27 total), so a 32-bit intermediate is sufficient. + */ + UINT32 X = A - B - C; + *Low = X & ((1 << 26) - 1); + return X >> 31; +} + +static inline UINT32 +Cmovznz32(_In_ UINT32 T, _In_ CONST UINT32 Z, _In_ CONST UINT32 Nz) +{ + T = -!!T; /* all set if nonzero, 0 if 0 */ + return (T & Nz) | ((~T) & Z); +} + +static inline VOID +FeFreeze(_Out_writes_all_(10) UINT32 Out[10], _In_reads_(10) CONST UINT32 In1[10]) +{ + CONST UINT32 X17 = In1[9]; + CONST UINT32 X18 = In1[8]; + CONST UINT32 X16 = In1[7]; + CONST UINT32 X14 = In1[6]; + CONST UINT32 X12 = In1[5]; + CONST UINT32 X10 = In1[4]; + CONST UINT32 X8 = In1[3]; + CONST UINT32 X6 = In1[2]; + CONST UINT32 X4 = In1[1]; + CONST UINT32 X2 = In1[0]; + UINT32 X20; + UINT8 /*bool*/ X21 = SubborrowU26(0x0, X2, 0x3ffffed, &X20); + UINT32 X23; + UINT8 /*bool*/ X24 = SubborrowU25(X21, X4, 0x1ffffff, &X23); + UINT32 X26; + UINT8 /*bool*/ X27 = SubborrowU26(X24, X6, 0x3ffffff, &X26); + UINT32 X29; + UINT8 /*bool*/ X30 = SubborrowU25(X27, X8, 0x1ffffff, &X29); + UINT32 X32; + UINT8 /*bool*/ X33 = SubborrowU26(X30, X10, 0x3ffffff, &X32); + UINT32 X35; + UINT8 /*bool*/ X36 = SubborrowU25(X33, X12, 0x1ffffff, &X35); + UINT32 X38; + UINT8 /*bool*/ X39 = SubborrowU26(X36, X14, 0x3ffffff, &X38); + UINT32 X41; + UINT8 /*bool*/ X42 = SubborrowU25(X39, X16, 0x1ffffff, &X41); + UINT32 X44; + UINT8 /*bool*/ X45 = SubborrowU26(X42, X18, 0x3ffffff, &X44); + UINT32 X47; + UINT8 /*bool*/ X48 = SubborrowU25(X45, X17, 0x1ffffff, &X47); + UINT32 X49 = Cmovznz32(X48, 0x0, 0xffffffff); + UINT32 X50 = (X49 & 0x3ffffed); + UINT32 X52; + UINT8 /*bool*/ X53 = AddcarryxU26(0x0, X20, X50, &X52); + UINT32 X54 = (X49 & 0x1ffffff); + UINT32 X56; + UINT8 /*bool*/ X57 = AddcarryxU25(X53, X23, X54, &X56); + UINT32 X58 = (X49 & 0x3ffffff); + UINT32 X60; + UINT8 /*bool*/ X61 = AddcarryxU26(X57, X26, X58, &X60); + UINT32 X62 = (X49 & 0x1ffffff); + UINT32 X64; + UINT8 /*bool*/ X65 = AddcarryxU25(X61, X29, X62, &X64); + UINT32 X66 = (X49 & 0x3ffffff); + UINT32 X68; + UINT8 /*bool*/ X69 = AddcarryxU26(X65, X32, X66, &X68); + UINT32 X70 = (X49 & 0x1ffffff); + UINT32 X72; + UINT8 /*bool*/ X73 = AddcarryxU25(X69, X35, X70, &X72); + UINT32 X74 = (X49 & 0x3ffffff); + UINT32 X76; + UINT8 /*bool*/ X77 = AddcarryxU26(X73, X38, X74, &X76); + UINT32 X78 = (X49 & 0x1ffffff); + UINT32 X80; + UINT8 /*bool*/ X81 = AddcarryxU25(X77, X41, X78, &X80); + UINT32 X82 = (X49 & 0x3ffffff); + UINT32 X84; + UINT8 /*bool*/ X85 = AddcarryxU26(X81, X44, X82, &X84); + UINT32 X86 = (X49 & 0x1ffffff); + UINT32 X88; + AddcarryxU25(X85, X47, X86, &X88); + Out[0] = X52; + Out[1] = X56; + Out[2] = X60; + Out[3] = X64; + Out[4] = X68; + Out[5] = X72; + Out[6] = X76; + Out[7] = X80; + Out[8] = X84; + Out[9] = X88; +} + +static inline VOID +FeTobytes(_Out_writes_bytes_all_(32) UINT8 S[32], _In_ CONST Fe *F) +{ + UINT32 H[10]; + FeFreeze(H, F->V); + S[0] = H[0] >> 0; + S[1] = H[0] >> 8; + S[2] = H[0] >> 16; + S[3] = (H[0] >> 24) | (H[1] << 2); + S[4] = H[1] >> 6; + S[5] = H[1] >> 14; + S[6] = (H[1] >> 22) | (H[2] << 3); + S[7] = H[2] >> 5; + S[8] = H[2] >> 13; + S[9] = (H[2] >> 21) | (H[3] << 5); + S[10] = H[3] >> 3; + S[11] = H[3] >> 11; + S[12] = (H[3] >> 19) | (H[4] << 6); + S[13] = H[4] >> 2; + S[14] = H[4] >> 10; + S[15] = H[4] >> 18; + S[16] = H[5] >> 0; + S[17] = H[5] >> 8; + S[18] = H[5] >> 16; + S[19] = (H[5] >> 24) | (H[6] << 1); + S[20] = H[6] >> 7; + S[21] = H[6] >> 15; + S[22] = (H[6] >> 23) | (H[7] << 3); + S[23] = H[7] >> 5; + S[24] = H[7] >> 13; + S[25] = (H[7] >> 21) | (H[8] << 4); + S[26] = H[8] >> 4; + S[27] = H[8] >> 12; + S[28] = (H[8] >> 20) | (H[9] << 6); + S[29] = H[9] >> 2; + S[30] = H[9] >> 10; + S[31] = H[9] >> 18; +} + +/* h = f */ +static inline VOID +FeCopy(_Out_ Fe *H, _In_ CONST Fe *F) +{ + RtlMoveMemory(H, F, sizeof(UINT32) * 10); +} + +static inline VOID +FeCopyLt(_Out_ FeLoose *H, _In_ CONST Fe *F) +{ + RtlMoveMemory(H, F, sizeof(UINT32) * 10); +} + +/* h = 0 */ +static inline VOID +Fe0(_Out_ Fe *H) +{ + RtlZeroMemory(H, sizeof(UINT32) * 10); +} + +/* h = 1 */ +static inline VOID +Fe1(_Out_ Fe *H) +{ + RtlZeroMemory(H, sizeof(UINT32) * 10); + H->V[0] = 1; +} + +static VOID +FeAddImpl(_Out_writes_all_(10) UINT32 Out[10], _In_reads_(10) CONST UINT32 In1[10], _In_reads_(10) CONST UINT32 In2[10]) +{ + CONST UINT32 X20 = In1[9]; + CONST UINT32 X21 = In1[8]; + CONST UINT32 X19 = In1[7]; + CONST UINT32 X17 = In1[6]; + CONST UINT32 X15 = In1[5]; + CONST UINT32 X13 = In1[4]; + CONST UINT32 X11 = In1[3]; + CONST UINT32 X9 = In1[2]; + CONST UINT32 X7 = In1[1]; + CONST UINT32 X5 = In1[0]; + CONST UINT32 X38 = In2[9]; + CONST UINT32 X39 = In2[8]; + CONST UINT32 X37 = In2[7]; + CONST UINT32 X35 = In2[6]; + CONST UINT32 X33 = In2[5]; + CONST UINT32 X31 = In2[4]; + CONST UINT32 X29 = In2[3]; + CONST UINT32 X27 = In2[2]; + CONST UINT32 X25 = In2[1]; + CONST UINT32 X23 = In2[0]; + Out[0] = (X5 + X23); + Out[1] = (X7 + X25); + Out[2] = (X9 + X27); + Out[3] = (X11 + X29); + Out[4] = (X13 + X31); + Out[5] = (X15 + X33); + Out[6] = (X17 + X35); + Out[7] = (X19 + X37); + Out[8] = (X21 + X39); + Out[9] = (X20 + X38); +} + +/* h = f + g + * Can overlap h with f or g. + */ +static inline VOID +FeAdd(_Out_ FeLoose *H, _In_ CONST Fe *F, _In_ CONST Fe *G) +{ + FeAddImpl(H->V, F->V, G->V); +} + +static VOID +FeSubImpl(_Out_writes_all_(10) UINT32 Out[10], _In_reads_(10) CONST UINT32 In1[10], _In_reads_(10) CONST UINT32 In2[10]) +{ + CONST UINT32 X20 = In1[9]; + CONST UINT32 X21 = In1[8]; + CONST UINT32 X19 = In1[7]; + CONST UINT32 X17 = In1[6]; + CONST UINT32 X15 = In1[5]; + CONST UINT32 X13 = In1[4]; + CONST UINT32 X11 = In1[3]; + CONST UINT32 X9 = In1[2]; + CONST UINT32 X7 = In1[1]; + CONST UINT32 X5 = In1[0]; + CONST UINT32 X38 = In2[9]; + CONST UINT32 X39 = In2[8]; + CONST UINT32 X37 = In2[7]; + CONST UINT32 X35 = In2[6]; + CONST UINT32 X33 = In2[5]; + CONST UINT32 X31 = In2[4]; + CONST UINT32 X29 = In2[3]; + CONST UINT32 X27 = In2[2]; + CONST UINT32 X25 = In2[1]; + CONST UINT32 X23 = In2[0]; + Out[0] = ((0x7ffffda + X5) - X23); + Out[1] = ((0x3fffffe + X7) - X25); + Out[2] = ((0x7fffffe + X9) - X27); + Out[3] = ((0x3fffffe + X11) - X29); + Out[4] = ((0x7fffffe + X13) - X31); + Out[5] = ((0x3fffffe + X15) - X33); + Out[6] = ((0x7fffffe + X17) - X35); + Out[7] = ((0x3fffffe + X19) - X37); + Out[8] = ((0x7fffffe + X21) - X39); + Out[9] = ((0x3fffffe + X20) - X38); +} + +/* h = f - g + * Can overlap h with f or g. + */ +static inline VOID +FeSub(_Out_ FeLoose *H, _In_ CONST Fe *F, _In_ CONST Fe *G) +{ + FeSubImpl(H->V, F->V, G->V); +} + +static VOID +FeMulImpl(_Out_writes_all_(10) UINT32 Out[10], _In_reads_(10) CONST UINT32 In1[10], _In_reads_(10) CONST UINT32 In2[10]) +{ + CONST UINT32 X20 = In1[9]; + CONST UINT32 X21 = In1[8]; + CONST UINT32 X19 = In1[7]; + CONST UINT32 X17 = In1[6]; + CONST UINT32 X15 = In1[5]; + CONST UINT32 X13 = In1[4]; + CONST UINT32 X11 = In1[3]; + CONST UINT32 X9 = In1[2]; + CONST UINT32 X7 = In1[1]; + CONST UINT32 X5 = In1[0]; + CONST UINT32 X38 = In2[9]; + CONST UINT32 X39 = In2[8]; + CONST UINT32 X37 = In2[7]; + CONST UINT32 X35 = In2[6]; + CONST UINT32 X33 = In2[5]; + CONST UINT32 X31 = In2[4]; + CONST UINT32 X29 = In2[3]; + CONST UINT32 X27 = In2[2]; + CONST UINT32 X25 = In2[1]; + CONST UINT32 X23 = In2[0]; + UINT64 X40 = ((UINT64)X23 * X5); + UINT64 X41 = (((UINT64)X23 * X7) + ((UINT64)X25 * X5)); + UINT64 X42 = ((((UINT64)(0x2 * X25) * X7) + ((UINT64)X23 * X9)) + ((UINT64)X27 * X5)); + UINT64 X43 = (((((UINT64)X25 * X9) + ((UINT64)X27 * X7)) + ((UINT64)X23 * X11)) + ((UINT64)X29 * X5)); + UINT64 X44 = + (((((UINT64)X27 * X9) + (0x2 * (((UINT64)X25 * X11) + ((UINT64)X29 * X7)))) + ((UINT64)X23 * X13)) + + ((UINT64)X31 * X5)); + UINT64 X45 = + (((((((UINT64)X27 * X11) + ((UINT64)X29 * X9)) + ((UINT64)X25 * X13)) + ((UINT64)X31 * X7)) + + ((UINT64)X23 * X15)) + + ((UINT64)X33 * X5)); + UINT64 X46 = + (((((0x2 * ((((UINT64)X29 * X11) + ((UINT64)X25 * X15)) + ((UINT64)X33 * X7))) + ((UINT64)X27 * X13)) + + ((UINT64)X31 * X9)) + + ((UINT64)X23 * X17)) + + ((UINT64)X35 * X5)); + UINT64 X47 = + (((((((((UINT64)X29 * X13) + ((UINT64)X31 * X11)) + ((UINT64)X27 * X15)) + ((UINT64)X33 * X9)) + + ((UINT64)X25 * X17)) + + ((UINT64)X35 * X7)) + + ((UINT64)X23 * X19)) + + ((UINT64)X37 * X5)); + UINT64 X48 = + (((((((UINT64)X31 * X13) + + (0x2 * (((((UINT64)X29 * X15) + ((UINT64)X33 * X11)) + ((UINT64)X25 * X19)) + ((UINT64)X37 * X7)))) + + ((UINT64)X27 * X17)) + + ((UINT64)X35 * X9)) + + ((UINT64)X23 * X21)) + + ((UINT64)X39 * X5)); + UINT64 X49 = + (((((((((((UINT64)X31 * X15) + ((UINT64)X33 * X13)) + ((UINT64)X29 * X17)) + ((UINT64)X35 * X11)) + + ((UINT64)X27 * X19)) + + ((UINT64)X37 * X9)) + + ((UINT64)X25 * X21)) + + ((UINT64)X39 * X7)) + + ((UINT64)X23 * X20)) + + ((UINT64)X38 * X5)); + UINT64 X50 = + (((((0x2 * ((((((UINT64)X33 * X15) + ((UINT64)X29 * X19)) + ((UINT64)X37 * X11)) + ((UINT64)X25 * X20)) + + ((UINT64)X38 * X7))) + + ((UINT64)X31 * X17)) + + ((UINT64)X35 * X13)) + + ((UINT64)X27 * X21)) + + ((UINT64)X39 * X9)); + UINT64 X51 = + (((((((((UINT64)X33 * X17) + ((UINT64)X35 * X15)) + ((UINT64)X31 * X19)) + ((UINT64)X37 * X13)) + + ((UINT64)X29 * X21)) + + ((UINT64)X39 * X11)) + + ((UINT64)X27 * X20)) + + ((UINT64)X38 * X9)); + UINT64 X52 = + (((((UINT64)X35 * X17) + + (0x2 * (((((UINT64)X33 * X19) + ((UINT64)X37 * X15)) + ((UINT64)X29 * X20)) + ((UINT64)X38 * X11)))) + + ((UINT64)X31 * X21)) + + ((UINT64)X39 * X13)); + UINT64 X53 = + (((((((UINT64)X35 * X19) + ((UINT64)X37 * X17)) + ((UINT64)X33 * X21)) + ((UINT64)X39 * X15)) + + ((UINT64)X31 * X20)) + + ((UINT64)X38 * X13)); + UINT64 X54 = + (((0x2 * ((((UINT64)X37 * X19) + ((UINT64)X33 * X20)) + ((UINT64)X38 * X15))) + ((UINT64)X35 * X21)) + + ((UINT64)X39 * X17)); + UINT64 X55 = (((((UINT64)X37 * X21) + ((UINT64)X39 * X19)) + ((UINT64)X35 * X20)) + ((UINT64)X38 * X17)); + UINT64 X56 = (((UINT64)X39 * X21) + (0x2 * (((UINT64)X37 * X20) + ((UINT64)X38 * X19)))); + UINT64 X57 = (((UINT64)X39 * X20) + ((UINT64)X38 * X21)); + UINT64 X58 = ((UINT64)(0x2 * X38) * X20); + UINT64 X59 = (X48 + (X58 << 0x4)); + UINT64 X60 = (X59 + (X58 << 0x1)); + UINT64 X61 = (X60 + X58); + UINT64 X62 = (X47 + (X57 << 0x4)); + UINT64 X63 = (X62 + (X57 << 0x1)); + UINT64 X64 = (X63 + X57); + UINT64 X65 = (X46 + (X56 << 0x4)); + UINT64 X66 = (X65 + (X56 << 0x1)); + UINT64 X67 = (X66 + X56); + UINT64 X68 = (X45 + (X55 << 0x4)); + UINT64 X69 = (X68 + (X55 << 0x1)); + UINT64 X70 = (X69 + X55); + UINT64 X71 = (X44 + (X54 << 0x4)); + UINT64 X72 = (X71 + (X54 << 0x1)); + UINT64 X73 = (X72 + X54); + UINT64 X74 = (X43 + (X53 << 0x4)); + UINT64 X75 = (X74 + (X53 << 0x1)); + UINT64 X76 = (X75 + X53); + UINT64 X77 = (X42 + (X52 << 0x4)); + UINT64 X78 = (X77 + (X52 << 0x1)); + UINT64 X79 = (X78 + X52); + UINT64 X80 = (X41 + (X51 << 0x4)); + UINT64 X81 = (X80 + (X51 << 0x1)); + UINT64 X82 = (X81 + X51); + UINT64 X83 = (X40 + (X50 << 0x4)); + UINT64 X84 = (X83 + (X50 << 0x1)); + UINT64 X85 = (X84 + X50); + UINT64 X86 = (X85 >> 0x1a); + UINT32 X87 = ((UINT32)X85 & 0x3ffffff); + UINT64 X88 = (X86 + X82); + UINT64 X89 = (X88 >> 0x19); + UINT32 X90 = ((UINT32)X88 & 0x1ffffff); + UINT64 X91 = (X89 + X79); + UINT64 X92 = (X91 >> 0x1a); + UINT32 X93 = ((UINT32)X91 & 0x3ffffff); + UINT64 X94 = (X92 + X76); + UINT64 X95 = (X94 >> 0x19); + UINT32 X96 = ((UINT32)X94 & 0x1ffffff); + UINT64 X97 = (X95 + X73); + UINT64 X98 = (X97 >> 0x1a); + UINT32 X99 = ((UINT32)X97 & 0x3ffffff); + UINT64 X100 = (X98 + X70); + UINT64 X101 = (X100 >> 0x19); + UINT32 X102 = ((UINT32)X100 & 0x1ffffff); + UINT64 X103 = (X101 + X67); + UINT64 X104 = (X103 >> 0x1a); + UINT32 X105 = ((UINT32)X103 & 0x3ffffff); + UINT64 X106 = (X104 + X64); + UINT64 X107 = (X106 >> 0x19); + UINT32 X108 = ((UINT32)X106 & 0x1ffffff); + UINT64 X109 = (X107 + X61); + UINT64 X110 = (X109 >> 0x1a); + UINT32 X111 = ((UINT32)X109 & 0x3ffffff); + UINT64 X112 = (X110 + X49); + UINT64 X113 = (X112 >> 0x19); + UINT32 X114 = ((UINT32)X112 & 0x1ffffff); + UINT64 X115 = (X87 + (0x13 * X113)); + UINT32 X116 = (UINT32)(X115 >> 0x1a); + UINT32 X117 = ((UINT32)X115 & 0x3ffffff); + UINT32 X118 = (X116 + X90); + UINT32 X119 = (X118 >> 0x19); + UINT32 X120 = (X118 & 0x1ffffff); + Out[0] = X117; + Out[1] = X120; + Out[2] = (X119 + X93); + Out[3] = X96; + Out[4] = X99; + Out[5] = X102; + Out[6] = X105; + Out[7] = X108; + Out[8] = X111; + Out[9] = X114; +} + +static inline VOID +FeMulTtt(_Out_ Fe *H, _In_ CONST Fe *F, _In_ CONST Fe *G) +{ + FeMulImpl(H->V, F->V, G->V); +} + +static inline VOID +FeMulTlt(_Out_ Fe *H, _In_ CONST FeLoose *F, _In_ CONST Fe *G) +{ + FeMulImpl(H->V, F->V, G->V); +} + +static inline VOID +FeMulTll(_Out_ Fe *H, _In_ CONST FeLoose *F, _In_ CONST FeLoose *G) +{ + FeMulImpl(H->V, F->V, G->V); +} + +static VOID +FeSqrImpl(_Out_writes_all_(10) UINT32 Out[10], _In_reads_(10) CONST UINT32 In1[10]) +{ + CONST UINT32 X17 = In1[9]; + CONST UINT32 X18 = In1[8]; + CONST UINT32 X16 = In1[7]; + CONST UINT32 X14 = In1[6]; + CONST UINT32 X12 = In1[5]; + CONST UINT32 X10 = In1[4]; + CONST UINT32 X8 = In1[3]; + CONST UINT32 X6 = In1[2]; + CONST UINT32 X4 = In1[1]; + CONST UINT32 X2 = In1[0]; + UINT64 X19 = ((UINT64)X2 * X2); + UINT64 X20 = ((UINT64)(0x2 * X2) * X4); + UINT64 X21 = (0x2 * (((UINT64)X4 * X4) + ((UINT64)X2 * X6))); + UINT64 X22 = (0x2 * (((UINT64)X4 * X6) + ((UINT64)X2 * X8))); + UINT64 X23 = ((((UINT64)X6 * X6) + ((UINT64)(0x4 * X4) * X8)) + ((UINT64)(0x2 * X2) * X10)); + UINT64 X24 = (0x2 * ((((UINT64)X6 * X8) + ((UINT64)X4 * X10)) + ((UINT64)X2 * X12))); + UINT64 X25 = (0x2 * (((((UINT64)X8 * X8) + ((UINT64)X6 * X10)) + ((UINT64)X2 * X14)) + ((UINT64)(0x2 * X4) * X12))); + UINT64 X26 = (0x2 * (((((UINT64)X8 * X10) + ((UINT64)X6 * X12)) + ((UINT64)X4 * X14)) + ((UINT64)X2 * X16))); + UINT64 X27 = + (((UINT64)X10 * X10) + + (0x2 * ((((UINT64)X6 * X14) + ((UINT64)X2 * X18)) + (0x2 * (((UINT64)X4 * X16) + ((UINT64)X8 * X12)))))); + UINT64 X28 = + (0x2 * ((((((UINT64)X10 * X12) + ((UINT64)X8 * X14)) + ((UINT64)X6 * X16)) + ((UINT64)X4 * X18)) + + ((UINT64)X2 * X17))); + UINT64 X29 = + (0x2 * (((((UINT64)X12 * X12) + ((UINT64)X10 * X14)) + ((UINT64)X6 * X18)) + + (0x2 * (((UINT64)X8 * X16) + ((UINT64)X4 * X17))))); + UINT64 X30 = (0x2 * (((((UINT64)X12 * X14) + ((UINT64)X10 * X16)) + ((UINT64)X8 * X18)) + ((UINT64)X6 * X17))); + UINT64 X31 = + (((UINT64)X14 * X14) + (0x2 * (((UINT64)X10 * X18) + (0x2 * (((UINT64)X12 * X16) + ((UINT64)X8 * X17)))))); + UINT64 X32 = (0x2 * ((((UINT64)X14 * X16) + ((UINT64)X12 * X18)) + ((UINT64)X10 * X17))); + UINT64 X33 = (0x2 * ((((UINT64)X16 * X16) + ((UINT64)X14 * X18)) + ((UINT64)(0x2 * X12) * X17))); + UINT64 X34 = (0x2 * (((UINT64)X16 * X18) + ((UINT64)X14 * X17))); + UINT64 X35 = (((UINT64)X18 * X18) + ((UINT64)(0x4 * X16) * X17)); + UINT64 X36 = ((UINT64)(0x2 * X18) * X17); + UINT64 X37 = ((UINT64)(0x2 * X17) * X17); + UINT64 X38 = (X27 + (X37 << 0x4)); + UINT64 X39 = (X38 + (X37 << 0x1)); + UINT64 X40 = (X39 + X37); + UINT64 X41 = (X26 + (X36 << 0x4)); + UINT64 X42 = (X41 + (X36 << 0x1)); + UINT64 X43 = (X42 + X36); + UINT64 X44 = (X25 + (X35 << 0x4)); + UINT64 X45 = (X44 + (X35 << 0x1)); + UINT64 X46 = (X45 + X35); + UINT64 X47 = (X24 + (X34 << 0x4)); + UINT64 X48 = (X47 + (X34 << 0x1)); + UINT64 X49 = (X48 + X34); + UINT64 X50 = (X23 + (X33 << 0x4)); + UINT64 X51 = (X50 + (X33 << 0x1)); + UINT64 X52 = (X51 + X33); + UINT64 X53 = (X22 + (X32 << 0x4)); + UINT64 X54 = (X53 + (X32 << 0x1)); + UINT64 X55 = (X54 + X32); + UINT64 X56 = (X21 + (X31 << 0x4)); + UINT64 X57 = (X56 + (X31 << 0x1)); + UINT64 X58 = (X57 + X31); + UINT64 X59 = (X20 + (X30 << 0x4)); + UINT64 X60 = (X59 + (X30 << 0x1)); + UINT64 X61 = (X60 + X30); + UINT64 X62 = (X19 + (X29 << 0x4)); + UINT64 X63 = (X62 + (X29 << 0x1)); + UINT64 X64 = (X63 + X29); + UINT64 X65 = (X64 >> 0x1a); + UINT32 X66 = ((UINT32)X64 & 0x3ffffff); + UINT64 X67 = (X65 + X61); + UINT64 X68 = (X67 >> 0x19); + UINT32 X69 = ((UINT32)X67 & 0x1ffffff); + UINT64 X70 = (X68 + X58); + UINT64 X71 = (X70 >> 0x1a); + UINT32 X72 = ((UINT32)X70 & 0x3ffffff); + UINT64 X73 = (X71 + X55); + UINT64 X74 = (X73 >> 0x19); + UINT32 X75 = ((UINT32)X73 & 0x1ffffff); + UINT64 X76 = (X74 + X52); + UINT64 X77 = (X76 >> 0x1a); + UINT32 X78 = ((UINT32)X76 & 0x3ffffff); + UINT64 X79 = (X77 + X49); + UINT64 X80 = (X79 >> 0x19); + UINT32 X81 = ((UINT32)X79 & 0x1ffffff); + UINT64 X82 = (X80 + X46); + UINT64 X83 = (X82 >> 0x1a); + UINT32 X84 = ((UINT32)X82 & 0x3ffffff); + UINT64 X85 = (X83 + X43); + UINT64 X86 = (X85 >> 0x19); + UINT32 X87 = ((UINT32)X85 & 0x1ffffff); + UINT64 X88 = (X86 + X40); + UINT64 X89 = (X88 >> 0x1a); + UINT32 X90 = ((UINT32)X88 & 0x3ffffff); + UINT64 X91 = (X89 + X28); + UINT64 X92 = (X91 >> 0x19); + UINT32 X93 = ((UINT32)X91 & 0x1ffffff); + UINT64 X94 = (X66 + (0x13 * X92)); + UINT32 X95 = (UINT32)(X94 >> 0x1a); + UINT32 X96 = ((UINT32)X94 & 0x3ffffff); + UINT32 X97 = (X95 + X69); + UINT32 X98 = (X97 >> 0x19); + UINT32 X99 = (X97 & 0x1ffffff); + Out[0] = X96; + Out[1] = X99; + Out[2] = (X98 + X72); + Out[3] = X75; + Out[4] = X78; + Out[5] = X81; + Out[6] = X84; + Out[7] = X87; + Out[8] = X90; + Out[9] = X93; +} + +static inline VOID +FeSqTl(_Out_ Fe *H, _In_ CONST FeLoose *F) +{ + FeSqrImpl(H->V, F->V); +} + +static inline VOID +FeSqTt(_Out_ Fe *H, _In_ CONST Fe *F) +{ + FeSqrImpl(H->V, F->V); +} + +static inline VOID +FeLooseInvert(_Out_ Fe *Out, _In_ CONST FeLoose *Z) +{ + Fe T0; + Fe T1; + Fe T2; + Fe T3; + LONG i; + + FeSqTl(&T0, Z); + FeSqTt(&T1, &T0); + for (i = 1; i < 2; ++i) + FeSqTt(&T1, &T1); + FeMulTlt(&T1, Z, &T1); + FeMulTtt(&T0, &T0, &T1); + FeSqTt(&T2, &T0); + FeMulTtt(&T1, &T1, &T2); + FeSqTt(&T2, &T1); + for (i = 1; i < 5; ++i) + FeSqTt(&T2, &T2); + FeMulTtt(&T1, &T2, &T1); + FeSqTt(&T2, &T1); + for (i = 1; i < 10; ++i) + FeSqTt(&T2, &T2); + FeMulTtt(&T2, &T2, &T1); + FeSqTt(&T3, &T2); + for (i = 1; i < 20; ++i) + FeSqTt(&T3, &T3); + FeMulTtt(&T2, &T3, &T2); + FeSqTt(&T2, &T2); + for (i = 1; i < 10; ++i) + FeSqTt(&T2, &T2); + FeMulTtt(&T1, &T2, &T1); + FeSqTt(&T2, &T1); + for (i = 1; i < 50; ++i) + FeSqTt(&T2, &T2); + FeMulTtt(&T2, &T2, &T1); + FeSqTt(&T3, &T2); + for (i = 1; i < 100; ++i) + FeSqTt(&T3, &T3); + FeMulTtt(&T2, &T3, &T2); + FeSqTt(&T2, &T2); + for (i = 1; i < 50; ++i) + FeSqTt(&T2, &T2); + FeMulTtt(&T1, &T2, &T1); + FeSqTt(&T1, &T1); + for (i = 1; i < 5; ++i) + FeSqTt(&T1, &T1); + FeMulTtt(Out, &T1, &T0); +} + +static inline VOID +FeInvert(_Out_ Fe *Out, _In_ CONST Fe *Z) +{ + FeLoose l; + FeCopyLt(&l, Z); + FeLooseInvert(Out, &l); +} + +/* Replace (f,g) with (g,f) if b == 1; + * replace (f,g) with (f,g) if b == 0. + * + * Preconditions: b in {0,1} + */ +static inline VOID +FeCswap(_Inout_ Fe *F, _Inout_ Fe *G, _In_ UINT32 B) +{ + LONG i; + B = 0 - B; + for (i = 0; i < 10; ++i) + { + UINT32 X = F->V[i] ^ G->V[i]; + X &= B; + F->V[i] ^= X; + G->V[i] ^= X; + } +} + +/* NOTE: based on fiat-crypto fe_mul, edited for in2=121666, 0, 0.*/ +static inline VOID +FeMul121666Impl(_Out_writes_all_(10) UINT32 Out[10], _In_reads_(10) CONST UINT32 In1[10]) +{ + CONST UINT32 X20 = In1[9]; + CONST UINT32 X21 = In1[8]; + CONST UINT32 X19 = In1[7]; + CONST UINT32 X17 = In1[6]; + CONST UINT32 X15 = In1[5]; + CONST UINT32 X13 = In1[4]; + CONST UINT32 X11 = In1[3]; + CONST UINT32 X9 = In1[2]; + CONST UINT32 X7 = In1[1]; + CONST UINT32 X5 = In1[0]; + CONST UINT32 X38 = 0; + CONST UINT32 X39 = 0; + CONST UINT32 X37 = 0; + CONST UINT32 X35 = 0; + CONST UINT32 X33 = 0; + CONST UINT32 X31 = 0; + CONST UINT32 X29 = 0; + CONST UINT32 X27 = 0; + CONST UINT32 X25 = 0; + CONST UINT32 X23 = 121666; + UINT64 X40 = ((UINT64)X23 * X5); + UINT64 X41 = (((UINT64)X23 * X7) + ((UINT64)X25 * X5)); + UINT64 X42 = ((((UINT64)(0x2 * X25) * X7) + ((UINT64)X23 * X9)) + ((UINT64)X27 * X5)); + UINT64 X43 = (((((UINT64)X25 * X9) + ((UINT64)X27 * X7)) + ((UINT64)X23 * X11)) + ((UINT64)X29 * X5)); + UINT64 X44 = + (((((UINT64)X27 * X9) + (0x2 * (((UINT64)X25 * X11) + ((UINT64)X29 * X7)))) + ((UINT64)X23 * X13)) + + ((UINT64)X31 * X5)); + UINT64 X45 = + (((((((UINT64)X27 * X11) + ((UINT64)X29 * X9)) + ((UINT64)X25 * X13)) + ((UINT64)X31 * X7)) + + ((UINT64)X23 * X15)) + + ((UINT64)X33 * X5)); + UINT64 X46 = + (((((0x2 * ((((UINT64)X29 * X11) + ((UINT64)X25 * X15)) + ((UINT64)X33 * X7))) + ((UINT64)X27 * X13)) + + ((UINT64)X31 * X9)) + + ((UINT64)X23 * X17)) + + ((UINT64)X35 * X5)); + UINT64 X47 = + (((((((((UINT64)X29 * X13) + ((UINT64)X31 * X11)) + ((UINT64)X27 * X15)) + ((UINT64)X33 * X9)) + + ((UINT64)X25 * X17)) + + ((UINT64)X35 * X7)) + + ((UINT64)X23 * X19)) + + ((UINT64)X37 * X5)); + UINT64 X48 = + (((((((UINT64)X31 * X13) + + (0x2 * (((((UINT64)X29 * X15) + ((UINT64)X33 * X11)) + ((UINT64)X25 * X19)) + ((UINT64)X37 * X7)))) + + ((UINT64)X27 * X17)) + + ((UINT64)X35 * X9)) + + ((UINT64)X23 * X21)) + + ((UINT64)X39 * X5)); + UINT64 X49 = + (((((((((((UINT64)X31 * X15) + ((UINT64)X33 * X13)) + ((UINT64)X29 * X17)) + ((UINT64)X35 * X11)) + + ((UINT64)X27 * X19)) + + ((UINT64)X37 * X9)) + + ((UINT64)X25 * X21)) + + ((UINT64)X39 * X7)) + + ((UINT64)X23 * X20)) + + ((UINT64)X38 * X5)); + UINT64 X50 = + (((((0x2 * ((((((UINT64)X33 * X15) + ((UINT64)X29 * X19)) + ((UINT64)X37 * X11)) + ((UINT64)X25 * X20)) + + ((UINT64)X38 * X7))) + + ((UINT64)X31 * X17)) + + ((UINT64)X35 * X13)) + + ((UINT64)X27 * X21)) + + ((UINT64)X39 * X9)); + UINT64 X51 = + (((((((((UINT64)X33 * X17) + ((UINT64)X35 * X15)) + ((UINT64)X31 * X19)) + ((UINT64)X37 * X13)) + + ((UINT64)X29 * X21)) + + ((UINT64)X39 * X11)) + + ((UINT64)X27 * X20)) + + ((UINT64)X38 * X9)); + UINT64 X52 = + (((((UINT64)X35 * X17) + + (0x2 * (((((UINT64)X33 * X19) + ((UINT64)X37 * X15)) + ((UINT64)X29 * X20)) + ((UINT64)X38 * X11)))) + + ((UINT64)X31 * X21)) + + ((UINT64)X39 * X13)); + UINT64 X53 = + (((((((UINT64)X35 * X19) + ((UINT64)X37 * X17)) + ((UINT64)X33 * X21)) + ((UINT64)X39 * X15)) + + ((UINT64)X31 * X20)) + + ((UINT64)X38 * X13)); + UINT64 X54 = + (((0x2 * ((((UINT64)X37 * X19) + ((UINT64)X33 * X20)) + ((UINT64)X38 * X15))) + ((UINT64)X35 * X21)) + + ((UINT64)X39 * X17)); + UINT64 X55 = (((((UINT64)X37 * X21) + ((UINT64)X39 * X19)) + ((UINT64)X35 * X20)) + ((UINT64)X38 * X17)); + UINT64 X56 = (((UINT64)X39 * X21) + (0x2 * (((UINT64)X37 * X20) + ((UINT64)X38 * X19)))); + UINT64 X57 = (((UINT64)X39 * X20) + ((UINT64)X38 * X21)); + UINT64 X58 = ((UINT64)(0x2 * X38) * X20); + UINT64 X59 = (X48 + (X58 << 0x4)); + UINT64 X60 = (X59 + (X58 << 0x1)); + UINT64 X61 = (X60 + X58); + UINT64 X62 = (X47 + (X57 << 0x4)); + UINT64 X63 = (X62 + (X57 << 0x1)); + UINT64 X64 = (X63 + X57); + UINT64 X65 = (X46 + (X56 << 0x4)); + UINT64 X66 = (X65 + (X56 << 0x1)); + UINT64 X67 = (X66 + X56); + UINT64 X68 = (X45 + (X55 << 0x4)); + UINT64 X69 = (X68 + (X55 << 0x1)); + UINT64 X70 = (X69 + X55); + UINT64 X71 = (X44 + (X54 << 0x4)); + UINT64 X72 = (X71 + (X54 << 0x1)); + UINT64 X73 = (X72 + X54); + UINT64 X74 = (X43 + (X53 << 0x4)); + UINT64 X75 = (X74 + (X53 << 0x1)); + UINT64 X76 = (X75 + X53); + UINT64 X77 = (X42 + (X52 << 0x4)); + UINT64 X78 = (X77 + (X52 << 0x1)); + UINT64 X79 = (X78 + X52); + UINT64 X80 = (X41 + (X51 << 0x4)); + UINT64 X81 = (X80 + (X51 << 0x1)); + UINT64 X82 = (X81 + X51); + UINT64 X83 = (X40 + (X50 << 0x4)); + UINT64 X84 = (X83 + (X50 << 0x1)); + UINT64 X85 = (X84 + X50); + UINT64 X86 = (X85 >> 0x1a); + UINT32 X87 = ((UINT32)X85 & 0x3ffffff); + UINT64 X88 = (X86 + X82); + UINT64 X89 = (X88 >> 0x19); + UINT32 X90 = ((UINT32)X88 & 0x1ffffff); + UINT64 X91 = (X89 + X79); + UINT64 X92 = (X91 >> 0x1a); + UINT32 X93 = ((UINT32)X91 & 0x3ffffff); + UINT64 X94 = (X92 + X76); + UINT64 X95 = (X94 >> 0x19); + UINT32 X96 = ((UINT32)X94 & 0x1ffffff); + UINT64 X97 = (X95 + X73); + UINT64 X98 = (X97 >> 0x1a); + UINT32 X99 = ((UINT32)X97 & 0x3ffffff); + UINT64 X100 = (X98 + X70); + UINT64 X101 = (X100 >> 0x19); + UINT32 X102 = ((UINT32)X100 & 0x1ffffff); + UINT64 X103 = (X101 + X67); + UINT64 X104 = (X103 >> 0x1a); + UINT32 X105 = ((UINT32)X103 & 0x3ffffff); + UINT64 X106 = (X104 + X64); + UINT64 X107 = (X106 >> 0x19); + UINT32 X108 = ((UINT32)X106 & 0x1ffffff); + UINT64 X109 = (X107 + X61); + UINT64 X110 = (X109 >> 0x1a); + UINT32 X111 = ((UINT32)X109 & 0x3ffffff); + UINT64 X112 = (X110 + X49); + UINT64 X113 = (X112 >> 0x19); + UINT32 X114 = ((UINT32)X112 & 0x1ffffff); + UINT64 X115 = (X87 + (0x13 * X113)); + UINT32 X116 = (UINT32)(X115 >> 0x1a); + UINT32 X117 = ((UINT32)X115 & 0x3ffffff); + UINT32 X118 = (X116 + X90); + UINT32 X119 = (X118 >> 0x19); + UINT32 X120 = (X118 & 0x1ffffff); + Out[0] = X117; + Out[1] = X120; + Out[2] = (X119 + X93); + Out[3] = X96; + Out[4] = X99; + Out[5] = X102; + Out[6] = X105; + Out[7] = X108; + Out[8] = X111; + Out[9] = X114; +} + +static inline VOID +FeMul121666(_Out_ Fe *H, _In_ CONST FeLoose *F) +{ + FeMul121666Impl(H->V, F->V); +} + +_Use_decl_annotations_ +BOOLEAN +Curve25519( + UINT8 Out[CURVE25519_KEY_SIZE], + CONST UINT8 Scalar[CURVE25519_KEY_SIZE], + CONST UINT8 Point[CURVE25519_KEY_SIZE]) +{ + Fe X1, X2, Z2, X3, Z3; + FeLoose X2l, Z2l, X3l; + UINT32 Swap = 0; + LONG Pos; + UINT8 E[32]; + + RtlCopyMemory(E, Scalar, 32); + Curve25519ClampSecret(E); + + /* The following implementation was transcribed to Coq and proven to + * correspond to unary scalar multiplication in affine coordinates given + * that x1 != 0 is the x coordinate of some point on the curve. It was + * also checked in Coq that doing a ladderstep with x1 = x3 = 0 gives + * z2' = z3' = 0, and z2 = z3 = 0 gives z2' = z3' = 0. The statement was + * quantified over the underlying field, so it applies to Curve25519 + * itself and the quadratic twist of Curve25519. It was not proven in + * Coq that prime-field arithmetic correctly simulates extension-field + * arithmetic on prime-field values. The decoding of the byte array + * representation of e was not considered. + * + * Specification of Montgomery curves in affine coordinates: + * <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Spec/MontgomeryCurve.v#L27> + * + * Proof that these form a group that is isomorphic to a Weierstrass + * curve: + * <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/AffineProofs.v#L35> + * + * Coq transcription and correctness proof of the loop + * (where scalarbits=255): + * <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/XZ.v#L118> + * <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/XZProofs.v#L278> + * preconditions: 0 <= e < 2^255 (not necessarily e < order), + * fe_invert(0) = 0 + */ + FeFrombytes(&X1, Point); + Fe1(&X2); + Fe0(&Z2); + FeCopy(&X3, &X1); + Fe1(&Z3); + + for (Pos = 254; Pos >= 0; --Pos) + { + Fe Tmp0, Tmp1; + FeLoose Tmp0l, Tmp1l; + /* loop invariant as of right before the test, for the case + * where x1 != 0: + * pos >= -1; if z2 = 0 then x2 is nonzero; if z3 = 0 then x3 + * is nonzero + * let r := e >> (pos+1) in the following equalities of + * projective points: + * to_xz (r*P) === if swap then (x3, z3) else (x2, z2) + * to_xz ((r+1)*P) === if swap then (x2, z2) else (x3, z3) + * x1 is the nonzero x coordinate of the nonzero + * point (r*P-(r+1)*P) + */ + UINT32 B = 1 & (E[Pos / 8] >> (Pos & 7)); + Swap ^= B; + FeCswap(&X2, &X3, Swap); + FeCswap(&Z2, &Z3, Swap); + Swap = B; + /* Coq transcription of ladderstep formula (called from + * transcribed loop): + * <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/XZ.v#L89> + * <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/XZProofs.v#L131> + * x1 != 0 + * <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/XZProofs.v#L217> + * x1 = 0 + * <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/XZProofs.v#L147> + */ + FeSub(&Tmp0l, &X3, &Z3); + FeSub(&Tmp1l, &X2, &Z2); + FeAdd(&X2l, &X2, &Z2); + FeAdd(&Z2l, &X3, &Z3); + FeMulTll(&Z3, &Tmp0l, &X2l); + FeMulTll(&Z2, &Z2l, &Tmp1l); + FeSqTl(&Tmp0, &Tmp1l); + FeSqTl(&Tmp1, &X2l); + FeAdd(&X3l, &Z3, &Z2); + FeSub(&Z2l, &Z3, &Z2); + FeMulTtt(&X2, &Tmp1, &Tmp0); + FeSub(&Tmp1l, &Tmp1, &Tmp0); + FeSqTl(&Z2, &Z2l); + FeMul121666(&Z3, &Tmp1l); + FeSqTl(&X3, &X3l); + FeAdd(&Tmp0l, &Tmp0, &Z3); + FeMulTtt(&Z3, &X1, &Z2); + FeMulTll(&Z2, &Tmp1l, &Tmp0l); + } + /* here pos=-1, so r=e, so to_xz (e*P) === if swap then (x3, z3) + * else (x2, z2) + */ + FeCswap(&X2, &X3, Swap); + FeCswap(&Z2, &Z3, Swap); + + FeInvert(&Z2, &Z2); + FeMulTtt(&X2, &X2, &Z2); + FeTobytes(Out, &X2); + + RtlSecureZeroMemory(&X1, sizeof(X1)); + RtlSecureZeroMemory(&X2, sizeof(X2)); + RtlSecureZeroMemory(&Z2, sizeof(Z2)); + RtlSecureZeroMemory(&X3, sizeof(X3)); + RtlSecureZeroMemory(&Z3, sizeof(Z3)); + RtlSecureZeroMemory(&X2l, sizeof(X2l)); + RtlSecureZeroMemory(&Z2l, sizeof(Z2l)); + RtlSecureZeroMemory(&X3l, sizeof(X3l)); + RtlSecureZeroMemory(&E, sizeof(E)); + + return !Curve25519IsNull(Out); +} + +#ifdef DBG +# include "selftest/chacha20poly1305.c" +# ifdef ALLOC_PRAGMA +# pragma alloc_text(INIT, CryptoSelftest) +# endif +_Use_decl_annotations_ +BOOLEAN CryptoSelftest(VOID) +{ + BOOLEAN Success = TRUE; + SIMD_STATE Simd; + SimdGet(&Simd); + ULONG FullSet = (ULONG)Simd.CpuFeatures; + Simd.CpuFeatures = 0; + do + { + if (!ChaCha20Poly1305Selftest(&Simd)) + { + LogDebug("chacha20poly1305 self-test combination 0x%lx: FAIL", Simd.CpuFeatures); + Success = FALSE; + } + Simd.CpuFeatures = ((ULONG)Simd.CpuFeatures - FullSet) & FullSet; + } while (Simd.CpuFeatures); + SimdPut(&Simd); + if (Success) + LogDebug("crypto self-tests: pass"); + return Success; +} +#endif diff --git a/driver/crypto.h b/driver/crypto.h new file mode 100644 index 0000000..e051368 --- /dev/null +++ b/driver/crypto.h @@ -0,0 +1,369 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +#include "undocumented.h" +#include <ndis.h> + +#if defined(_M_AMD64) +typedef enum +{ + CPU_FEATURE_SSSE3 = 1 << 0, + CPU_FEATURE_AVX = 1 << 1, + CPU_FEATURE_AVX2 = 1 << 2, + CPU_FEATURE_AVX512F = 1 << 3, + CPU_FEATURE_AVX512VL = 1 << 4, + CPU_FEATURE_AVX512IFMA = 1 << 5, +} CPU_FEATURE; + +typedef struct _SIMD_STATE +{ + CPU_FEATURE CpuFeatures; + XSTATE_SAVE XState; + BOOLEAN HasSavedXState; +} SIMD_STATE; + +_IRQL_requires_max_(DISPATCH_LEVEL) +_When_(State->HasSavedXState, _Kernel_float_saved_) +_At_(State->XState, _When_(State->HasSavedXState, _Kernel_acquires_resource_(FloatState))) +VOID +SimdGet(_Out_ SIMD_STATE *State); + +_IRQL_requires_max_(DISPATCH_LEVEL) +_When_(State->HasSavedXState, _Kernel_float_restored_) +_At_( + State->XState, + _When_(State->HasSavedXState, _Kernel_requires_resource_held_(FloatState) _Kernel_releases_resource_(FloatState))) +VOID +SimdPut(_Inout_ SIMD_STATE *State); +#else +typedef struct _SIMD_STATE +{ + CHAR CpuFeatures; +} SIMD_STATE; + +_IRQL_requires_max_(DISPATCH_LEVEL) +static inline VOID +SimdGet(_Out_ SIMD_STATE *State) +{ + State->CpuFeatures = 0; +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +static inline VOID +SimdPut(_Inout_ SIMD_STATE *State) +{ +} +#endif + +_Must_inspect_result_ +static FORCEINLINE BOOLEAN +CryptoEqualMemory16(_In_reads_bytes_(16) CONST VOID *Data1, _In_reads_bytes_(16) CONST VOID *Data2) +{ +#if _WIN64 + CONST volatile ULONG64 *D1 = Data1, *D2 = Data2; + volatile ULONG64 NotEqual = (ReadULong64NoFence(&D1[0]) ^ ReadULong64NoFence(&D2[0])) | + (ReadULong64NoFence(&D1[1]) ^ ReadULong64NoFence(&D2[1])); +#else + CONST volatile ULONG *D1 = Data1, *D2 = Data2; + volatile ULONG NotEqual = + (ReadULongNoFence(&D1[0]) ^ ReadULongNoFence(&D2[0])) | (ReadULongNoFence(&D1[1]) ^ ReadULongNoFence(&D2[1])) | + (ReadULongNoFence(&D1[2]) ^ ReadULongNoFence(&D2[2])) | (ReadULongNoFence(&D1[3]) ^ ReadULongNoFence(&D2[3])); +#endif + return !NotEqual; +} + +_Must_inspect_result_ +static FORCEINLINE BOOLEAN +CryptoEqualMemory32(_In_reads_bytes_(32) CONST VOID *Data1, _In_reads_bytes_(32) CONST VOID *Data2) +{ +#if _WIN64 + CONST volatile ULONG64 *D1 = Data1, *D2 = Data2; + volatile ULONG64 NotEqual = (ReadULong64NoFence(&D1[0]) ^ ReadULong64NoFence(&D2[0])) | + (ReadULong64NoFence(&D1[1]) ^ ReadULong64NoFence(&D2[1])) | + (ReadULong64NoFence(&D1[2]) ^ ReadULong64NoFence(&D2[2])) | + (ReadULong64NoFence(&D1[3]) ^ ReadULong64NoFence(&D2[3])); +#else + CONST volatile ULONG *D1 = Data1, *D2 = Data2; + volatile ULONG NotEqual = + (ReadULongNoFence(&D1[0]) ^ ReadULongNoFence(&D2[0])) | (ReadULongNoFence(&D1[1]) ^ ReadULongNoFence(&D2[1])) | + (ReadULongNoFence(&D1[2]) ^ ReadULongNoFence(&D2[2])) | (ReadULongNoFence(&D1[3]) ^ ReadULongNoFence(&D2[3])) | + (ReadULongNoFence(&D1[4]) ^ ReadULongNoFence(&D2[4])) | (ReadULongNoFence(&D1[5]) ^ ReadULongNoFence(&D2[5])) | + (ReadULongNoFence(&D1[6]) ^ ReadULongNoFence(&D2[6])) | (ReadULongNoFence(&D1[7]) ^ ReadULongNoFence(&D2[7])); +#endif + return !NotEqual; +} + +#pragma warning(disable : 28159) /* We're bug checking in case somebody's RNG is borked. */ +static inline VOID +CryptoRandom(_Out_writes_bytes_all_(Len) PVOID RandomData, _In_ SIZE_T Len) +{ +#ifdef SDV_HACKS + /* SDV refuses to run if we link against cng.lib, so for SDV mode, we just insert a stub + * function instead. Then, out of an abundance of caution, we make sure that this always + * bug checks in case somebody's build system somehow winds up building this by accident. + */ + if (Len) + KeBugCheck(CRYPTO_LIBRARY_INTERNAL_ERROR); + RtlFillMemory(RandomData, Len, 'A'); +#else + /* CryptoRandom is documented as "Always returns TRUE." We see from reverse engineering that + * it returns FALSE if AesRNGState_generate fails, and that fails if a size addition overflows, + * which presumably it won't given that we only ever pass small values of Len. So just assert + * here that the documentation is correct. + */ + if (!SystemPrng(RandomData, Len)) + KeBugCheck(CRYPTO_LIBRARY_INTERNAL_ERROR); +#endif +} + +enum CHACHA20POLY1305_LENGTHS +{ + XCHACHA20POLY1305_NONCE_SIZE = 24, + CHACHA20POLY1305_KEY_SIZE = 32, + CHACHA20POLY1305_AUTHTAG_SIZE = 16 +}; + +VOID +ChaCha20Poly1305Encrypt( + _Out_writes_bytes_all_(SrcLen + CHACHA20POLY1305_AUTHTAG_SIZE) UINT8 *Dst, + _In_reads_bytes_(SrcLen) CONST UINT8 *Src, + _In_ CONST SIZE_T SrcLen, + _In_reads_bytes_(AdLen) CONST UINT8 *Ad, + _In_ CONST SIZE_T AdLen, + _In_ CONST UINT64 Nonce, + _In_ CONST UINT8 Key[CHACHA20POLY1305_KEY_SIZE]); + +_Must_inspect_result_ +BOOLEAN +ChaCha20Poly1305Decrypt( + _Out_writes_bytes_all_(SrcLen - CHACHA20POLY1305_AUTHTAG_SIZE) UINT8 *Dst, + _In_reads_bytes_(SrcLen) CONST UINT8 *Src, + _In_ CONST SIZE_T SrcLen, + _In_reads_bytes_(AdLen) CONST UINT8 *Ad, + _In_ CONST SIZE_T AdLen, + _In_ CONST UINT64 Nonce, + _In_ CONST UINT8 Key[CHACHA20POLY1305_KEY_SIZE]); + +_Must_inspect_result_ +BOOLEAN +ChaCha20Poly1305EncryptMdl( + _Out_writes_bytes_all_(SrcLen + CHACHA20POLY1305_AUTHTAG_SIZE) UINT8 *Dst, + _In_ MDL *Src, + _In_ CONST ULONG SrcLen, + _In_ CONST ULONG SrcOffset, + _In_reads_bytes_(AdLen) CONST UINT8 *Ad, + _In_ CONST SIZE_T AdLen, + _In_ CONST UINT64 Nonce, + _In_ CONST UINT8 Key[CHACHA20POLY1305_KEY_SIZE], + _In_opt_ CONST SIMD_STATE *Simd); + +_Must_inspect_result_ +BOOLEAN +ChaCha20Poly1305DecryptMdl( + _Out_writes_bytes_all_(SrcLen - CHACHA20POLY1305_AUTHTAG_SIZE) UINT8 *Dst, + _In_ MDL *Src, + _In_ CONST ULONG SrcLen, + _In_ CONST ULONG SrcOffset, + _In_reads_bytes_(AdLen) CONST UINT8 *Ad, + _In_ CONST SIZE_T AdLen, + _In_ CONST UINT64 Nonce, + _In_ CONST UINT8 Key[CHACHA20POLY1305_KEY_SIZE], + _In_opt_ CONST SIMD_STATE *Simd); + +VOID +XChaCha20Poly1305Encrypt( + _Out_writes_bytes_all_(SrcLen + CHACHA20POLY1305_AUTHTAG_SIZE) UINT8 *Dst, + _In_reads_bytes_(SrcLen) CONST UINT8 *Src, + _In_ CONST SIZE_T SrcLen, + _In_reads_bytes_(AdLen) CONST UINT8 *Ad, + _In_ CONST SIZE_T AdLen, + _In_ CONST UINT8 Nonce[XCHACHA20POLY1305_NONCE_SIZE], + _In_ CONST UINT8 Key[CHACHA20POLY1305_KEY_SIZE]); + +_Must_inspect_result_ +BOOLEAN +XChaCha20Poly1305Decrypt( + _Out_writes_bytes_all_(SrcLen - CHACHA20POLY1305_AUTHTAG_SIZE) UINT8 *Dst, + _In_reads_bytes_(SrcLen) CONST UINT8 *Src, + _In_ CONST SIZE_T SrcLen, + _In_reads_bytes_(AdLen) CONST UINT8 *Ad, + _In_ CONST SIZE_T AdLen, + _In_ CONST UINT8 Nonce[XCHACHA20POLY1305_NONCE_SIZE], + _In_ CONST UINT8 Key[CHACHA20POLY1305_KEY_SIZE]); + +enum BLAKE2S_LENGTHS +{ + BLAKE2S_BLOCK_SIZE = 64, + BLAKE2S_HASH_SIZE = 32, + BLAKE2S_KEY_SIZE = 32 +}; + +typedef struct _BLAKE2S_STATE +{ + UINT32 H[8]; + UINT32 T[2]; + UINT32 F[2]; + UINT8 Buf[BLAKE2S_BLOCK_SIZE]; + ULONG BufLen; + ULONG OutLen; +} BLAKE2S_STATE; + +VOID +Blake2sInit(_Out_ BLAKE2S_STATE *State, _In_ CONST SIZE_T OutLen); + +VOID +Blake2sInitKey( + _Out_ BLAKE2S_STATE *State, + _In_ CONST SIZE_T OutLen, + _In_reads_bytes_(KeyLen) CONST UINT8 *Key, + _In_ CONST SIZE_T KeyLen); + +VOID +Blake2sUpdate(_Inout_ BLAKE2S_STATE *State, _In_reads_bytes_(InLen) CONST UINT8 *In, _In_ SIZE_T InLen); + +VOID +Blake2sFinal(_Inout_ BLAKE2S_STATE *State, _Out_writes_bytes_all_(State->OutLen) UINT8 *Out); + +VOID +Blake2s( + _Out_writes_bytes_all_(OutLen) UINT8 *Out, + _In_reads_bytes_(InLen) CONST UINT8 *In, + _In_reads_bytes_(KeyLen) CONST UINT8 *Key, + _In_ CONST SIZE_T OutLen, + _In_ CONST SIZE_T InLen, + _In_ CONST SIZE_T KeyLen); + +VOID +Blake2s256Hmac( + _Out_writes_bytes_all_(BLAKE2S_HASH_SIZE) UINT8 *Out, + _In_reads_bytes_(InLen) CONST UINT8 *In, + _In_reads_bytes_(KeyLen) CONST UINT8 *Key, + _In_ CONST SIZE_T InLen, + _In_ CONST SIZE_T KeyLen); + +typedef struct _SIPHASH_KEY +{ + UINT64 Key[2]; +} SIPHASH_KEY; + +UINT64 +Siphash(_In_reads_bytes_(Len) CONST VOID *Data, _In_ SIZE_T Len, _In_ CONST SIPHASH_KEY *Key); +UINT64 +Siphash1u64(_In_ CONST UINT64 A, _In_ CONST SIPHASH_KEY *Key); +UINT64 +Siphash2u64(_In_ CONST UINT64 A, _In_ CONST UINT64 B, _In_ CONST SIPHASH_KEY *Key); +UINT64 +Siphash3u64(_In_ CONST UINT64 A, _In_ CONST UINT64 B, _In_ CONST UINT64 C, _In_ CONST SIPHASH_KEY *Key); +UINT64 +Siphash4u64( + _In_ CONST UINT64 A, + _In_ CONST UINT64 B, + _In_ CONST UINT64 C, + _In_ CONST UINT64 D, + _In_ CONST SIPHASH_KEY *Key); +UINT64 +Siphash1u32(_In_ CONST UINT32 A, _In_ CONST SIPHASH_KEY *Key); +UINT64 +Siphash3u32(_In_ CONST UINT32 A, _In_ CONST UINT32 B, _In_ CONST UINT32 C, _In_ CONST SIPHASH_KEY *Key); + +static inline UINT64 +Siphash2u32(_In_ CONST UINT32 A, _In_ CONST UINT32 B, _In_ CONST SIPHASH_KEY *Key) +{ + return Siphash1u64((UINT64)B << 32 | A, Key); +} +static inline UINT64 +Siphash4u32( + _In_ CONST UINT32 A, + _In_ CONST UINT32 B, + _In_ CONST UINT32 C, + _In_ CONST UINT32 D, + _In_ CONST SIPHASH_KEY *Key) +{ + return Siphash2u64((UINT64)B << 32 | A, (UINT64)D << 32 | C, Key); +} + +typedef struct _HSIPHASH_KEY +{ + ULONG_PTR Key[2]; +} HSIPHASH_KEY; + +UINT32 +Hsiphash(_In_reads_bytes_(Len) CONST VOID *Data, _In_ SIZE_T Len, _In_ CONST HSIPHASH_KEY *Key); +UINT32 +Hsiphash1u32(_In_ CONST UINT32 A, _In_ CONST HSIPHASH_KEY *Key); +UINT32 +Hsiphash2u32(_In_ CONST UINT32 A, _In_ CONST UINT32 B, _In_ CONST HSIPHASH_KEY *Key); +UINT32 +Hsiphash3u32(_In_ CONST UINT32 A, _In_ CONST UINT32 B, _In_ CONST UINT32 C, _In_ CONST HSIPHASH_KEY *Key); +UINT32 +Hsiphash4u32( + _In_ CONST UINT32 A, + _In_ CONST UINT32 B, + _In_ CONST UINT32 C, + _In_ CONST UINT32 D, + _In_ CONST HSIPHASH_KEY *Key); + +enum CURVE25519_LENGTHS +{ + CURVE25519_KEY_SIZE = 32 +}; + +_Must_inspect_result_ +BOOLEAN +Curve25519( + _Out_writes_bytes_all_(CURVE25519_KEY_SIZE) UINT8 Out[CURVE25519_KEY_SIZE], + _In_reads_bytes_(CURVE25519_KEY_SIZE) CONST UINT8 Scalar[CURVE25519_KEY_SIZE], + _In_reads_bytes_(CURVE25519_KEY_SIZE) CONST UINT8 Point[CURVE25519_KEY_SIZE]); + +_Must_inspect_result_ +static inline BOOLEAN +Curve25519GeneratePublic( + _Out_writes_bytes_all_(CURVE25519_KEY_SIZE) UINT8 Pub[CURVE25519_KEY_SIZE], + _In_reads_bytes_(CURVE25519_KEY_SIZE) CONST UINT8 Secret[CURVE25519_KEY_SIZE]) +{ + static CONST UINT8 Basepoint[CURVE25519_KEY_SIZE] = { 9 }; + return Curve25519(Pub, Secret, Basepoint); +} + +static inline VOID +Curve25519ClampSecret(_Inout_updates_bytes_(CURVE25519_KEY_SIZE) UINT8 Secret[CURVE25519_KEY_SIZE]) +{ + Secret[0] &= 248; + Secret[31] = (Secret[31] & 127) | 64; +} + +static inline VOID +Curve25519GenerateSecret(_Out_writes_bytes_all_(CURVE25519_KEY_SIZE) UINT8 Secret[CURVE25519_KEY_SIZE]) +{ + CryptoRandom(Secret, CURVE25519_KEY_SIZE); + Curve25519ClampSecret(Secret); +} + +_Must_inspect_result_ +static FORCEINLINE BOOLEAN +Curve25519IsNull(_In_reads_bytes_(CURVE25519_KEY_SIZE) CONST UINT8 Pub[CURVE25519_KEY_SIZE]) +{ +#if _WIN64 + CONST volatile ULONG64 *P = (CONST volatile ULONG64 *)Pub; + volatile ULONG64 NotZero = + ReadULong64NoFence(&P[0]) | ReadULong64NoFence(&P[1]) | ReadULong64NoFence(&P[2]) | ReadULong64NoFence(&P[3]); +#else + CONST volatile ULONG *P = (CONST volatile ULONG *)Pub; + volatile ULONG NotZero = ReadULongNoFence(&P[0]) | ReadULongNoFence(&P[1]) | ReadULongNoFence(&P[2]) | + ReadULongNoFence(&P[3]) | ReadULongNoFence(&P[4]) | ReadULongNoFence(&P[5]) | + ReadULongNoFence(&P[6]) | ReadULongNoFence(&P[7]); +#endif + return !NotZero; +} + +VOID CryptoDriverEntry(VOID); + +#ifdef DBG +_IRQL_requires_max_(PASSIVE_LEVEL) +BOOLEAN +CryptoSelftest(VOID); +#endif diff --git a/driver/device.c b/driver/device.c new file mode 100644 index 0000000..976a009 --- /dev/null +++ b/driver/device.c @@ -0,0 +1,1035 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include "interlocked.h" +#include "containers.h" +#include "device.h" +#include "ioctl.h" +#include "messages.h" +#include "peer.h" +#include "queueing.h" +#include "ratelimiter.h" +#include "socket.h" +#include "timers.h" +#include "logging.h" +#include <ntstrsafe.h> +#include <netioapi.h> + +#pragma warning(disable : 28175) /* undocumented: the member of struct should not be accessed by a driver */ + +#define NDIS_MINIPORT_VERSION_MIN ((NDIS_MINIPORT_MINIMUM_MAJOR_VERSION << 16) | NDIS_MINIPORT_MINIMUM_MINOR_VERSION) +#define NDIS_MINIPORT_VERSION_MAX ((NDIS_MINIPORT_MAJOR_VERSION << 16) | NDIS_MINIPORT_MINOR_VERSION) + +#define VENDOR_NAME "WireGuard Tunnel" +#define VENDOR_ID 0xFFFFFF00 +#define LINK_SPEED 100000000000ULL /* 100gbps */ +#define BUFFER_SPACE 0x4000000 /* 64MiB */ + +static UINT NdisVersion; +static NDIS_HANDLE NdisMiniportDriverHandle; +LIST_ENTRY DeviceList; +EX_PUSH_LOCK DeviceListLock; + +MINIPORT_UNLOAD Unload; + +_Use_decl_annotations_ +VOID +DeviceStart(WG_DEVICE *Wg) +{ + WG_PEER *Peer; + + LIST_FOR_EACH_ENTRY (Peer, &Wg->PeerList, WG_PEER, PeerList) + { + PacketSendStagedPackets(Peer); + if (Peer->PersistentKeepaliveInterval) + PacketSendKeepalive(Peer); + } +} + +static MINIPORT_RESTART Restart; +_Use_decl_annotations_ +static NDIS_STATUS +Restart(NDIS_HANDLE MiniportAdapterContext, PNDIS_MINIPORT_RESTART_PARAMETERS MiniportRestartParameters) +{ + WG_DEVICE *Wg = (WG_DEVICE *)MiniportAdapterContext; + + MuAcquirePushLockExclusive(&Wg->DeviceUpdateLock); + ExReInitializeRundownProtection(&Wg->ItemsInFlight); + if (ReadBooleanNoFence(&Wg->IsUp)) + DeviceStart(Wg); + MuReleasePushLockExclusive(&Wg->DeviceUpdateLock); + return NDIS_STATUS_SUCCESS; +} + +_Use_decl_annotations_ +VOID +DeviceStop(WG_DEVICE *Wg) +{ + WG_PEER *Peer; + + LIST_FOR_EACH_ENTRY (Peer, &Wg->PeerList, WG_PEER, PeerList) + { + PacketPurgeStagedPackets(Peer); + TimersStop(Peer); + NoiseHandshakeClear(&Peer->Handshake); + NoiseKeypairsClear(&Peer->Keypairs); + NoiseResetLastSentHandshake(&Peer->LastSentHandshake); + } + FreeIncomingHandshakes(Wg); +} + +static MINIPORT_PAUSE Pause; +_Use_decl_annotations_ +static NDIS_STATUS +Pause(NDIS_HANDLE MiniportAdapterContext, PNDIS_MINIPORT_PAUSE_PARAMETERS MiniportPauseParameters) +{ + WG_DEVICE *Wg = (WG_DEVICE *)MiniportAdapterContext; + + MuAcquirePushLockExclusive(&Wg->DeviceUpdateLock); + ExWaitForRundownProtectionRelease(&Wg->ItemsInFlight); + if (ReadBooleanNoFence(&Wg->IsUp)) + DeviceStop(Wg); + MuReleasePushLockExclusive(&Wg->DeviceUpdateLock); + return NDIS_STATUS_SUCCESS; +} + +_Use_decl_annotations_ +VOID +DeviceIndicateConnectionStatus(NDIS_HANDLE MiniportAdapterHandle, NDIS_MEDIA_CONNECT_STATE MediaConnectState) +{ + NDIS_LINK_STATE State = { .Header = { .Type = NDIS_OBJECT_TYPE_DEFAULT, + .Revision = NDIS_LINK_STATE_REVISION_1, + .Size = NDIS_SIZEOF_LINK_STATE_REVISION_1 }, + .MediaConnectState = MediaConnectState, + .MediaDuplexState = MediaDuplexStateFull, + .XmitLinkSpeed = LINK_SPEED, + .RcvLinkSpeed = LINK_SPEED, + .PauseFunctions = NdisPauseFunctionsUnsupported }; + + NDIS_STATUS_INDICATION Indication = { .Header = { .Type = NDIS_OBJECT_TYPE_STATUS_INDICATION, + .Revision = NDIS_STATUS_INDICATION_REVISION_1, + .Size = NDIS_SIZEOF_STATUS_INDICATION_REVISION_1 }, + .SourceHandle = MiniportAdapterHandle, + .StatusCode = NDIS_STATUS_LINK_STATE, + .StatusBuffer = &State, + .StatusBufferSize = sizeof(State) }; + + NdisMIndicateStatusEx(MiniportAdapterHandle, &Indication); +} + +static MINIPORT_SEND_NET_BUFFER_LISTS SendNetBufferLists; +_Use_decl_annotations_ +static VOID +SendNetBufferLists( + NDIS_HANDLE MiniportAdapterContext, + NET_BUFFER_LIST *NetBufferLists, + NDIS_PORT_NUMBER PortNumber, + ULONG SendFlags) +{ + WG_DEVICE *Wg = (WG_DEVICE *)MiniportAdapterContext; + ULONG CompleteFlags = 0; + if (SendFlags & NDIS_SEND_FLAGS_DISPATCH_LEVEL) + CompleteFlags |= NDIS_SEND_COMPLETE_FLAGS_DISPATCH_LEVEL; + for (NET_BUFFER_LIST *Nbl = NetBufferLists, *NextNbl; Nbl; Nbl = NextNbl) + { + NextNbl = NET_BUFFER_LIST_NEXT_NBL(Nbl); + NET_BUFFER_LIST_NEXT_NBL(Nbl) = NULL; + + if (!ExAcquireRundownProtection(&Wg->ItemsInFlight)) + { + NET_BUFFER_LIST_STATUS(Nbl) = NDIS_STATUS_PAUSED; + NdisMSendNetBufferListsComplete(Wg->MiniportAdapterHandle, Nbl, CompleteFlags); + ++Wg->Statistics.ifOutDiscards; + continue; + } + if (!ReadBooleanNoFence(&Wg->IsUp)) + { + NET_BUFFER_LIST_STATUS(Nbl) = NDIS_STATUS_MEDIA_DISCONNECTED; + goto returnNbl; + } + + NET_BUFFER *Nb = NET_BUFFER_LIST_FIRST_NB(Nbl); + if (!Nb) + { + LogInfoRatelimited(Wg, "Missing NET_BUFFER"); + NET_BUFFER_LIST_STATUS(Nbl) = NDIS_STATUS_FAILURE; + ++Wg->Statistics.ifOutErrors; + goto returnNbl; + } + + NET_BUFFER_LIST *CloneNbl = MemAllocateNetBufferListWithClonedGeometry( + Wg->NblPool, Wg->NbPool, Nbl, sizeof(MESSAGE_DATA) + NoiseEncryptedLen(0) + MESSAGE_PADDING_MULTIPLE - 1); + if (!CloneNbl) + { + NET_BUFFER_LIST_STATUS(Nbl) = NDIS_STATUS_RESOURCES; + goto returnNbl; + } + Nbl = CloneNbl; + + CONST UINT16_BE Protocol = NET_BUFFER_LIST_PROTOCOL(Nbl); + IPV4HDR *Header4 = NULL; + IPV6HDR *Header6 = NULL; + VOID *Header = NULL; + /* Potential TOCTOU? Generally NDIS considers headers fair game for R/W, but raw sockets + * and hyper-v devices make me fear that a physically local user might be able to modify + * Header4/6 while we're reading it. + */ + if (Protocol == Htons(NDIS_ETH_TYPE_IPV4)) + Header4 = Header = NdisGetDataBuffer(Nb, sizeof(IPV4HDR), NULL, 1, 0); + else if (Protocol == Htons(NDIS_ETH_TYPE_IPV6)) + Header6 = Header = NdisGetDataBuffer(Nb, sizeof(IPV6HDR), NULL, 1, 0); + else + { + LogInfoRatelimited(Wg, "Unsupported NBL protocol"); + NET_BUFFER_LIST_STATUS(Nbl) = NDIS_STATUS_FAILURE; + ++Wg->Statistics.ifOutErrors; + goto returnNbl; + } + if ((!Header4 || Header4->Version != 4 || Ntohs(Header4->TotLen) != NET_BUFFER_DATA_LENGTH(Nb)) && + (!Header6 || Header6->Version != 6 || + Ntohs(Header6->PayloadLen) + sizeof(IPV6HDR) != NET_BUFFER_DATA_LENGTH(Nb))) + { + LogInfoRatelimited(Wg, "Invalid IP packet"); + NET_BUFFER_LIST_STATUS(Nbl) = NDIS_STATUS_FAILURE; + ++Wg->Statistics.ifOutErrors; + goto returnNbl; + } + + WG_PEER *Peer = AllowedIpsLookupDst(&Wg->PeerAllowedIps, Protocol, Header); + if (!Peer) + { + NET_BUFFER_LIST_STATUS(Nbl) = NDIS_STATUS_FAILURE; + ++Wg->Statistics.ifOutErrors; + goto cleanupICMP; + } + ADDRESS_FAMILY Family = ReadUShortNoFence(&Peer->Endpoint.Addr.si_family); + if (Family != AF_INET && Family != AF_INET6) + { + LogInfoRatelimited( + Wg, "No valid endpoint has been configured or discovered for peer %llu", Peer->InternalId); + NET_BUFFER_LIST_STATUS(Nbl) = NDIS_STATUS_FAILURE; + ++Wg->Statistics.ifOutErrors; + goto cleanupPeer; + } + + KIRQL Irql; + KeAcquireSpinLock(&Peer->StagedPacketQueue.Lock, &Irql); + /* If the queue is getting too big, we start removing the oldest packets + * until it's small again. We do this before adding the new packet, so + * we don't remove GSO segments that are in excess. + */ + while (NetBufferListQueueLength(&Peer->StagedPacketQueue) > MAX_STAGED_PACKETS) + { + NET_BUFFER_LIST *NblToDiscard = NetBufferListDequeue(&Peer->StagedPacketQueue); + _Analysis_assume_(NblToDiscard); /* NetBufferListQueueLength() > MAX_STAGED_PACKETS implies + NetBufferListDequeue() returns a NBL. */ + NET_BUFFER_LIST_STATUS(NblToDiscard) = NDIS_STATUS_FAILURE; + ++Wg->Statistics.ifOutDiscards; + FreeSendNetBufferList(Wg, NblToDiscard, CompleteFlags | NDIS_SEND_COMPLETE_FLAGS_DISPATCH_LEVEL); + } + NetBufferListEnqueue(&Peer->StagedPacketQueue, Nbl); + KeReleaseSpinLock(&Peer->StagedPacketQueue.Lock, Irql); + + PacketSendStagedPackets(Peer); + PeerPut(Peer); + continue; + + cleanupPeer: + PeerPut(Peer); + cleanupICMP: + /* TODO: If v4, send ICMP_DEST_UNREACH/ICMP_HOST_UNREACH. + * If v6, send ICMPV6_DEST_UNREACH/ICMPV6_ADDR_UNREACH. + */ + returnNbl: + FreeSendNetBufferList(Wg, Nbl, CompleteFlags); + ++Wg->Statistics.ifOutDiscards; + } +} + +static MINIPORT_CANCEL_SEND CancelSend; +_Use_decl_annotations_ +static VOID +CancelSend(NDIS_HANDLE MiniportAdapterContext, PVOID CancelId) +{ +} + +static MINIPORT_RETURN_NET_BUFFER_LISTS ReturnNetBufferLists; +_Use_decl_annotations_ +static VOID +ReturnNetBufferLists(NDIS_HANDLE MiniportAdapterContext, PNET_BUFFER_LIST NetBufferLists, ULONG ReturnFlags) +{ + WG_DEVICE *Wg = (WG_DEVICE *)MiniportAdapterContext; + FreeReceiveNetBufferList(Wg, NetBufferLists); +} + +static EX_CALLBACK_FUNCTION MtuRegistryChange; +_Use_decl_annotations_ +static NTSTATUS +MtuRegistryChange(_In_ PVOID CallbackContext, _In_opt_ PVOID Argument1, _In_opt_ PVOID Argument2) +{ + WG_DEVICE *Wg = CallbackContext; + REG_NOTIFY_CLASS NotificationClass = (REG_NOTIFY_CLASS)(ULONG_PTR)Argument1; + REG_POST_OPERATION_INFORMATION *Post = (REG_POST_OPERATION_INFORMATION *)Argument2; + REG_SET_VALUE_KEY_INFORMATION *Pre; + UNICODE_STRING MtuKeyName = RTL_CONSTANT_STRING(L"MTU"); + ULONG_PTR Obj1, Obj2; + + if (NotificationClass != RegNtPostSetValueKey || !Post || Post->Status != STATUS_SUCCESS) + goto out; + Pre = Post->PreInformation; + if (!Pre || Pre->Object != Post->Object || Pre->DataSize != sizeof(ULONG) || Pre->Type != REG_DWORD || + !RtlEqualUnicodeString(Pre->ValueName, &MtuKeyName, TRUE)) + goto out; + if (!NT_SUCCESS(CmCallbackGetKeyObjectID(&Wg->MtuRegistryNotifier, Post->Object, &Obj1, NULL)) || + !NT_SUCCESS(CmCallbackGetKeyObjectID(&Wg->MtuRegistryNotifier, Wg->MtuRegistryKeyObject, &Obj2, NULL)) || + Obj1 != Obj2) + goto out; + Wg->Mtu = min(MTU_MAX, *(DWORD *)Pre->Data); +out: + return STATUS_SUCCESS; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +_Must_inspect_result_ +static NTSTATUS +SetupMtuRegistryWatcher(_Inout_ WG_DEVICE *Wg, _In_ CONST NET_LUID *Luid) +{ + Wg->Mtu = 1500 - DATA_PACKET_MINIMUM_LENGTH; + + UNICODE_STRING Prefix = RTL_CONSTANT_STRING( + L"\\REGISTRY\\MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters\\Interfaces\\"); + UNICODE_STRING RegKey; + WCHAR RegKeyBuf[128]; + RtlInitEmptyUnicodeString(&RegKey, RegKeyBuf, sizeof(RegKeyBuf)); + NTSTATUS Status = RtlUnicodeStringCat(&RegKey, &Prefix); + if (!NT_SUCCESS(Status)) + return Status; + + GUID InterfaceGuid; + Status = ConvertInterfaceLuidToGuid(Luid, &InterfaceGuid); + if (!NT_SUCCESS(Status)) + return Status; + UNICODE_STRING InterfaceGuidString; + Status = RtlStringFromGUID(&InterfaceGuid, &InterfaceGuidString); + if (!NT_SUCCESS(Status)) + return Status; + Status = RtlUnicodeStringCat(&RegKey, &InterfaceGuidString); + RtlFreeUnicodeString(&InterfaceGuidString); + if (!NT_SUCCESS(Status)) + return Status; + + OBJECT_ATTRIBUTES ObjectAttributes = { 0 }; + InitializeObjectAttributes(&ObjectAttributes, &RegKey, OBJ_CASE_INSENSITIVE | OBJ_KERNEL_HANDLE, NULL, NULL); + HANDLE KeyHandle; + Status = ZwCreateKey( + &KeyHandle, KEY_QUERY_VALUE | KEY_SET_VALUE, &ObjectAttributes, 0, NULL, REG_OPTION_NON_VOLATILE, NULL); + if (!NT_SUCCESS(Status)) + return Status; + + UNICODE_STRING MtuName = RTL_CONSTANT_STRING(L"MTU"); + struct + { + KEY_VALUE_FULL_INFORMATION Info; + UCHAR Buf[63]; + } InfoBuf; + ULONG Len; + Status = ZwQueryValueKey(KeyHandle, &MtuName, KeyValueFullInformation, &InfoBuf.Info, sizeof(InfoBuf), &Len); + if (NT_SUCCESS(Status) && InfoBuf.Info.DataLength == sizeof(ULONG) && InfoBuf.Info.Type == REG_DWORD) + Wg->Mtu = min(MTU_MAX, *(ULONG *)((ULONG_PTR)&InfoBuf.Info + InfoBuf.Info.DataOffset)); + else + Status = ZwSetValueKey(KeyHandle, &MtuName, 0, REG_DWORD, &Wg->Mtu, sizeof(Wg->Mtu)); + if (!NT_SUCCESS(Status)) + goto cleanupHandle; + + Status = ObReferenceObjectByHandle(KeyHandle, 0, NULL, KernelMode, &Wg->MtuRegistryKeyObject, NULL); + if (!NT_SUCCESS(Status)) + goto cleanupHandle; + ZwClose(KeyHandle); + + Status = CmRegisterCallback(MtuRegistryChange, Wg, &Wg->MtuRegistryNotifier); + if (!NT_SUCCESS(Status)) + goto cleanupObject; + + return STATUS_SUCCESS; + +cleanupObject: + ObDereferenceObject(Wg->MtuRegistryKeyObject); + return Status; +cleanupHandle: + ZwClose(KeyHandle); + return Status; +} + +static MINIPORT_HALT HaltEx; +_Use_decl_annotations_ +static VOID +HaltEx(NDIS_HANDLE MiniportAdapterContext, NDIS_HALT_ACTION HaltAction) +{ + WG_DEVICE *Wg = (WG_DEVICE *)MiniportAdapterContext; + IoctlHalt(Wg); + MuAcquirePushLockExclusive(&DeviceListLock); + RemoveEntryList(&Wg->DeviceList); + MuReleasePushLockExclusive(&DeviceListLock); + MuAcquirePushLockExclusive(&Wg->DeviceUpdateLock); + Wg->IncomingPort = 0; + SocketReinit(Wg, NULL, NULL, 0); + if (Wg->SocketOwnerProcess) + { + ObDereferenceObject(Wg->SocketOwnerProcess); + Wg->SocketOwnerProcess = NULL; + } + CmUnRegisterCallback(Wg->MtuRegistryNotifier); + ObDereferenceObject(Wg->MtuRegistryKeyObject); + PeerRemoveAll(Wg); + MulticoreWorkQueueDestroy(&Wg->DecryptThreads); + MulticoreWorkQueueDestroy(&Wg->EncryptThreads); + MulticoreWorkQueueDestroy(&Wg->RxThreads); + MulticoreWorkQueueDestroy(&Wg->TxThreads); + MulticoreWorkQueueDestroy(&Wg->HandshakeRxThreads); + MulticoreWorkQueueDestroy(&Wg->HandshakeTxThreads); + PtrRingCleanup(&Wg->DecryptQueue, NULL); + PtrRingCleanup(&Wg->EncryptQueue, NULL); + RcuBarrier(); + NoiseStaticIdentityClear(&Wg->StaticIdentity); + FreeIncomingHandshakes(Wg); + MemFree(Wg->IndexHashtable); + MemFree(Wg->PeerHashtable); + MuReleasePushLockExclusive(&Wg->DeviceUpdateLock); + + NdisFreeNetBufferPool(Wg->NbPool); + NdisFreeNetBufferListPool(Wg->NblPool); + WritePointerNoFence(&Wg->MiniportAdapterHandle, NULL); + LogInfo(Wg, "Interface destroyed"); + MemFree(Wg); +} + +#pragma warning(suppress : 28194) /* `Wg` is aliased in NdisMSetMiniportAttributes. */ +_IRQL_requires_max_(PASSIVE_LEVEL) +static NDIS_STATUS +RegisterAdapter(_In_ NDIS_HANDLE MiniportAdapterHandle, _In_ __drv_aliasesMem WG_DEVICE *Wg) +{ + NDIS_MINIPORT_ADAPTER_REGISTRATION_ATTRIBUTES AdapterRegistrationAttributes = { + .Header = { .Type = NDIS_OBJECT_TYPE_MINIPORT_ADAPTER_REGISTRATION_ATTRIBUTES, + .Revision = NdisVersion < NDIS_RUNTIME_VERSION_630 + ? NDIS_MINIPORT_ADAPTER_REGISTRATION_ATTRIBUTES_REVISION_1 + : NDIS_MINIPORT_ADAPTER_REGISTRATION_ATTRIBUTES_REVISION_2, + .Size = NdisVersion < NDIS_RUNTIME_VERSION_630 + ? NDIS_SIZEOF_MINIPORT_ADAPTER_REGISTRATION_ATTRIBUTES_REVISION_1 + : NDIS_SIZEOF_MINIPORT_ADAPTER_REGISTRATION_ATTRIBUTES_REVISION_2 }, + .AttributeFlags = NDIS_MINIPORT_ATTRIBUTES_NO_HALT_ON_SUSPEND | NDIS_MINIPORT_ATTRIBUTES_SURPRISE_REMOVE_OK, + .InterfaceType = NdisInterfaceInternal, + .MiniportAdapterContext = Wg + }; + NDIS_STATUS Status = NdisMSetMiniportAttributes( + MiniportAdapterHandle, (PNDIS_MINIPORT_ADAPTER_ATTRIBUTES)&AdapterRegistrationAttributes); + if (!NT_SUCCESS(Status)) + return Status; + + NDIS_PM_CAPABILITIES PmCapabilities = { + .Header = { .Type = NDIS_OBJECT_TYPE_DEFAULT, + .Revision = NdisVersion < NDIS_RUNTIME_VERSION_630 ? NDIS_PM_CAPABILITIES_REVISION_1 + : NDIS_PM_CAPABILITIES_REVISION_2, + .Size = NdisVersion < NDIS_RUNTIME_VERSION_630 ? NDIS_SIZEOF_NDIS_PM_CAPABILITIES_REVISION_1 + : NDIS_SIZEOF_NDIS_PM_CAPABILITIES_REVISION_2 }, + .MinMagicPacketWakeUp = NdisDeviceStateUnspecified, + .MinPatternWakeUp = NdisDeviceStateUnspecified, + .MinLinkChangeWakeUp = NdisDeviceStateUnspecified + }; + static NDIS_OID SupportedOids[] = { OID_GEN_MAXIMUM_TOTAL_SIZE, + OID_GEN_CURRENT_LOOKAHEAD, + OID_GEN_TRANSMIT_BUFFER_SPACE, + OID_GEN_RECEIVE_BUFFER_SPACE, + OID_GEN_TRANSMIT_BLOCK_SIZE, + OID_GEN_RECEIVE_BLOCK_SIZE, + OID_GEN_VENDOR_DESCRIPTION, + OID_GEN_VENDOR_ID, + OID_GEN_VENDOR_DRIVER_VERSION, + OID_GEN_XMIT_OK, + OID_GEN_RCV_OK, + OID_GEN_CURRENT_PACKET_FILTER, + OID_GEN_STATISTICS, + OID_GEN_INTERRUPT_MODERATION, + OID_GEN_LINK_PARAMETERS, + OID_PNP_SET_POWER, + OID_PNP_QUERY_POWER }; + NDIS_MINIPORT_ADAPTER_GENERAL_ATTRIBUTES AdapterGeneralAttributes = { + .Header = { .Type = NDIS_OBJECT_TYPE_MINIPORT_ADAPTER_GENERAL_ATTRIBUTES, + .Revision = NDIS_MINIPORT_ADAPTER_GENERAL_ATTRIBUTES_REVISION_2, + .Size = NDIS_SIZEOF_MINIPORT_ADAPTER_GENERAL_ATTRIBUTES_REVISION_2 }, + .MediaType = NdisMediumIP, + .PhysicalMediumType = NdisPhysicalMediumUnspecified, + .MtuSize = MTU_MAX, + .MaxXmitLinkSpeed = LINK_SPEED, + .MaxRcvLinkSpeed = LINK_SPEED, + .RcvLinkSpeed = LINK_SPEED, + .XmitLinkSpeed = LINK_SPEED, + .MediaConnectState = MediaConnectStateDisconnected, + .MediaDuplexState = MediaDuplexStateFull, + .LookaheadSize = MTU_MAX, + .MacOptions = + NDIS_MAC_OPTION_TRANSFERS_NOT_PEND | NDIS_MAC_OPTION_COPY_LOOKAHEAD_DATA | NDIS_MAC_OPTION_NO_LOOPBACK, + .SupportedPacketFilters = NDIS_PACKET_TYPE_DIRECTED | NDIS_PACKET_TYPE_ALL_MULTICAST | + NDIS_PACKET_TYPE_BROADCAST | NDIS_PACKET_TYPE_ALL_LOCAL | + NDIS_PACKET_TYPE_ALL_FUNCTIONAL, + .AccessType = NET_IF_ACCESS_BROADCAST, + .DirectionType = NET_IF_DIRECTION_SENDRECEIVE, + .ConnectionType = NET_IF_CONNECTION_DEDICATED, + .IfType = IF_TYPE_PROP_VIRTUAL, + .IfConnectorPresent = FALSE, + .SupportedStatistics = Wg->Statistics.SupportedStatistics, + .SupportedPauseFunctions = NdisPauseFunctionsUnsupported, + .SupportedOidList = SupportedOids, + .SupportedOidListLength = sizeof(SupportedOids), + .AutoNegotiationFlags = + NDIS_LINK_STATE_XMIT_LINK_SPEED_AUTO_NEGOTIATED | NDIS_LINK_STATE_RCV_LINK_SPEED_AUTO_NEGOTIATED | + NDIS_LINK_STATE_DUPLEX_AUTO_NEGOTIATED | NDIS_LINK_STATE_PAUSE_FUNCTIONS_AUTO_NEGOTIATED, + .PowerManagementCapabilitiesEx = &PmCapabilities + }; + Status = + NdisMSetMiniportAttributes(MiniportAdapterHandle, (PNDIS_MINIPORT_ADAPTER_ATTRIBUTES)&AdapterGeneralAttributes); + if (!NT_SUCCESS(Status)) + return Status; + + NDIS_OFFLOAD Offload = { + .Header = { .Type = NDIS_OBJECT_TYPE_OFFLOAD, + .Revision = NdisVersion < NDIS_RUNTIME_VERSION_630 ? NDIS_OFFLOAD_REVISION_2 + : NdisVersion < NDIS_RUNTIME_VERSION_650 ? NDIS_OFFLOAD_REVISION_3 + : NdisVersion < NDIS_RUNTIME_VERSION_670 ? NDIS_OFFLOAD_REVISION_4 + : NdisVersion < NDIS_RUNTIME_VERSION_683 ? NDIS_OFFLOAD_REVISION_5 + : NDIS_OFFLOAD_REVISION_6, + .Size = NdisVersion < NDIS_RUNTIME_VERSION_630 ? NDIS_SIZEOF_NDIS_OFFLOAD_REVISION_2 + : NdisVersion < NDIS_RUNTIME_VERSION_650 ? NDIS_SIZEOF_NDIS_OFFLOAD_REVISION_3 + : NdisVersion < NDIS_RUNTIME_VERSION_670 ? NDIS_SIZEOF_NDIS_OFFLOAD_REVISION_4 + : NdisVersion < NDIS_RUNTIME_VERSION_683 ? NDIS_SIZEOF_NDIS_OFFLOAD_REVISION_5 + : NDIS_SIZEOF_NDIS_OFFLOAD_REVISION_6 }, + .Checksum = { .IPv4Receive = { .IpOptionsSupported = NDIS_OFFLOAD_SUPPORTED, + .TcpOptionsSupported = NDIS_OFFLOAD_SUPPORTED, + .TcpChecksum = NDIS_OFFLOAD_SUPPORTED, + .UdpChecksum = NDIS_OFFLOAD_SUPPORTED, + .IpChecksum = NDIS_OFFLOAD_SUPPORTED }, + .IPv6Receive = { .IpExtensionHeadersSupported = NDIS_OFFLOAD_SUPPORTED, + .TcpOptionsSupported = NDIS_OFFLOAD_SUPPORTED, + .TcpChecksum = NDIS_OFFLOAD_SUPPORTED, + .UdpChecksum = NDIS_OFFLOAD_SUPPORTED } }, + }; + NDIS_TCP_CONNECTION_OFFLOAD ConnectionOffload = { + .Header = { .Type = NDIS_OBJECT_TYPE_DEFAULT, + .Revision = NDIS_TCP_CONNECTION_OFFLOAD_REVISION_1, + .Size = NDIS_SIZEOF_TCP_CONNECTION_OFFLOAD_REVISION_1 }, + }; + NDIS_MINIPORT_ADAPTER_OFFLOAD_ATTRIBUTES AdapterOffloadAttributes = { + .Header = { .Type = NDIS_OBJECT_TYPE_MINIPORT_ADAPTER_OFFLOAD_ATTRIBUTES, + .Revision = NDIS_MINIPORT_ADAPTER_OFFLOAD_ATTRIBUTES_REVISION_1, + .Size = NDIS_SIZEOF_MINIPORT_ADAPTER_OFFLOAD_ATTRIBUTES_REVISION_1 }, + .DefaultOffloadConfiguration = &Offload, + .HardwareOffloadCapabilities = &Offload, + .DefaultTcpConnectionOffloadConfiguration = &ConnectionOffload, + .TcpConnectionOffloadHardwareCapabilities = &ConnectionOffload, + }; + Status = + NdisMSetMiniportAttributes(MiniportAdapterHandle, (PNDIS_MINIPORT_ADAPTER_ATTRIBUTES)&AdapterOffloadAttributes); + if (!NT_SUCCESS(Status)) + return Status; + + return NDIS_STATUS_SUCCESS; +} + +static MINIPORT_INITIALIZE InitializeEx; +_Use_decl_annotations_ +static NDIS_STATUS +InitializeEx( + NDIS_HANDLE MiniportAdapterHandle, + NDIS_HANDLE MiniportDriverContext, + PNDIS_MINIPORT_INIT_PARAMETERS MiniportInitParameters) +{ + NTSTATUS Status = WskInit(); + if (!NT_SUCCESS(Status)) + return Status; + + WG_DEVICE *Wg = MemAllocateAndZero(sizeof(*Wg)); + if (!Wg) + return NDIS_STATUS_RESOURCES; + Wg->MiniportAdapterHandle = MiniportAdapterHandle; + Wg->InterfaceIndex = MiniportInitParameters->IfIndex; + Wg->InterfaceLuid = MiniportInitParameters->NetLuid; + LogRingInit(&Wg->Log); + + NdisMGetDeviceProperty(MiniportAdapterHandle, NULL, &Wg->FunctionalDeviceObject, NULL, NULL, NULL); + if (!Wg->FunctionalDeviceObject) + { + Status = STATUS_INVALID_PARAMETER; + goto cleanupWg; + } + NT_ASSERT(!Wg->FunctionalDeviceObject->Reserved); + /* Reverse engineering indicates that we'd be better off calling + * NdisWdfGetAdapterContextFromAdapterHandle(functional_device), which points to our WG_DEVICE object + * directly, but this isn't available before Windows 10, so for now we just stick it into this reserved field. + * Revisit this when we drop support for old Windows versions. */ + Wg->FunctionalDeviceObject->Reserved = Wg; + + NET_BUFFER_LIST_POOL_PARAMETERS NblPoolParameters = { + .Header = { .Type = NDIS_OBJECT_TYPE_DEFAULT, + .Revision = NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1, + .Size = NDIS_SIZEOF_NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1 }, + .ProtocolId = NDIS_PROTOCOL_ID_DEFAULT, + .PoolTag = MEMORY_TAG + }; + Wg->NblPool = NdisAllocateNetBufferListPool(MiniportAdapterHandle, &NblPoolParameters); + if (!Wg->NblPool) + { + Status = STATUS_INSUFFICIENT_RESOURCES; + goto cleanupWg; + } + NET_BUFFER_POOL_PARAMETERS NbPoolParameters = { .Header = { .Type = NDIS_OBJECT_TYPE_DEFAULT, + .Revision = NET_BUFFER_POOL_PARAMETERS_REVISION_1, + .Size = + NDIS_SIZEOF_NET_BUFFER_POOL_PARAMETERS_REVISION_1 }, + .PoolTag = MEMORY_TAG }; + Wg->NbPool = NdisAllocateNetBufferPool(MiniportAdapterHandle, &NbPoolParameters); + if (!Wg->NbPool) + { + Status = STATUS_INSUFFICIENT_RESOURCES; + goto cleanupNblPool; + } + ExInitializeRundownProtection(&Wg->ItemsInFlight); + ExRundownCompleted(&Wg->ItemsInFlight); /* Wait until Restart is called to mark this active. */ + + MuInitializePushLock(&Wg->StaticIdentity.Lock); + MuInitializePushLock(&Wg->SocketUpdateLock); + MuInitializePushLock(&Wg->DeviceUpdateLock); + PeerSerialInit(&Wg->TxQueue); + PeerSerialInit(&Wg->RxQueue); + PeerSerialInit(&Wg->HandshakeTxQueue); + NetBufferListInitQueue(&Wg->HandshakeRxQueue); + AllowedIpsInit(&Wg->PeerAllowedIps); + CookieCheckerInit(&Wg->CookieChecker, Wg); + InitializeListHead(&Wg->PeerList); + Wg->DeviceUpdateGen = 1; + + Status = STATUS_INSUFFICIENT_RESOURCES; + + Wg->PeerHashtable = PubkeyHashtableAlloc(); + if (!Wg->PeerHashtable) + goto cleanupNbPool; + + Wg->IndexHashtable = IndexHashtableAlloc(); + if (!Wg->IndexHashtable) + goto cleanupPeerHashtable; + + Wg->Statistics.Header.Type = NDIS_OBJECT_TYPE_DEFAULT; + Wg->Statistics.Header.Revision = NDIS_STATISTICS_INFO_REVISION_1; + Wg->Statistics.Header.Size = NDIS_SIZEOF_STATISTICS_INFO_REVISION_1; + Wg->Statistics.SupportedStatistics = + NDIS_STATISTICS_FLAGS_VALID_DIRECTED_FRAMES_RCV | NDIS_STATISTICS_FLAGS_VALID_MULTICAST_FRAMES_RCV | + NDIS_STATISTICS_FLAGS_VALID_BROADCAST_FRAMES_RCV | NDIS_STATISTICS_FLAGS_VALID_BYTES_RCV | + NDIS_STATISTICS_FLAGS_VALID_RCV_DISCARDS | NDIS_STATISTICS_FLAGS_VALID_RCV_ERROR | + NDIS_STATISTICS_FLAGS_VALID_DIRECTED_FRAMES_XMIT | NDIS_STATISTICS_FLAGS_VALID_MULTICAST_FRAMES_XMIT | + NDIS_STATISTICS_FLAGS_VALID_BROADCAST_FRAMES_XMIT | NDIS_STATISTICS_FLAGS_VALID_BYTES_XMIT | + NDIS_STATISTICS_FLAGS_VALID_XMIT_ERROR | NDIS_STATISTICS_FLAGS_VALID_XMIT_DISCARDS | + NDIS_STATISTICS_FLAGS_VALID_DIRECTED_BYTES_RCV | NDIS_STATISTICS_FLAGS_VALID_MULTICAST_BYTES_RCV | + NDIS_STATISTICS_FLAGS_VALID_BROADCAST_BYTES_RCV | NDIS_STATISTICS_FLAGS_VALID_DIRECTED_BYTES_XMIT | + NDIS_STATISTICS_FLAGS_VALID_MULTICAST_BYTES_XMIT | NDIS_STATISTICS_FLAGS_VALID_BROADCAST_BYTES_XMIT; + + Status = PtrRingInit(&Wg->EncryptQueue, MAX_QUEUED_PACKETS); + if (!NT_SUCCESS(Status)) + goto cleanupIndexHashtable; + + Status = PtrRingInit(&Wg->DecryptQueue, MAX_QUEUED_PACKETS); + if (!NT_SUCCESS(Status)) + goto cleanupEncryptQueue; + + Status = MulticoreWorkQueueInit(&Wg->EncryptThreads, PacketEncryptWorker); + if (!NT_SUCCESS(Status)) + goto cleanupDecryptQueue; + + Status = MulticoreWorkQueueInit(&Wg->DecryptThreads, PacketDecryptWorker); + if (!NT_SUCCESS(Status)) + goto cleanupEncryptThreads; + + Status = MulticoreWorkQueueInit(&Wg->TxThreads, PacketTxWorker); + if (!NT_SUCCESS(Status)) + goto cleanupDecryptThreads; + + Status = MulticoreWorkQueueInit(&Wg->RxThreads, PacketRxWorker); + if (!NT_SUCCESS(Status)) + goto cleanupTxThreads; + + Status = MulticoreWorkQueueInit(&Wg->HandshakeTxThreads, PacketHandshakeTxWorker); + if (!NT_SUCCESS(Status)) + goto cleanupRxThreads; + + Status = MulticoreWorkQueueInit(&Wg->HandshakeRxThreads, PacketHandshakeRxWorker); + if (!NT_SUCCESS(Status)) + goto cleanupHandshakeTxThreads; + + Status = SetupMtuRegistryWatcher(Wg, &MiniportInitParameters->NetLuid); + if (!NT_SUCCESS(Status)) + goto cleanupHandshakeRxThreads; + + Status = RegisterAdapter(MiniportAdapterHandle, Wg); + if (!NT_SUCCESS(Status)) + goto cleanupMtuRegistryNotifier; + + KeInitializeEvent(&Wg->DeviceRemoved, NotificationEvent, FALSE); + MuAcquirePushLockExclusive(&DeviceListLock); + InsertHeadList(&DeviceList, &Wg->DeviceList); + MuReleasePushLockExclusive(&DeviceListLock); + + LogInfo(Wg, "Interface created"); + + return NDIS_STATUS_SUCCESS; + +cleanupMtuRegistryNotifier: + CmUnRegisterCallback(Wg->MtuRegistryNotifier); + ObDereferenceObject(Wg->MtuRegistryKeyObject); +cleanupHandshakeRxThreads: + MulticoreWorkQueueDestroy(&Wg->HandshakeRxThreads); +cleanupHandshakeTxThreads: + MulticoreWorkQueueDestroy(&Wg->HandshakeTxThreads); +cleanupRxThreads: + MulticoreWorkQueueDestroy(&Wg->RxThreads); +cleanupTxThreads: + MulticoreWorkQueueDestroy(&Wg->TxThreads); +cleanupDecryptThreads: + MulticoreWorkQueueDestroy(&Wg->DecryptThreads); +cleanupEncryptThreads: + MulticoreWorkQueueDestroy(&Wg->EncryptThreads); +cleanupDecryptQueue: + PtrRingCleanup(&Wg->DecryptQueue, NULL); +cleanupEncryptQueue: + PtrRingCleanup(&Wg->EncryptQueue, NULL); +cleanupIndexHashtable: + MemFree(Wg->IndexHashtable); +cleanupPeerHashtable: + MemFree(Wg->PeerHashtable); +cleanupNbPool: + NdisFreeNetBufferPool(Wg->NbPool); +cleanupNblPool: + NdisFreeNetBufferListPool(Wg->NblPool); +cleanupWg: + MemFree(Wg); + if (Status == STATUS_INSUFFICIENT_RESOURCES) + return NDIS_STATUS_RESOURCES; + NdisWriteErrorLogEntry(MiniportAdapterHandle, NDIS_ERROR_CODE_DRIVER_FAILURE, 1, Status); + return NDIS_STATUS_FAILURE; +} + +static MINIPORT_DEVICE_PNP_EVENT_NOTIFY DevicePnPEventNotify; +_Use_decl_annotations_ +static VOID +DevicePnPEventNotify(NDIS_HANDLE MiniportAdapterContext, PNET_DEVICE_PNP_EVENT NetDevicePnPEvent) +{ +} + +static MINIPORT_SHUTDOWN ShutdownEx; +_Use_decl_annotations_ +static VOID +ShutdownEx(NDIS_HANDLE MiniportAdapterContext, NDIS_SHUTDOWN_ACTION ShutdownAction) +{ +} + +_IRQL_requires_max_(APC_LEVEL) +_Must_inspect_result_ +static NDIS_STATUS +OidQueryWrite(_Inout_ NDIS_OID_REQUEST *OidRequest, _In_ ULONG Value) +{ + if (OidRequest->DATA.QUERY_INFORMATION.InformationBufferLength < sizeof(Value)) + { + OidRequest->DATA.QUERY_INFORMATION.BytesNeeded = sizeof(Value); + OidRequest->DATA.QUERY_INFORMATION.BytesWritten = 0; + return NDIS_STATUS_BUFFER_TOO_SHORT; + } + + OidRequest->DATA.QUERY_INFORMATION.BytesNeeded = OidRequest->DATA.QUERY_INFORMATION.BytesWritten = sizeof(Value); + RtlCopyMemory(OidRequest->DATA.QUERY_INFORMATION.InformationBuffer, &Value, sizeof(Value)); + return NDIS_STATUS_SUCCESS; +} + +_IRQL_requires_max_(APC_LEVEL) +_Must_inspect_result_ +static NDIS_STATUS +OidQueryWrite32or64(_Inout_ NDIS_OID_REQUEST *OidRequest, _In_ ULONG64 Value) +{ + ULONG Truncated; + + if (OidRequest->DATA.QUERY_INFORMATION.InformationBufferLength < sizeof(Truncated)) + { + OidRequest->DATA.QUERY_INFORMATION.BytesNeeded = sizeof(Value); + OidRequest->DATA.QUERY_INFORMATION.BytesWritten = 0; + return NDIS_STATUS_BUFFER_TOO_SHORT; + } + + if (OidRequest->DATA.QUERY_INFORMATION.InformationBufferLength < sizeof(Value)) + { + if (!NT_SUCCESS(RtlULong64ToULong(Value, &Truncated))) + { + OidRequest->DATA.QUERY_INFORMATION.BytesNeeded = sizeof(Value); + OidRequest->DATA.QUERY_INFORMATION.BytesWritten = 0; + return NDIS_STATUS_BUFFER_TOO_SHORT; + } + OidRequest->DATA.QUERY_INFORMATION.BytesNeeded = OidRequest->DATA.QUERY_INFORMATION.BytesWritten = + sizeof(Truncated); + RtlCopyMemory(OidRequest->DATA.QUERY_INFORMATION.InformationBuffer, &Truncated, sizeof(Truncated)); + return NDIS_STATUS_SUCCESS; + } + + OidRequest->DATA.QUERY_INFORMATION.BytesNeeded = OidRequest->DATA.QUERY_INFORMATION.BytesWritten = sizeof(Value); + RtlCopyMemory(OidRequest->DATA.QUERY_INFORMATION.InformationBuffer, &Value, sizeof(Value)); + return NDIS_STATUS_SUCCESS; +} + +_IRQL_requires_max_(APC_LEVEL) +_Must_inspect_result_ +static NDIS_STATUS +OidQueryWriteBuf(_Inout_ NDIS_OID_REQUEST *OidRequest, _In_reads_bytes_(Size) CONST VOID *Buf, _In_ ULONG Size) +{ + if (OidRequest->DATA.QUERY_INFORMATION.InformationBufferLength < Size) + { + OidRequest->DATA.QUERY_INFORMATION.BytesNeeded = Size; + OidRequest->DATA.QUERY_INFORMATION.BytesWritten = 0; + return NDIS_STATUS_BUFFER_TOO_SHORT; + } + + OidRequest->DATA.QUERY_INFORMATION.BytesNeeded = OidRequest->DATA.QUERY_INFORMATION.BytesWritten = Size; + RtlCopyMemory(OidRequest->DATA.QUERY_INFORMATION.InformationBuffer, Buf, Size); + return NDIS_STATUS_SUCCESS; +} + +_IRQL_requires_max_(APC_LEVEL) +_Must_inspect_result_ +static NDIS_STATUS +OidQuery(_Inout_ WG_DEVICE *Wg, _Inout_ NDIS_OID_REQUEST *OidRequest) +{ + NT_ASSERT( + OidRequest->RequestType == NdisRequestQueryInformation || + OidRequest->RequestType == NdisRequestQueryStatistics); + + switch (OidRequest->DATA.QUERY_INFORMATION.Oid) + { + case OID_GEN_MAXIMUM_TOTAL_SIZE: + case OID_GEN_TRANSMIT_BLOCK_SIZE: + case OID_GEN_RECEIVE_BLOCK_SIZE: + return OidQueryWrite(OidRequest, MTU_MAX); + + case OID_GEN_TRANSMIT_BUFFER_SPACE: + return OidQueryWrite(OidRequest, BUFFER_SPACE); + + case OID_GEN_RECEIVE_BUFFER_SPACE: + return OidQueryWrite(OidRequest, BUFFER_SPACE); + + case OID_GEN_VENDOR_ID: + return OidQueryWrite(OidRequest, Htonl(VENDOR_ID)); + + case OID_GEN_VENDOR_DESCRIPTION: + return OidQueryWriteBuf(OidRequest, VENDOR_NAME, sizeof(VENDOR_NAME)); + + case OID_GEN_VENDOR_DRIVER_VERSION: + return OidQueryWrite(OidRequest, (WIREGUARD_VERSION_MAJ << 16) | WIREGUARD_VERSION_MIN); + + case OID_GEN_XMIT_OK: + return OidQueryWrite32or64( + OidRequest, + Wg->Statistics.ifHCOutUcastPkts + Wg->Statistics.ifHCOutMulticastPkts + + Wg->Statistics.ifHCOutBroadcastPkts); + + case OID_GEN_RCV_OK: + return OidQueryWrite32or64( + OidRequest, + Wg->Statistics.ifHCInUcastPkts + Wg->Statistics.ifHCInMulticastPkts + Wg->Statistics.ifHCInBroadcastPkts); + + case OID_GEN_STATISTICS: + return OidQueryWriteBuf(OidRequest, &Wg->Statistics, sizeof(Wg->Statistics)); + + case OID_GEN_INTERRUPT_MODERATION: { + static CONST NDIS_INTERRUPT_MODERATION_PARAMETERS InterruptParameters = { + .Header = { .Type = NDIS_OBJECT_TYPE_DEFAULT, + .Revision = NDIS_INTERRUPT_MODERATION_PARAMETERS_REVISION_1, + .Size = NDIS_SIZEOF_INTERRUPT_MODERATION_PARAMETERS_REVISION_1 }, + .InterruptModeration = NdisInterruptModerationNotSupported + }; + return OidQueryWriteBuf(OidRequest, &InterruptParameters, sizeof(InterruptParameters)); + } + + case OID_PNP_QUERY_POWER: + OidRequest->DATA.QUERY_INFORMATION.BytesNeeded = OidRequest->DATA.QUERY_INFORMATION.BytesWritten = 0; + return NDIS_STATUS_SUCCESS; + } + + OidRequest->DATA.QUERY_INFORMATION.BytesWritten = 0; + return NDIS_STATUS_NOT_SUPPORTED; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +static NDIS_STATUS +OidSet(_Inout_ WG_DEVICE *Wg, _Inout_ NDIS_OID_REQUEST *OidRequest) +{ + NT_ASSERT(OidRequest->RequestType == NdisRequestSetInformation); + + OidRequest->DATA.SET_INFORMATION.BytesNeeded = OidRequest->DATA.SET_INFORMATION.BytesRead = 0; + + switch (OidRequest->DATA.SET_INFORMATION.Oid) + { + case OID_GEN_CURRENT_PACKET_FILTER: + case OID_GEN_CURRENT_LOOKAHEAD: + if (OidRequest->DATA.SET_INFORMATION.InformationBufferLength != 4) + { + OidRequest->DATA.SET_INFORMATION.BytesNeeded = 4; + return NDIS_STATUS_INVALID_LENGTH; + } + OidRequest->DATA.SET_INFORMATION.BytesRead = 4; + return NDIS_STATUS_SUCCESS; + + case OID_GEN_LINK_PARAMETERS: + OidRequest->DATA.SET_INFORMATION.BytesRead = OidRequest->DATA.SET_INFORMATION.InformationBufferLength; + return NDIS_STATUS_SUCCESS; + + case OID_GEN_INTERRUPT_MODERATION: + return NDIS_STATUS_INVALID_DATA; + + case OID_PNP_SET_POWER: + if (OidRequest->DATA.SET_INFORMATION.InformationBufferLength != sizeof(NDIS_DEVICE_POWER_STATE)) + { + OidRequest->DATA.SET_INFORMATION.BytesNeeded = sizeof(NDIS_DEVICE_POWER_STATE); + return NDIS_STATUS_INVALID_LENGTH; + } + OidRequest->DATA.SET_INFORMATION.BytesRead = sizeof(NDIS_DEVICE_POWER_STATE); + NDIS_DEVICE_POWER_STATE PowerState; + RtlCopyMemory(&PowerState, OidRequest->DATA.SET_INFORMATION.InformationBuffer, sizeof(PowerState)); + if (PowerState >= NdisDeviceStateD1) + RcuBarrier(); + return NDIS_STATUS_SUCCESS; + } + + return NDIS_STATUS_NOT_SUPPORTED; +} + +static MINIPORT_OID_REQUEST OidRequest; +_Use_decl_annotations_ +static NDIS_STATUS +OidRequest(NDIS_HANDLE MiniportAdapterContext, PNDIS_OID_REQUEST OidRequest) +{ + switch (OidRequest->RequestType) + { + case NdisRequestQueryInformation: + case NdisRequestQueryStatistics: + return OidQuery(MiniportAdapterContext, OidRequest); + + case NdisRequestSetInformation: + return OidSet(MiniportAdapterContext, OidRequest); + + default: + return NDIS_STATUS_INVALID_OID; + } +} + +static MINIPORT_CANCEL_OID_REQUEST CancelOidRequest; +_Use_decl_annotations_ +static VOID +CancelOidRequest(NDIS_HANDLE MiniportAdapterContext, PVOID RequestId) +{ +} + +static MINIPORT_DIRECT_OID_REQUEST DirectOidRequest; +_Use_decl_annotations_ +static NDIS_STATUS +DirectOidRequest(NDIS_HANDLE MiniportAdapterContext, PNDIS_OID_REQUEST OidRequest) +{ + switch (OidRequest->RequestType) + { + case NdisRequestQueryInformation: + case NdisRequestQueryStatistics: + case NdisRequestSetInformation: + return NDIS_STATUS_NOT_SUPPORTED; + + default: + return NDIS_STATUS_INVALID_OID; + } +} + +static MINIPORT_CANCEL_DIRECT_OID_REQUEST CancelDirectOidRequest; +_Use_decl_annotations_ +static VOID +CancelDirectOidRequest(NDIS_HANDLE MiniportAdapterContext, PVOID RequestId) +{ +} + +static MINIPORT_SYNCHRONOUS_OID_REQUEST SynchronousOidRequest; +_Use_decl_annotations_ +static NDIS_STATUS +SynchronousOidRequest(NDIS_HANDLE MiniportAdapterContext, PNDIS_OID_REQUEST OidRequest) +{ + switch (OidRequest->RequestType) + { + case NdisRequestQueryInformation: + case NdisRequestQueryStatistics: + case NdisRequestSetInformation: + return NDIS_STATUS_NOT_SUPPORTED; + + default: + return NDIS_STATUS_INVALID_OID; + } +} + +#ifdef ALLOC_PRAGMA +# pragma alloc_text(INIT, DeviceDriverEntry) +#endif +_Use_decl_annotations_ +NTSTATUS +DeviceDriverEntry(DRIVER_OBJECT *DriverObject, UNICODE_STRING *RegistryPath) +{ + NTSTATUS Status; + + MuInitializePushLock(&DeviceListLock); + InitializeListHead(&DeviceList); + + NdisVersion = NdisGetVersion(); + if (NdisVersion < NDIS_MINIPORT_VERSION_MIN) + return NDIS_STATUS_UNSUPPORTED_REVISION; + if (NdisVersion > NDIS_MINIPORT_VERSION_MAX) + NdisVersion = NDIS_MINIPORT_VERSION_MAX; + + NDIS_MINIPORT_DRIVER_CHARACTERISTICS Miniport = { + .Header = { .Type = NDIS_OBJECT_TYPE_MINIPORT_DRIVER_CHARACTERISTICS, + .Revision = NdisVersion < NDIS_RUNTIME_VERSION_680 + ? NDIS_MINIPORT_DRIVER_CHARACTERISTICS_REVISION_2 + : NDIS_MINIPORT_DRIVER_CHARACTERISTICS_REVISION_3, + .Size = NdisVersion < NDIS_RUNTIME_VERSION_680 + ? NDIS_SIZEOF_MINIPORT_DRIVER_CHARACTERISTICS_REVISION_2 + : NDIS_SIZEOF_MINIPORT_DRIVER_CHARACTERISTICS_REVISION_3 }, + + .MajorNdisVersion = (UCHAR)((NdisVersion & 0x00ff0000) >> 16), + .MinorNdisVersion = (UCHAR)(NdisVersion & 0x000000ff), + + .MajorDriverVersion = WIREGUARD_VERSION_MAJ, + .MinorDriverVersion = WIREGUARD_VERSION_MIN, + + .InitializeHandlerEx = InitializeEx, + .HaltHandlerEx = HaltEx, + .UnloadHandler = Unload, + .PauseHandler = Pause, + .RestartHandler = Restart, + .OidRequestHandler = OidRequest, + .SendNetBufferListsHandler = SendNetBufferLists, + .ReturnNetBufferListsHandler = ReturnNetBufferLists, + .CancelSendHandler = CancelSend, + .DevicePnPEventNotifyHandler = DevicePnPEventNotify, + .ShutdownHandlerEx = ShutdownEx, + .CancelOidRequestHandler = CancelOidRequest, + .DirectOidRequestHandler = DirectOidRequest, + .CancelDirectOidRequestHandler = CancelDirectOidRequest, + .SynchronousOidRequestHandler = SynchronousOidRequest + }; + Status = NdisMRegisterMiniportDriver(DriverObject, RegistryPath, NULL, &Miniport, &NdisMiniportDriverHandle); + if (!NT_SUCCESS(Status)) + return Status; + IoctlDriverEntry(DriverObject); + return STATUS_SUCCESS; +} + +VOID DeviceUnload(VOID) +{ + NdisMDeregisterMiniportDriver(NdisMiniportDriverHandle); + RcuBarrier(); +} diff --git a/driver/device.h b/driver/device.h new file mode 100644 index 0000000..426131d --- /dev/null +++ b/driver/device.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +#include "allowedips.h" +#include "containers.h" +#include "cookie.h" +#include "noise.h" +#include "peerlookup.h" +#include "rcu.h" +#include "logging.h" +#include <ntifs.h> /* Must be included before <wdm.h> */ +#include <wdm.h> + +extern LIST_ENTRY DeviceList; +extern EX_PUSH_LOCK DeviceListLock; + +typedef struct _PREV_QUEUE +{ + NET_BUFFER_LIST *Head, *Tail, *Peeked; + NET_BUFFER_LIST Empty; + LONG Count; +} PREV_QUEUE; + +// Would be nice to have the MULTICORE_* stuff in queueing.h where it belongs. +typedef struct _MULTICORE_WORKTHREAD MULTICORE_WORKTHREAD; +typedef struct _MULTICORE_WORKQUEUE MULTICORE_WORKQUEUE; + +typedef _Function_class_(MULTICORE_WORKQUEUE_ROUTINE) +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +MULTICORE_WORKQUEUE_ROUTINE(_In_ MULTICORE_WORKQUEUE *); +typedef MULTICORE_WORKQUEUE_ROUTINE *PMULTICORE_WORKQUEUE_ROUTINE; + +struct _MULTICORE_WORKTHREAD +{ + SLIST_ENTRY Entry; + PKTHREAD Thread; + PROCESSOR_NUMBER Processor; + MULTICORE_WORKTHREAD *NextThread; + MULTICORE_WORKQUEUE *WorkQueue; +}; + +struct _MULTICORE_WORKQUEUE +{ + MULTICORE_WORKTHREAD *FirstThread; + KEVENT NewWork, NewCpus, Dead; + PMULTICORE_WORKQUEUE_ROUTINE Func; + PVOID NewCpuNotifier; + PKTHREAD WorkerSpawnerThread; +}; + +typedef struct _SOCKET SOCKET; + +typedef struct _PEER_SERIAL_ENTRY PEER_SERIAL_ENTRY; +struct _PEER_SERIAL_ENTRY +{ + PEER_SERIAL_ENTRY *Next; + SHORT Requeue; +}; + +typedef struct _PEER_SERIAL +{ + PEER_SERIAL_ENTRY *First, **Last; + KSPIN_LOCK Lock; +} PEER_SERIAL; + +typedef struct _WG_DEVICE +{ + NDIS_HANDLE MiniportAdapterHandle; /* This is actually a pointer to NDIS_MINIPORT_BLOCK struct. */ + DEVICE_OBJECT *FunctionalDeviceObject; + NDIS_STATISTICS_INFO Statistics; + NDIS_HANDLE NblPool, NbPool; + EX_RUNDOWN_REF ItemsInFlight; + PTR_RING EncryptQueue, DecryptQueue; + PEER_SERIAL TxQueue, RxQueue, HandshakeTxQueue; + NET_BUFFER_LIST_QUEUE HandshakeRxQueue; + MULTICORE_WORKQUEUE EncryptThreads, DecryptThreads; + MULTICORE_WORKQUEUE TxThreads, RxThreads; + MULTICORE_WORKQUEUE HandshakeTxThreads, HandshakeRxThreads; + SOCKET __rcu *Sock4, *Sock6; + NOISE_STATIC_IDENTITY StaticIdentity; + COOKIE_CHECKER CookieChecker; + PUBKEY_HASHTABLE *PeerHashtable; + INDEX_HASHTABLE *IndexHashtable; + ALLOWEDIPS_TABLE PeerAllowedIps; + EX_PUSH_LOCK DeviceUpdateLock, SocketUpdateLock; + LIST_ENTRY PeerList; + ULONG NumPeers, DeviceUpdateGen; + NET_IFINDEX InterfaceIndex; + NET_LUID InterfaceLuid; + PEPROCESS SocketOwnerProcess; + UINT16 IncomingPort; + BOOLEAN IsUp, IsDeviceRemoving; + PVOID MtuRegistryKeyObject; + LARGE_INTEGER MtuRegistryNotifier; + ULONG Mtu; + LOG_RING Log; + LIST_ENTRY DeviceList; + KEVENT DeviceRemoved; + PKTHREAD HandleForceCloseThread; +} WG_DEVICE; + +_Requires_lock_held_(Wg->DeviceUpdateLock) +VOID +DeviceStart(_Inout_ WG_DEVICE *Wg); + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_held_(Wg->DeviceUpdateLock) +VOID +DeviceStop(_Inout_ WG_DEVICE *Wg); + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +DeviceIndicateConnectionStatus(_In_ NDIS_HANDLE MiniportAdapterHandle, _In_ NDIS_MEDIA_CONNECT_STATE MediaConnectState); + +DRIVER_INITIALIZE DeviceDriverEntry; + +VOID DeviceUnload(VOID); diff --git a/driver/driver.vcxproj b/driver/driver.vcxproj new file mode 100644 index 0000000..e074df9 --- /dev/null +++ b/driver/driver.vcxproj @@ -0,0 +1,110 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Label="Globals"> + <ProjectGuid>{8B282C8F-5870-44C3-9A2A-B9091F4E9F68}</ProjectGuid> + <RootNamespace>driver</RootNamespace> + <ProjectName>driver</ProjectName> + </PropertyGroup> + <PropertyGroup Label="Configuration"> + <PlatformToolset>WindowsKernelModeDriver10.0</PlatformToolset> + <ConfigurationType>Driver</ConfigurationType> + <DriverType>WDM</DriverType> + <ForcedTargetVersion Condition="'$(SDVHacks)'=='true'">Windows10</ForcedTargetVersion> + </PropertyGroup> + <Import Project="..\wireguard-nt.props" /> + <PropertyGroup> + <TargetName>wireguard</TargetName> + </PropertyGroup> + <ItemDefinitionGroup> + <ClCompile> + <PreprocessorDefinitions>NDIS_MINIPORT_DRIVER=1;NDIS620_MINIPORT=1;NDIS683_MINIPORT=1;NDIS_WDM=1;POOL_ZERO_DOWN_LEVEL_SUPPORT;POOL_NX_OPTIN=1;_NO_CRT_STDIO_INLINE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions Condition="'$(SDVHacks)'=='true'">SDV_HACKS;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <AdditionalOptions>/volatile:iso %(AdditionalOptions)</AdditionalOptions> + <DisableSpecificWarnings>4100;4200;4201;$(DisableSpecificWarnings)</DisableSpecificWarnings> + </ClCompile> + <Link> + <AdditionalDependencies>ndis.lib;netio.lib;ntstrsafe.lib;uuid.lib;wdmsec.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(SDVHacks)'!='true'">cng.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(TargetVersion)'=='Windows7' OR '$(TargetVersion)'=='Windows8'">fltmgr.lib;%(AdditionalDependencies)</AdditionalDependencies> + </Link> + </ItemDefinitionGroup> + <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'"> + <Inf> + <TimeStamp>$(WireGuardVersion)</TimeStamp> + </Inf> + </ItemDefinitionGroup> + <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'"> + <Inf> + <TimeStamp>*</TimeStamp> + </Inf> + </ItemDefinitionGroup> + <ItemGroup> + <ClCompile Include="allowedips.c" /> + <ClCompile Include="cookie.c" /> + <ClCompile Include="crypto.c" /> + <ClCompile Include="device.c" /> + <ClCompile Include="ioctl.c" /> + <ClCompile Include="logging.c" /> + <ClCompile Include="main.c" /> + <ClCompile Include="memory.c" /> + <ClCompile Include="noise.c" /> + <ClCompile Include="peer.c" /> + <ClCompile Include="peerlookup.c" /> + <ClCompile Include="queueing.c" /> + <ClCompile Include="ratelimiter.c" /> + <ClCompile Include="rcu.c" /> + <ClCompile Include="receive.c" /> + <ClCompile Include="selftest\allowedips.c"> + <ExcludedFromBuild>true</ExcludedFromBuild> + </ClCompile> + <ClCompile Include="selftest\counter.c"> + <ExcludedFromBuild>true</ExcludedFromBuild> + </ClCompile> + <ClCompile Include="selftest\chacha20poly1305.c"> + <ExcludedFromBuild>true</ExcludedFromBuild> + </ClCompile> + <ClCompile Include="selftest\ratelimiter.c"> + <ExcludedFromBuild>true</ExcludedFromBuild> + </ClCompile> + <ClCompile Include="send.c" /> + <ClCompile Include="socket.c" /> + <ClCompile Include="timers.c" /> + </ItemGroup> + <ItemGroup> + <ResourceCompile Include="wireguard.rc" /> + </ItemGroup> + <ItemGroup> + <Inf Include="wireguard.inf" /> + <FilesToPackage Include="$(TargetPath)" /> + </ItemGroup> + <ItemGroup> + <ClInclude Include="allowedips.h" /> + <ClInclude Include="arithmetic.h" /> + <ClInclude Include="interlocked.h" /> + <ClInclude Include="containers.h" /> + <ClInclude Include="cookie.h" /> + <ClInclude Include="crypto.h" /> + <ClInclude Include="device.h" /> + <ClInclude Include="ioctl.h" /> + <ClInclude Include="logging.h" /> + <ClInclude Include="memory.h" /> + <ClInclude Include="messages.h" /> + <ClInclude Include="noise.h" /> + <ClInclude Include="peer.h" /> + <ClInclude Include="peerlookup.h" /> + <ClInclude Include="queueing.h" /> + <ClInclude Include="ratelimiter.h" /> + <ClInclude Include="rcu.h" /> + <ClInclude Include="socket.h" /> + <ClInclude Include="timers.h" /> + <ClInclude Include="undocumented.h" /> + </ItemGroup> + <ItemGroup> + <MASM Include="crypto-amd64.asm"> + <ExcludedFromBuild Condition="'$(Platform)'!='x64'">true</ExcludedFromBuild> + </MASM> + </ItemGroup> + <Import Project="..\wireguard-nt.props.user" Condition="exists('..\wireguard-nt.props.user')" /> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> + <ImportGroup Label="ExtensionTargets" /> +</Project> diff --git a/driver/driver.vcxproj.filters b/driver/driver.vcxproj.filters new file mode 100644 index 0000000..4abf03e --- /dev/null +++ b/driver/driver.vcxproj.filters @@ -0,0 +1,165 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup> + <Filter Include="Source Files"> + <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier> + <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions> + </Filter> + <Filter Include="Header Files"> + <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier> + <Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions> + </Filter> + <Filter Include="Resource Files"> + <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier> + <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav</Extensions> + </Filter> + <Filter Include="Source Files\selftest"> + <UniqueIdentifier>{e9eb2c98-a43d-4ec7-b155-5e0ad977bc4a}</UniqueIdentifier> + </Filter> + </ItemGroup> + <ItemGroup> + <ClCompile Include="memory.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="rcu.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="crypto.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="allowedips.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="selftest\allowedips.c"> + <Filter>Source Files\selftest</Filter> + </ClCompile> + <ClCompile Include="peer.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="timers.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="cookie.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="ratelimiter.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="selftest\ratelimiter.c"> + <Filter>Source Files\selftest</Filter> + </ClCompile> + <ClCompile Include="peerlookup.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="noise.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="queueing.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="ioctl.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="send.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="receive.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="selftest\counter.c"> + <Filter>Source Files\selftest</Filter> + </ClCompile> + <ClCompile Include="socket.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="device.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="main.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="logging.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="selftest\chacha20poly1305.c"> + <Filter>Source Files\selftest</Filter> + </ClCompile> + </ItemGroup> + <ItemGroup> + <ResourceCompile Include="wireguard.rc"> + <Filter>Resource Files</Filter> + </ResourceCompile> + </ItemGroup> + <ItemGroup> + <Inf Include="wireguard.inf"> + <Filter>Source Files</Filter> + </Inf> + </ItemGroup> + <ItemGroup> + <ClInclude Include="undocumented.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="memory.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="rcu.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="crypto.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="allowedips.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="cookie.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="device.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="messages.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="noise.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="peer.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="peerlookup.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="queueing.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="ratelimiter.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="socket.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="timers.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="ioctl.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="interlocked.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="containers.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="arithmetic.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="logging.h"> + <Filter>Header Files</Filter> + </ClInclude> + </ItemGroup> + <ItemGroup> + <MASM Include="crypto-amd64.asm"> + <Filter>Source Files</Filter> + </MASM> + </ItemGroup> +</Project>
\ No newline at end of file diff --git a/driver/interlocked.h b/driver/interlocked.h new file mode 100644 index 0000000..a8f1be5 --- /dev/null +++ b/driver/interlocked.h @@ -0,0 +1,199 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +#include <ntifs.h> /* Must be included before <wdm.h> */ +#include <wdm.h> +#include <fltkernel.h> + +#pragma warning(suppress : 28194) /* `Value` is aliased in WritePointerNoFence. */ +static inline VOID +__WritePointerNoFence(_Out_ _Interlocked_operand_ PVOID volatile *Destination, _In_opt_ __drv_aliasesMem PVOID Value) +{ + _Analysis_assume_(Value); /* The _In_ should be an _In_opt_. */ + WritePointerNoFence(Destination, Value); +} +/* Suppresses warning about strict type matches, which don't quite make sense in this context. */ +#define WritePointerNoFence(P, V) __WritePointerNoFence((PVOID *)(P), V) + +#pragma warning(suppress : 28194) /* `Value` is aliased in WritePointerRelease. */ +static inline VOID +__WritePointerRelease(_Out_ _Interlocked_operand_ PVOID volatile *Destination, _In_opt_ __drv_aliasesMem PVOID Value) +{ + _Analysis_assume_(Value); /* The _In_ should be an _In_opt_. */ + WritePointerRelease(Destination, Value); +} +/* Suppresses warning about strict type matches, which don't quite make sense in this context. */ +#define WritePointerRelease(P, V) __WritePointerRelease((PVOID *)(P), V) + +static inline VOID WriteMemoryBarrier(VOID) +{ +#if defined(_ARM64_) + __dmb(_ARM64_BARRIER_ISHST); +#elif defined(_ARM_) + __dmb(_ARM_BARRIER_ISHST); +#elif defined(_AMD64_) || defined(_X86_) + /* Strong ordering on Intel */ +#else +# error "Unknown arch. Consult smp_wmb." +#endif + _ReadWriteBarrier(); +} + +#ifdef _WIN64 +# define InterlockedBitTestAndSetPtr(Addr, Nr) InterlockedBitTestAndSet64(Addr, Nr) +#else +# define InterlockedBitTestAndSetPtr(Addr, Nr) InterlockedBitTestAndSet(Addr, Nr) +#endif + +#ifndef InterlockedExchangePointerRelease +_Ret_writes_(_Inexpressible_(Unknown)) static inline PVOID InterlockedExchangePointerRelease( + _Inout_ _At_( + *Target, + _Pre_writable_byte_size_(_Inexpressible_(Unknown)) _Post_writable_byte_size_(_Inexpressible_(Unknown))) + _Interlocked_operand_ PVOID volatile *Target, + _In_opt_ PVOID Value) +{ +# if defined(_ARM64_) + __dmb(_ARM64_BARRIER_ISH); +# elif defined(_ARM_) + __dmb(_ARM_BARRIER_ISH); +# elif defined(_AMD64_) || defined(_X86_) + /* Atomic instructions are already serializing on Intel */ +# else +# error "Unknown arch. Consult __atomic_release_fence/smp_mb__before_atomic." +# endif + return InterlockedExchangePointer(Target, Value); +} +#endif + +_Must_inspect_result_ +static inline BOOLEAN +InterlockedIncrementUnless(_Inout_ _Interlocked_operand_ LONG volatile *Destination, _In_ LONG Unless) +{ + for (LONG C = ReadNoFence(Destination), X;; C = X) + { + if (C == Unless) + return FALSE; + X = InterlockedCompareExchange(Destination, C + 1, C); + if (X == C) + return TRUE; + } +} + +_Must_inspect_result_ +static inline BOOLEAN +InterlockedIncrementUnless64(_Inout_ _Interlocked_operand_ LONG64 volatile *Destination, _In_ LONG64 Unless) +{ + for (LONG64 C = ReadNoFence64(Destination), X;; C = X) + { + if (C == Unless) + return FALSE; + X = InterlockedCompareExchange64(Destination, C + 1, C); + if (X == C) + return TRUE; + } +} + +typedef LONG64 KREF; + +static inline VOID +KrefInit(_Out_ KREF *Kref) +{ + WriteRaw64(Kref, 1); +} + +static inline VOID +KrefGet(_Inout_ KREF *Kref) +{ + InterlockedIncrement64(Kref); +} + +_Must_inspect_result_ +static inline BOOLEAN +KrefGetUnlessZero(_Inout_ KREF *Kref) +{ + return InterlockedIncrementUnless64(Kref, 0); +} + +static inline BOOLEAN +KrefPut(_Inout_ KREF *Kref, _In_ VOID (*Release)(_In_ KREF *Kref)) +{ + if (!InterlockedDecrement64(Kref)) + { + Release(Kref); + return TRUE; + } + return FALSE; +} + +_IRQL_requires_max_(APC_LEVEL) +static inline VOID +MuInitializePushLock(_Out_ PEX_PUSH_LOCK PushLock) +{ +#ifdef EX_LEGACY_PUSH_LOCKS + FltInitializePushLock(PushLock); +#else + ExInitializePushLock(PushLock); +#endif +} + +_Acquires_lock_(_Global_critical_region_) +_IRQL_requires_max_(APC_LEVEL) +static inline VOID +MuAcquirePushLockExclusive(_Inout_ _Requires_lock_not_held_(*_Curr_) _Acquires_exclusive_lock_(*_Curr_) + PEX_PUSH_LOCK PushLock) +{ +#ifdef EX_LEGACY_PUSH_LOCKS + FltAcquirePushLockExclusive(PushLock); +#else + KeEnterCriticalRegion(); + ExAcquirePushLockExclusive(PushLock); +#endif +} + +_Acquires_lock_(_Global_critical_region_) +_IRQL_requires_max_(APC_LEVEL) +static inline VOID +MuAcquirePushLockShared(_Inout_ _Requires_lock_not_held_(*_Curr_) _Acquires_shared_lock_(*_Curr_) + PEX_PUSH_LOCK PushLock) +{ +#ifdef EX_LEGACY_PUSH_LOCKS + FltAcquirePushLockShared(PushLock); +#else + KeEnterCriticalRegion(); + ExAcquirePushLockShared(PushLock); +#endif +} + +_Releases_lock_(_Global_critical_region_) +_IRQL_requires_max_(APC_LEVEL) +static inline VOID +MuReleasePushLockExclusive(_Inout_ _Requires_exclusive_lock_held_(*_Curr_) _Releases_exclusive_lock_(*_Curr_) + PEX_PUSH_LOCK PushLock) +{ +#ifdef EX_LEGACY_PUSH_LOCKS + FltReleasePushLock(PushLock); +#else + ExReleasePushLockExclusive(PushLock); + KeLeaveCriticalRegion(); +#endif +} + +_Releases_lock_(_Global_critical_region_) +_IRQL_requires_max_(APC_LEVEL) +static inline VOID +MuReleasePushLockShared(_Inout_ _Requires_shared_lock_held_(*_Curr_) _Releases_shared_lock_(*_Curr_) + PEX_PUSH_LOCK PushLock) +{ + _Analysis_suppress_lock_checking_(PushLock); +#ifdef EX_LEGACY_PUSH_LOCKS + FltReleasePushLock(PushLock); +#else + ExReleasePushLockShared(PushLock); + KeLeaveCriticalRegion(); +#endif +} diff --git a/driver/ioctl.c b/driver/ioctl.c new file mode 100644 index 0000000..46164d0 --- /dev/null +++ b/driver/ioctl.c @@ -0,0 +1,760 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include "device.h" +#include "ioctl.h" +#include "peer.h" +#include "queueing.h" +#include "socket.h" +#include "timers.h" +#include "logging.h" +#include <ntddk.h> + +#define SIZE_OF_EMBEDDED(A, B) \ + FIELD_OFFSET( \ + struct { \ + A _A; \ + B _B; \ + }, \ + _B) + +static_assert( + SIZE_OF_EMBEDDED(WG_IOCTL_INTERFACE, WG_IOCTL_PEER) == (sizeof(WG_IOCTL_INTERFACE)), + "Non-symmetric interface stacking"); +static_assert( + SIZE_OF_EMBEDDED(WG_IOCTL_PEER, WG_IOCTL_ALLOWED_IP) == SIZE_OF_EMBEDDED(WG_IOCTL_PEER, WG_IOCTL_INTERFACE), + "Non-symmetric peer stacking"); +static_assert( + SIZE_OF_EMBEDDED(WG_IOCTL_PEER, WG_IOCTL_ALLOWED_IP) == (sizeof(WG_IOCTL_PEER)), + "Non-symmetric peer stacking"); +static_assert(SIZE_OF_EMBEDDED(WG_IOCTL_PEER, WG_IOCTL_PEER) == (sizeof(WG_IOCTL_PEER)), "Non-symmetric peer stacking"); +static_assert( + SIZE_OF_EMBEDDED(WG_IOCTL_ALLOWED_IP, WG_IOCTL_ALLOWED_IP) == SIZE_OF_EMBEDDED(WG_IOCTL_ALLOWED_IP, WG_IOCTL_PEER), + "Non-symmetric allowed IP stacking"); +static_assert( + SIZE_OF_EMBEDDED(WG_IOCTL_ALLOWED_IP, WG_IOCTL_PEER) == (sizeof(WG_IOCTL_ALLOWED_IP)), + "Non-symmetric allowed IP stacking"); +static_assert( + SIZE_OF_EMBEDDED(WG_IOCTL_ALLOWED_IP, WG_IOCTL_ALLOWED_IP) == (sizeof(WG_IOCTL_ALLOWED_IP)), + "Non-symmetric allowed IP stacking"); + +#undef SIZE_OF_EMBEDDED + +#pragma warning(disable : 28175) /* undocumented: the member of struct should not be accessed by a driver */ + +static DRIVER_DISPATCH *NdisDispatchDeviceControl, *NdisDispatchCreate, *NdisDispatchPnp; +/* The following binary blob security descriptor was generated via: + * PSECURITY_DESCRIPTOR Sd; + * ULONG SdLen; + * ConvertStringSecurityDescriptorToSecurityDescriptorA("O:SYD:P(A;;FA;;;SY)(A;;FA;;;BA)S:(ML;;NWNRNX;;;HI)", + * SDDL_REVISION_1, &Sd, &SdLen); + * for (ULONG i = 0; i < SdLen; ++i) + * printf("0x%02x%s%s", ((UCHAR *)Sd)[i], i == SdLen - 1 ? "" : ",", i == SdLen - 1 || i % 8 == 7 ? "\n": " "); + */ +static SECURITY_DESCRIPTOR *DispatchSecurityDescriptor = (SECURITY_DESCRIPTOR *)(__declspec(align(8)) UCHAR[]){ + 0x01, 0x00, 0x14, 0x90, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, + 0x00, 0x02, 0x00, 0x1c, 0x00, 0x01, 0x00, 0x00, 0x00, 0x11, 0x00, 0x14, 0x00, 0x07, 0x00, 0x00, 0x00, 0x01, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x30, 0x00, 0x00, 0x02, 0x00, 0x34, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x14, 0x00, 0xff, 0x01, 0x1f, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x12, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x18, 0x00, 0xff, 0x01, 0x1f, 0x00, 0x01, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x20, 0x00, 0x00, + 0x00, 0x20, 0x02, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x12, 0x00, 0x00, 0x00 +}; + +_IRQL_requires_max_(PASSIVE_LEVEL) +static BOOLEAN +HasAccess(_In_ ACCESS_MASK DesiredAccess, _In_ KPROCESSOR_MODE AccessMode, _Out_ NTSTATUS *Status) +{ + SECURITY_SUBJECT_CONTEXT SubjectContext; + SeCaptureSubjectContext(&SubjectContext); + ACCESS_MASK GrantedAccess; + BOOLEAN HasAccess = SeAccessCheck( + DispatchSecurityDescriptor, + &SubjectContext, + FALSE, + DesiredAccess, + 0, + NULL, + IoGetFileObjectGenericMapping(), + AccessMode, + &GrantedAccess, + Status); + SeReleaseSubjectContext(&SubjectContext); + return HasAccess; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +static VOID +Get(_In_ DEVICE_OBJECT *DeviceObject, _Inout_ IRP *Irp) +{ + Irp->IoStatus.Information = 0; + if (!HasAccess(FILE_READ_DATA, Irp->RequestorMode, &Irp->IoStatus.Status)) + return; + + WG_IOCTL_INTERFACE *IoctlInterface = NULL; + if (Irp->MdlAddress) + { + IoctlInterface = MmGetSystemAddressForMdlSafe(Irp->MdlAddress, NormalPagePriority | MdlMappingNoExecute); + if (!IoctlInterface) + { + Irp->IoStatus.Status = STATUS_INSUFFICIENT_RESOURCES; + return; + } + } + + WG_DEVICE *Wg = DeviceObject->Reserved; + if (!Wg || ReadBooleanNoFence(&Wg->IsDeviceRemoving)) + { + Irp->IoStatus.Status = NDIS_STATUS_ADAPTER_REMOVED; + return; + } + + MuAcquirePushLockExclusive(&Wg->DeviceUpdateLock); + + ULONG OutSize = IoctlInterface ? MmGetMdlByteCount(Irp->MdlAddress) : 0; + ULONG64 FinalSize = sizeof(WG_IOCTL_INTERFACE); + if (OutSize >= FinalSize) + { + IoctlInterface->Flags = 0; + IoctlInterface->PeersCount = 0; + if (Wg->IncomingPort != 0) + { + IoctlInterface->ListenPort = Wg->IncomingPort; + IoctlInterface->Flags |= WG_IOCTL_INTERFACE_HAS_LISTEN_PORT; + } + MuAcquirePushLockShared(&Wg->StaticIdentity.Lock); + if (Wg->StaticIdentity.HasIdentity) + { + RtlCopyMemory(IoctlInterface->PrivateKey, Wg->StaticIdentity.StaticPrivate, NOISE_PUBLIC_KEY_LEN); + RtlCopyMemory(IoctlInterface->PublicKey, Wg->StaticIdentity.StaticPublic, NOISE_PUBLIC_KEY_LEN); + IoctlInterface->Flags |= WG_IOCTL_INTERFACE_HAS_PUBLIC_KEY | WG_IOCTL_INTERFACE_HAS_PRIVATE_KEY; + } + MuReleasePushLockShared(&Wg->StaticIdentity.Lock); + } + + WG_IOCTL_PEER *IoctlPeer = (WG_IOCTL_PEER *)((UCHAR *)IoctlInterface + sizeof(WG_IOCTL_INTERFACE)); + WG_PEER *Peer; + LIST_FOR_EACH_ENTRY (Peer, &Wg->PeerList, WG_PEER, PeerList) + { + FinalSize += sizeof(WG_IOCTL_PEER); + if (OutSize >= FinalSize) + { + ++IoctlInterface->PeersCount; + IoctlPeer->Flags = WG_IOCTL_PEER_HAS_PERSISTENT_KEEPALIVE; + IoctlPeer->ProtocolVersion = 1; + IoctlPeer->PersistentKeepalive = Peer->PersistentKeepaliveInterval; + IoctlPeer->RxBytes = Peer->RxBytes; + IoctlPeer->TxBytes = Peer->TxBytes; + IoctlPeer->LastHandshake = Peer->WalltimeLastHandshake.QuadPart; + IoctlPeer->AllowedIPsCount = 0; + MuAcquirePushLockShared(&Peer->Handshake.Lock); + RtlCopyMemory(IoctlPeer->PublicKey, Peer->Handshake.RemoteStatic, NOISE_PUBLIC_KEY_LEN); + RtlCopyMemory(IoctlPeer->PresharedKey, Peer->Handshake.PresharedKey, NOISE_SYMMETRIC_KEY_LEN); + IoctlPeer->Flags |= WG_IOCTL_PEER_HAS_PUBLIC_KEY | WG_IOCTL_PEER_HAS_PRESHARED_KEY; + MuReleasePushLockShared(&Peer->Handshake.Lock); + KIRQL Irql; + Irql = ExAcquireSpinLockShared(&Peer->EndpointLock); + if (Peer->Endpoint.Addr.si_family == AF_INET) + IoctlPeer->Endpoint.Ipv4 = Peer->Endpoint.Addr.Ipv4; + else if (Peer->Endpoint.Addr.si_family == AF_INET6) + IoctlPeer->Endpoint.Ipv6 = Peer->Endpoint.Addr.Ipv6; + IoctlPeer->Flags |= WG_IOCTL_PEER_HAS_ENDPOINT; + ExReleaseSpinLockShared(&Peer->EndpointLock, Irql); + } + + WG_IOCTL_ALLOWED_IP *IoctlAllowedIp = (WG_IOCTL_ALLOWED_IP *)((UCHAR *)IoctlPeer + sizeof(WG_IOCTL_PEER)); + ALLOWEDIPS_NODE *AllowedIpsNode; + ULONG AllowedIpsLimit = MAXULONG; + LIST_FOR_EACH_ENTRY (AllowedIpsNode, &Peer->AllowedIpsList, ALLOWEDIPS_NODE, PeerList) + { + if (!(--AllowedIpsLimit)) + break; + FinalSize += sizeof(WG_IOCTL_ALLOWED_IP); + if (OutSize >= FinalSize) + { + ++IoctlPeer->AllowedIPsCount; + IoctlAllowedIp->AddressFamily = + AllowedIpsReadNode(AllowedIpsNode, (UINT8 *)&IoctlAllowedIp->Address, &IoctlAllowedIp->Cidr); + } + ++IoctlAllowedIp; + } + IoctlPeer = (WG_IOCTL_PEER *)IoctlAllowedIp; + } + MuReleasePushLockExclusive(&Wg->DeviceUpdateLock); + + Irp->IoStatus.Status = OutSize >= FinalSize ? STATUS_SUCCESS + : FinalSize <= MAXULONG ? STATUS_BUFFER_OVERFLOW + : STATUS_SECTION_TOO_BIG; + Irp->IoStatus.Information = (ULONG_PTR)FinalSize; +} + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_held_(Wg->DeviceUpdateLock) +_Must_inspect_result_ +static NTSTATUS +SetListenPort(_Inout_ WG_DEVICE *Wg, _In_ USHORT ListenPort) +{ + if (Wg->IncomingPort == ListenPort) + return STATUS_SUCCESS; + WG_PEER *Peer; + LIST_FOR_EACH_ENTRY (Peer, &Wg->PeerList, WG_PEER, PeerList) + SocketClearPeerEndpointSrc(Peer); + if (ReadBooleanNoFence(&Wg->IsUp)) + return SocketInit(Wg, ListenPort); + Wg->IncomingPort = ListenPort; + return STATUS_SUCCESS; +} + +_Requires_lock_held_(Wg->DeviceUpdateLock) +_Must_inspect_result_ +static NTSTATUS +SetPrivateKey(_Inout_ WG_DEVICE *Wg, _In_ CONST UCHAR PrivateKey[WG_KEY_LEN]) +{ + UINT8 PublicKey[NOISE_PUBLIC_KEY_LEN]; + WG_PEER *Peer, *Temp; + if (CryptoEqualMemory32(Wg->StaticIdentity.StaticPrivate, PrivateKey)) + return STATUS_SUCCESS; + + /* We remove before setting, to prevent race, which means doing two 25519-genpub ops. */ + if (Curve25519GeneratePublic(PublicKey, PrivateKey)) + { + Peer = PubkeyHashtableLookup(Wg->PeerHashtable, PublicKey); + if (Peer) + { + PeerPut(Peer); + _Analysis_assume_same_lock_(Peer->Device->DeviceUpdateLock, Wg->DeviceUpdateLock); + PeerRemove(Peer); + } + } + + MuAcquirePushLockExclusive(&Wg->StaticIdentity.Lock); + NoiseSetStaticIdentityPrivateKey(&Wg->StaticIdentity, PrivateKey); + LIST_FOR_EACH_ENTRY_SAFE (Peer, Temp, &Wg->PeerList, WG_PEER, PeerList) + { + _Analysis_assume_same_lock_(Peer->Device->DeviceUpdateLock, Wg->DeviceUpdateLock); + NoisePrecomputeStaticStatic(Peer); + NoiseExpireCurrentPeerKeypairs(Peer); + } + _Analysis_assume_same_lock_(Wg->CookieChecker.Device->DeviceUpdateLock, Wg->DeviceUpdateLock); + CookieCheckerPrecomputeDeviceKeys(&Wg->CookieChecker); + MuReleasePushLockExclusive(&Wg->StaticIdentity.Lock); + return STATUS_SUCCESS; +} + +_Requires_lock_held_(Wg->DeviceUpdateLock) +_Must_inspect_result_ +static NTSTATUS +SetPeer(_Inout_ WG_DEVICE *Wg, _Inout_ CONST volatile WG_IOCTL_PEER **UnsafeIoctlPeerPtr, _Inout_ ULONG *RemainingSize) +{ + if (*RemainingSize < sizeof(WG_IOCTL_PEER)) + return STATUS_INVALID_PARAMETER; + *RemainingSize -= sizeof(WG_IOCTL_PEER); + + CONST volatile WG_IOCTL_PEER *UnsafeIoctlPeer = *UnsafeIoctlPeerPtr; + WG_IOCTL_PEER IoctlPeer = *UnsafeIoctlPeer; + + NTSTATUS Status = STATUS_INVALID_PARAMETER; + if (!(IoctlPeer.Flags & WG_IOCTL_PEER_HAS_PUBLIC_KEY)) + goto cleanupStack; + Status = STATUS_NOT_IMPLEMENTED; + if ((IoctlPeer.Flags & WG_IOCTL_PEER_HAS_PROTOCOL_VERSION) && IoctlPeer.ProtocolVersion != 0 && + IoctlPeer.ProtocolVersion > 1) + goto cleanupStack; + Status = STATUS_INVALID_PARAMETER; + ULONG AllowedIPsSize; + if (!NT_SUCCESS(RtlULongMult(IoctlPeer.AllowedIPsCount, sizeof(WG_IOCTL_ALLOWED_IP), &AllowedIPsSize))) + goto cleanupStack; + if (*RemainingSize < AllowedIPsSize) + goto cleanupStack; + *RemainingSize -= AllowedIPsSize; + CONST WG_IOCTL_ALLOWED_IP *UnsafeIoctlAllowedIp = + (CONST WG_IOCTL_ALLOWED_IP *)((UCHAR *)UnsafeIoctlPeer + sizeof(WG_IOCTL_PEER)); + *UnsafeIoctlPeerPtr = (CONST WG_IOCTL_PEER *)((UCHAR *)UnsafeIoctlAllowedIp + AllowedIPsSize); + + Status = STATUS_SUCCESS; + WG_PEER *Peer = PubkeyHashtableLookup(Wg->PeerHashtable, IoctlPeer.PublicKey); + if (!Peer) + { + /* Peer doesn't exist yet. Add a new one. */ + if (IoctlPeer.Flags & (WG_IOCTL_PEER_REMOVE | WG_IOCTL_PEER_UPDATE)) + goto cleanupStack; + + /* The peer is new, so there aren't allowed IPs to remove. */ + IoctlPeer.Flags &= ~WG_IOCTL_PEER_REPLACE_ALLOWED_IPS; + + MuAcquirePushLockShared(&Wg->StaticIdentity.Lock); + if (Wg->StaticIdentity.HasIdentity && + RtlEqualMemory(IoctlPeer.PublicKey, Wg->StaticIdentity.StaticPublic, NOISE_PUBLIC_KEY_LEN)) + { + /* We silently ignore peers that have the same public key as the device. The reason we do it silently is + * that we'd like for people to be able to reuse the same set of API calls across peers. */ + MuReleasePushLockShared(&Wg->StaticIdentity.Lock); + goto cleanupStack; + } + MuReleasePushLockShared(&Wg->StaticIdentity.Lock); + + Status = PeerCreate( + Wg, + IoctlPeer.PublicKey, + (IoctlPeer.Flags & WG_IOCTL_PEER_HAS_PRESHARED_KEY) ? IoctlPeer.PresharedKey : NULL, + &Peer); + if (!NT_SUCCESS(Status)) + goto cleanupStack; + /* Take additional reference, as though we've just been looked up. */ + PeerGet(Peer); + } + + if (IoctlPeer.Flags & WG_IOCTL_PEER_REMOVE) + { + _Analysis_assume_same_lock_(Peer->Device->DeviceUpdateLock, Wg->DeviceUpdateLock); + PeerRemove(Peer); + Status = STATUS_SUCCESS; + goto cleanupPeer; + } + + if (IoctlPeer.Flags & WG_IOCTL_PEER_HAS_PRESHARED_KEY) + { + MuAcquirePushLockExclusive(&Peer->Handshake.Lock); + RtlCopyMemory(&Peer->Handshake.PresharedKey, IoctlPeer.PresharedKey, NOISE_SYMMETRIC_KEY_LEN); + MuReleasePushLockExclusive(&Peer->Handshake.Lock); + } + + if (IoctlPeer.Flags & WG_IOCTL_PEER_HAS_ENDPOINT) + { + SIZE_T Size; + if ((Size = sizeof(SOCKADDR_IN), IoctlPeer.Endpoint.si_family == AF_INET) || + (Size = sizeof(SOCKADDR_IN6), IoctlPeer.Endpoint.si_family == AF_INET6)) + { + ENDPOINT Endpoint = { { { 0 } } }; + RtlCopyMemory(&Endpoint.Addr, &IoctlPeer.Endpoint, Size); + SocketSetPeerEndpoint(Peer, &Endpoint); + } + } + + if (IoctlPeer.Flags & WG_IOCTL_PEER_REPLACE_ALLOWED_IPS) + AllowedIpsRemoveByPeer(&Wg->PeerAllowedIps, Peer, &Wg->DeviceUpdateLock); + + for (ULONG i = 0; i < IoctlPeer.AllowedIPsCount; ++i) + { + WG_IOCTL_ALLOWED_IP IoctlAllowedIp = UnsafeIoctlAllowedIp[i]; + if (IoctlAllowedIp.AddressFamily == AF_INET && IoctlAllowedIp.Cidr <= 32) + Status = AllowedIpsInsertV4( + &Peer->Device->PeerAllowedIps, + &IoctlAllowedIp.Address.V4, + IoctlAllowedIp.Cidr, + Peer, + &Wg->DeviceUpdateLock); + else if (IoctlAllowedIp.AddressFamily == AF_INET6 && IoctlAllowedIp.Cidr <= 128) + Status = AllowedIpsInsertV6( + &Peer->Device->PeerAllowedIps, + &IoctlAllowedIp.Address.V6, + IoctlAllowedIp.Cidr, + Peer, + &Wg->DeviceUpdateLock); + else + Status = STATUS_INVALID_PARAMETER; + if (!NT_SUCCESS(Status)) + goto cleanupPeer; + } + + BOOLEAN IsUp = ReadBooleanNoFence(&Wg->IsUp); + if (IoctlPeer.Flags & WG_IOCTL_PEER_HAS_PERSISTENT_KEEPALIVE) + { + CONST BOOLEAN SendKeepalive = !Peer->PersistentKeepaliveInterval && IoctlPeer.PersistentKeepalive && IsUp; + Peer->PersistentKeepaliveInterval = IoctlPeer.PersistentKeepalive; + if (SendKeepalive) + PacketSendKeepalive(Peer); + } + + if (IsUp) + PacketSendStagedPackets(Peer); + + Status = STATUS_SUCCESS; +cleanupPeer: + PeerPut(Peer); +cleanupStack: + RtlSecureZeroMemory(&IoctlPeer, sizeof(IoctlPeer)); + return Status; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +static NTSTATUS +SetInterface( + _Inout_ WG_DEVICE *Wg, + _In_ CONST volatile WG_IOCTL_INTERFACE *UnsafeIoctlInterface, + _Inout_ ULONG *RemainingSize) +{ + if (*RemainingSize < sizeof(WG_IOCTL_INTERFACE)) + return STATUS_INVALID_PARAMETER; + *RemainingSize = *RemainingSize - sizeof(WG_IOCTL_INTERFACE); + + WG_IOCTL_INTERFACE IoctlInterface = *UnsafeIoctlInterface; + + MuAcquirePushLockExclusive(&Wg->DeviceUpdateLock); + + ++Wg->DeviceUpdateGen; + + NTSTATUS Status; + if (IoctlInterface.Flags & WG_IOCTL_INTERFACE_HAS_LISTEN_PORT) + { + Status = SetListenPort(Wg, IoctlInterface.ListenPort); + if (!NT_SUCCESS(Status)) + goto cleanupLock; + } + + if (IoctlInterface.Flags & WG_IOCTL_INTERFACE_REPLACE_PEERS) + PeerRemoveAll(Wg); + + if (IoctlInterface.Flags & WG_IOCTL_INTERFACE_HAS_PRIVATE_KEY) + { + Status = SetPrivateKey(Wg, IoctlInterface.PrivateKey); + if (!NT_SUCCESS(Status)) + goto cleanupLock; + } + + CONST volatile WG_IOCTL_PEER *UnsafeIoctlPeer = + (CONST volatile WG_IOCTL_PEER *)((UCHAR *)UnsafeIoctlInterface + sizeof(WG_IOCTL_INTERFACE)); + for (ULONG i = 0; i < IoctlInterface.PeersCount; ++i) + { + Status = SetPeer(Wg, &UnsafeIoctlPeer, RemainingSize); + if (!NT_SUCCESS(Status)) + goto cleanupLock; + } + + Status = STATUS_SUCCESS; +cleanupLock: + MuReleasePushLockExclusive(&Wg->DeviceUpdateLock); + RtlSecureZeroMemory(&IoctlInterface, sizeof(IoctlInterface)); + return Status; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +static VOID +Set(_In_ DEVICE_OBJECT *DeviceObject, _Inout_ IRP *Irp) +{ + Irp->IoStatus.Information = 0; + if (!HasAccess(FILE_WRITE_DATA, Irp->RequestorMode, &Irp->IoStatus.Status)) + return; + + if (!Irp->MdlAddress) + { + Irp->IoStatus.Status = STATUS_INVALID_PARAMETER; + return; + } + CONST volatile WG_IOCTL_INTERFACE *UnsafeIoctlInterface = + MmGetSystemAddressForMdlSafe(Irp->MdlAddress, NormalPagePriority | MdlMappingNoExecute | MdlMappingNoWrite); + if (!UnsafeIoctlInterface) + { + Irp->IoStatus.Status = STATUS_INSUFFICIENT_RESOURCES; + return; + } + + WG_DEVICE *Wg = DeviceObject->Reserved; + if (!Wg || ReadBooleanNoFence(&Wg->IsDeviceRemoving)) + { + Irp->IoStatus.Status = NDIS_STATUS_ADAPTER_REMOVED; + return; + } + + ULONG InSize = MmGetMdlByteCount(Irp->MdlAddress); + ULONG RemainingSize = InSize; + Irp->IoStatus.Status = SetInterface(Wg, UnsafeIoctlInterface, &RemainingSize); + Irp->IoStatus.Information = InSize - RemainingSize; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +_Requires_lock_held_(&Wg->DeviceUpdateLock) +static NTSTATUS +Up(_Inout_ WG_DEVICE *Wg) +{ + if (ReadBooleanNoFence(&Wg->IsUp)) + return STATUS_ALREADY_COMPLETE; + NT_ASSERT(!Wg->SocketOwnerProcess); + Wg->SocketOwnerProcess = PsGetCurrentProcess(); + if (Wg->SocketOwnerProcess) + ObReferenceObject(Wg->SocketOwnerProcess); + NTSTATUS Status = SocketInit(Wg, Wg->IncomingPort); + if (!NT_SUCCESS(Status)) + return Status; + WriteBooleanNoFence(&Wg->IsUp, TRUE); + DeviceStart(Wg); + DeviceIndicateConnectionStatus(Wg->MiniportAdapterHandle, MediaConnectStateConnected); + LogInfo(Wg, "Interface up"); + return STATUS_SUCCESS; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +_Requires_lock_held_(&Wg->DeviceUpdateLock) +static NTSTATUS +Down(_Inout_ WG_DEVICE *Wg) +{ + if (!ReadBooleanNoFence(&Wg->IsUp)) + return STATUS_ALREADY_COMPLETE; + WriteBooleanNoFence(&Wg->IsUp, FALSE); + RcuSynchronize(); + DeviceIndicateConnectionStatus(Wg->MiniportAdapterHandle, MediaConnectStateDisconnected); + DeviceStop(Wg); + SocketReinit(Wg, NULL, NULL, 0); + if (Wg->SocketOwnerProcess) + { + ObDereferenceObject(Wg->SocketOwnerProcess); + Wg->SocketOwnerProcess = NULL; + } + LogInfo(Wg, "Interface down"); + return STATUS_SUCCESS; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +static VOID +AdapterState(_In_ DEVICE_OBJECT *DeviceObject, _Inout_ IRP *Irp) +{ + Irp->IoStatus.Information = 0; + if (!HasAccess(FILE_WRITE_DATA, Irp->RequestorMode, &Irp->IoStatus.Status)) + return; + IO_STACK_LOCATION *Stack = IoGetCurrentIrpStackLocation(Irp); + if (Stack->Parameters.DeviceIoControl.InputBufferLength != sizeof(WG_IOCTL_ADAPTER_STATE)) + { + Irp->IoStatus.Status = STATUS_INVALID_PARAMETER; + return; + } + WG_IOCTL_ADAPTER_STATE Op; + RtlCopyMemory(&Op, Irp->AssociatedIrp.SystemBuffer, sizeof(Op)); + + WG_DEVICE *Wg = DeviceObject->Reserved; + if (!Wg || ReadBooleanNoFence(&Wg->IsDeviceRemoving)) + { + Irp->IoStatus.Status = NDIS_STATUS_ADAPTER_REMOVED; + return; + } + MuAcquirePushLockExclusive(&Wg->DeviceUpdateLock); + switch (Op) + { + case WG_IOCTL_ADAPTER_STATE_QUERY: + if (Stack->Parameters.DeviceIoControl.OutputBufferLength == sizeof(Op)) + { + Op = ReadBooleanNoFence(&Wg->IsUp) ? WG_IOCTL_ADAPTER_STATE_UP : WG_IOCTL_ADAPTER_STATE_DOWN; + RtlCopyMemory(Irp->AssociatedIrp.SystemBuffer, &Op, sizeof(Op)); + Irp->IoStatus.Information = sizeof(Op); + } + else + Irp->IoStatus.Status = STATUS_INVALID_PARAMETER; + break; + case WG_IOCTL_ADAPTER_STATE_DOWN: + Irp->IoStatus.Status = Down(Wg); + break; + case WG_IOCTL_ADAPTER_STATE_UP: + Irp->IoStatus.Status = Up(Wg); + break; + default: + Irp->IoStatus.Status = STATUS_INVALID_PARAMETER; + } + MuReleasePushLockExclusive(&Wg->DeviceUpdateLock); +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +static VOID +ReadLogLine(_In_ DEVICE_OBJECT *DeviceObject, _Inout_ IRP *Irp) +{ + Irp->IoStatus.Information = 0; + if (!HasAccess(FILE_READ_DATA, Irp->RequestorMode, &Irp->IoStatus.Status)) + return; + + WG_DEVICE *Wg = DeviceObject->Reserved; + if (!Wg || ReadBooleanNoFence(&Wg->IsDeviceRemoving)) + { + Irp->IoStatus.Status = NDIS_STATUS_ADAPTER_REMOVED; + return; + } + + IO_STACK_LOCATION *Stack = IoGetCurrentIrpStackLocation(Irp); + if (Stack->Parameters.DeviceIoControl.OutputBufferLength < WG_MAX_LOG_LINE_LEN) + { + Irp->IoStatus.Status = STATUS_BUFFER_TOO_SMALL; + return; + } + Irp->IoStatus.Status = LogRingRead(&Wg->Log, Irp->AssociatedIrp.SystemBuffer, &Wg->IsDeviceRemoving); + if (NT_SUCCESS(Irp->IoStatus.Status)) + Irp->IoStatus.Information = WG_MAX_LOG_LINE_LEN; +} + +static KSTART_ROUTINE ForceCloseHandlesAfterDelay; +_Use_decl_annotations_ +static VOID +ForceCloseHandlesAfterDelay(PVOID StartContext) +{ + WG_DEVICE *Wg = StartContext; + DEVICE_OBJECT *DeviceObject; + NTSTATUS Status; + PEPROCESS Process; + KAPC_STATE ApcState; + PVOID Object = NULL; + ULONG VerifierFlags = 0; + OBJECT_HANDLE_INFORMATION HandleInfo; + SYSTEM_HANDLE_INFORMATION_EX *HandleTable = NULL; + + if (KeWaitForSingleObject( + &Wg->DeviceRemoved, + Executive, + KernelMode, + FALSE, + &(LARGE_INTEGER){ .QuadPart = -SEC_TO_SYS_TIME_UNITS(5) }) != STATUS_TIMEOUT) + return; + + DeviceObject = Wg->FunctionalDeviceObject; + if (!DeviceObject) + return; + + LogWarn(Wg, "Force closing all adapter handles after having waited 5 seconds"); + + MmIsVerifierEnabled(&VerifierFlags); + + for (ULONG Size = 0, RequestedSize; + (Status = ZwQuerySystemInformation(SystemExtendedHandleInformation, HandleTable, Size, &RequestedSize)) == + STATUS_INFO_LENGTH_MISMATCH; + Size = RequestedSize) + { + if (HandleTable) + MemFree(HandleTable); + HandleTable = MemAllocate(RequestedSize); + if (!HandleTable) + return; + } + if (!NT_SUCCESS(Status) || !HandleTable) + goto cleanup; + + HANDLE CurrentProcessId = PsGetCurrentProcessId(); + for (ULONG_PTR Index = 0; Index < HandleTable->NumberOfHandles; ++Index) + { + FILE_OBJECT *FileObject = HandleTable->Handles[Index].Object; + if (!FileObject || FileObject->Type != 5 || FileObject->DeviceObject != DeviceObject) + continue; + HANDLE ProcessId = HandleTable->Handles[Index].UniqueProcessId; + if (ProcessId == CurrentProcessId) + continue; + Status = PsLookupProcessByProcessId(ProcessId, &Process); + if (!NT_SUCCESS(Status)) + continue; + KeStackAttachProcess(Process, &ApcState); + if (!VerifierFlags) + Status = ObReferenceObjectByHandle( + HandleTable->Handles[Index].HandleValue, 0, NULL, UserMode, &Object, &HandleInfo); + if (NT_SUCCESS(Status)) + { + if (VerifierFlags || Object == FileObject) + ObCloseHandle(HandleTable->Handles[Index].HandleValue, UserMode); + if (!VerifierFlags) + ObfDereferenceObject(Object); + } + KeUnstackDetachProcess(&ApcState); + ObfDereferenceObject(Process); + } +cleanup: + if (HandleTable) + MemFree(HandleTable); +} + +_Dispatch_type_(IRP_MJ_DEVICE_CONTROL) +static DRIVER_DISPATCH_PAGED DispatchDeviceControl; +_Use_decl_annotations_ +static NTSTATUS +DispatchDeviceControl(DEVICE_OBJECT *DeviceObject, IRP *Irp) +{ + IO_STACK_LOCATION *Stack = IoGetCurrentIrpStackLocation(Irp); + switch (Stack->Parameters.DeviceIoControl.IoControlCode) + { + case WG_IOCTL_GET: + Get(DeviceObject, Irp); + break; + case WG_IOCTL_SET: + Set(DeviceObject, Irp); + break; + case WG_IOCTL_SET_ADAPTER_STATE: + AdapterState(DeviceObject, Irp); + break; + case WG_IOCTL_READ_LOG_LINE: + ReadLogLine(DeviceObject, Irp); + break; + default: + return NdisDispatchDeviceControl(DeviceObject, Irp); + } + NTSTATUS Status = Irp->IoStatus.Status; + IoCompleteRequest(Irp, IO_NO_INCREMENT); + return Status; +} + +_Dispatch_type_(IRP_MJ_CREATE) +static DRIVER_DISPATCH_PAGED DispatchCreate; +_Use_decl_annotations_ +static NTSTATUS +DispatchCreate(DEVICE_OBJECT *DeviceObject, IRP *Irp) +{ + WG_DEVICE *Wg = DeviceObject->Reserved; + if (Wg && ReadBooleanNoFence(&Wg->IsDeviceRemoving)) + { + Irp->IoStatus.Status = NDIS_STATUS_ADAPTER_REMOVED; + IoCompleteRequest(Irp, IO_NO_INCREMENT); + return NDIS_STATUS_ADAPTER_REMOVED; + } + return NdisDispatchCreate(DeviceObject, Irp); +} + +_Dispatch_type_(IRP_MJ_PNP) +static DRIVER_DISPATCH_PAGED DispatchPnp; +_Use_decl_annotations_ +static NTSTATUS +DispatchPnp(DEVICE_OBJECT *DeviceObject, IRP *Irp) +{ + IO_STACK_LOCATION *Stack = IoGetCurrentIrpStackLocation(Irp); + if (Stack->MinorFunction != IRP_MN_QUERY_REMOVE_DEVICE && Stack->MinorFunction != IRP_MN_SURPRISE_REMOVAL) + goto ndisDispatch; + + WG_DEVICE *Wg = DeviceObject->Reserved; + if (!Wg) + goto ndisDispatch; + WriteBooleanNoFence(&Wg->IsDeviceRemoving, TRUE); + KeSetEvent(&Wg->Log.NewEntry, IO_NO_INCREMENT, FALSE); + + OBJECT_ATTRIBUTES ObjectAttributes; + InitializeObjectAttributes(&ObjectAttributes, NULL, OBJ_KERNEL_HANDLE, NULL, NULL); + HANDLE Handle; + if (NT_SUCCESS(PsCreateSystemThread( + &Handle, THREAD_ALL_ACCESS, &ObjectAttributes, NULL, NULL, ForceCloseHandlesAfterDelay, Wg))) + { +#pragma warning(suppress : 28126) /* Handle is a kernel handle. */ + ObReferenceObjectByHandle(Handle, SYNCHRONIZE, NULL, KernelMode, &Wg->HandleForceCloseThread, NULL); + ZwClose(Handle); + } + +ndisDispatch: + return NdisDispatchPnp(DeviceObject, Irp); +} + +_Use_decl_annotations_ +VOID +IoctlHalt(WG_DEVICE *Wg) +{ + WritePointerNoFence(&Wg->FunctionalDeviceObject->Reserved, NULL); + KeSetEvent(&Wg->DeviceRemoved, IO_NETWORK_INCREMENT, FALSE); + if (Wg->HandleForceCloseThread) + { + KeWaitForSingleObject(Wg->HandleForceCloseThread, Executive, KernelMode, FALSE, NULL); + ObDereferenceObject(Wg->HandleForceCloseThread); + Wg->HandleForceCloseThread = NULL; + } +} + +#ifdef ALLOC_PRAGMA +# pragma alloc_text(INIT, IoctlDriverEntry) +#endif +_Use_decl_annotations_ +VOID +IoctlDriverEntry(DRIVER_OBJECT *DriverObject) +{ + NdisDispatchDeviceControl = DriverObject->MajorFunction[IRP_MJ_DEVICE_CONTROL]; + NdisDispatchCreate = DriverObject->MajorFunction[IRP_MJ_CREATE]; + NdisDispatchPnp = DriverObject->MajorFunction[IRP_MJ_PNP]; + DriverObject->MajorFunction[IRP_MJ_DEVICE_CONTROL] = DispatchDeviceControl; + DriverObject->MajorFunction[IRP_MJ_CREATE] = DispatchCreate; + DriverObject->MajorFunction[IRP_MJ_PNP] = DispatchPnp; +} diff --git a/driver/ioctl.h b/driver/ioctl.h new file mode 100644 index 0000000..2d4d97b --- /dev/null +++ b/driver/ioctl.h @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +#ifdef _KERNEL_MODE +# include "device.h" +# include <ntifs.h> /* Must be included before <wdm.h> */ +# include <wdm.h> +# include <ndis.h> +#else +# include <winsock2.h> +# include <Windows.h> +# include <ws2def.h> +# include <ws2ipdef.h> +# include <devioctl.h> +#endif + +#pragma warning(push) +#pragma warning(disable : 4324) /* structure was padded due to alignment specifier */ + +#define WG_KEY_LEN 32 +#define WG_MAX_LOG_LINE_LEN 128 + +typedef __declspec(align(8)) struct _WG_IOCTL_ALLOWED_IP +{ + union + { + IN_ADDR V4; + IN6_ADDR V6; + } Address; + ADDRESS_FAMILY AddressFamily; + UCHAR Cidr; +} WG_IOCTL_ALLOWED_IP; + +typedef enum +{ + WG_IOCTL_PEER_HAS_PUBLIC_KEY = 1 << 0, + WG_IOCTL_PEER_HAS_PRESHARED_KEY = 1 << 1, + WG_IOCTL_PEER_HAS_PERSISTENT_KEEPALIVE = 1 << 2, + WG_IOCTL_PEER_HAS_ENDPOINT = 1 << 3, + WG_IOCTL_PEER_HAS_PROTOCOL_VERSION = 1 << 4, + WG_IOCTL_PEER_REPLACE_ALLOWED_IPS = 1 << 5, + WG_IOCTL_PEER_REMOVE = 1 << 6, + WG_IOCTL_PEER_UPDATE = 1 << 7 +} WG_IOCTL_PEER_FLAG; + +typedef __declspec(align(8)) struct _WG_IOCTL_PEER +{ + WG_IOCTL_PEER_FLAG Flags; + ULONG ProtocolVersion; /* 0 = latest protocol, 1 = this protocol. */ + UCHAR PublicKey[WG_KEY_LEN]; + UCHAR PresharedKey[WG_KEY_LEN]; + USHORT PersistentKeepalive; + SOCKADDR_INET Endpoint; + ULONG64 TxBytes; + ULONG64 RxBytes; + ULONG64 LastHandshake; + ULONG AllowedIPsCount; +} WG_IOCTL_PEER; + +typedef enum +{ + WG_IOCTL_INTERFACE_HAS_PUBLIC_KEY = 1 << 0, + WG_IOCTL_INTERFACE_HAS_PRIVATE_KEY = 1 << 1, + WG_IOCTL_INTERFACE_HAS_LISTEN_PORT = 1 << 2, + WG_IOCTL_INTERFACE_REPLACE_PEERS = 1 << 3 +} WG_IOCTL_INTERFACE_FLAG; + +typedef __declspec(align(8)) struct _WG_IOCTL_INTERFACE +{ + WG_IOCTL_INTERFACE_FLAG Flags; + USHORT ListenPort; + UCHAR PrivateKey[WG_KEY_LEN]; + UCHAR PublicKey[WG_KEY_LEN]; + ULONG PeersCount; +} WG_IOCTL_INTERFACE; + +typedef enum +{ + WG_IOCTL_ADAPTER_STATE_DOWN = 0, + WG_IOCTL_ADAPTER_STATE_UP = 1, + WG_IOCTL_ADAPTER_STATE_QUERY = 2 +} WG_IOCTL_ADAPTER_STATE; + +/* Get adapter properties. + * + * The lpOutBuffer and nOutBufferSize parameters of DeviceIoControl() must describe an user allocated buffer + * and its size in bytes. The buffer will be filled with a WG_IOCTL_INTERFACE struct followed by zero or more + * WG_IOCTL_PEER structs. Should all data not fit into the buffer, ERROR_MORE_DATA is returned with the required + * size of the buffer. + */ +#define WG_IOCTL_GET CTL_CODE(45208U, 321, METHOD_OUT_DIRECT, FILE_READ_DATA | FILE_WRITE_DATA) + +/* Set adapter properties. + * + * The lpInBuffer and nInBufferSize parameters of DeviceIoControl() must describe a WG_IOCTL_INTERFACE struct followed + * by PeersCount times WG_IOCTL_PEER struct. + */ +#define WG_IOCTL_SET CTL_CODE(45208U, 322, METHOD_IN_DIRECT, FILE_READ_DATA | FILE_WRITE_DATA) + +/* Bring adapter up, down, or query existing adapter state. Input is verb. Output is current state after operation. */ +#define WG_IOCTL_SET_ADAPTER_STATE CTL_CODE(45208U, 323, METHOD_BUFFERED, FILE_READ_DATA | FILE_WRITE_DATA) + +/* Read the next line in the adapter log. */ +#define WG_IOCTL_READ_LOG_LINE CTL_CODE(45208U, 324, METHOD_BUFFERED, FILE_READ_DATA | FILE_WRITE_DATA) + +#ifdef _KERNEL_MODE + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_not_held_(_Global_critical_region_) +VOID +IoctlHalt(_Inout_ WG_DEVICE *Wg); + +_IRQL_requires_max_(APC_LEVEL) +VOID +IoctlDriverEntry(_In_ DRIVER_OBJECT *DriverObject); + +#endif + +#pragma warning(pop) diff --git a/driver/logging.c b/driver/logging.c new file mode 100644 index 0000000..047d701 --- /dev/null +++ b/driver/logging.c @@ -0,0 +1,190 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include "logging.h" +#include "ioctl.h" +#include <ntstrsafe.h> + +static_assert(WG_MAX_LOG_LINE_LEN == MAX_LOG_LINE_LEN, "Log length mismatch"); + +_Use_decl_annotations_ +VOID +LogRingInit(LOG_RING *Log) +{ + Log->CurrentWriters = Log->FirstAndLength = 0; + KeInitializeEvent(&Log->NewEntry, SynchronizationEvent, FALSE); +} + +typedef struct _FIRST_AND_LENGTH +{ + USHORT First, Length; +} FIRST_AND_LENGTH; +static_assert(sizeof(FIRST_AND_LENGTH) == sizeof(((LOG_RING *)0)->FirstAndLength), "First and length size mismatch"); + +_Use_decl_annotations_ +VOID +LogRingWrite(LOG_RING *Log, PCSTR Format, ...) +{ + if (InterlockedIncrement(&Log->CurrentWriters) >= BUFFERED_LOG_ENTRIES - 1) + goto out; /* Drop log entries if there's contention, rather than block. */ + + USHORT Index; + for (LONG OldFal = ReadNoFence(&Log->FirstAndLength);;) + { + LONG NewFal = OldFal; + FIRST_AND_LENGTH *Fal = (FIRST_AND_LENGTH *)&NewFal; + if (Fal->Length == BUFFERED_LOG_ENTRIES) + Fal->First = (Fal->First + 1) & BUFFERED_LOG_ENTRIES_MASK; + else + ++Fal->Length; + Index = (Fal->First + Fal->Length - 1) & BUFFERED_LOG_ENTRIES_MASK; + LONG CurFal = InterlockedCompareExchange(&Log->FirstAndLength, NewFal, OldFal); + if (CurFal == OldFal) + break; + OldFal = CurFal; + } + + va_list Args; + va_start(Args, Format); + RtlStringCbVPrintfA(Log->Entries[Index], MAX_LOG_LINE_LEN, Format, Args); + va_end(Args); + KeSetEvent(&Log->NewEntry, IO_NO_INCREMENT, FALSE); +out: + InterlockedDecrement(&Log->CurrentWriters); +} + +_Use_decl_annotations_ +NTSTATUS +LogRingRead(LOG_RING *Log, CHAR Line[MAX_LOG_LINE_LEN], BOOLEAN *WhileFalse) +{ + NTSTATUS Status; + + while (!ReadBooleanNoFence(WhileFalse)) + { + USHORT Index; + for (LONG OldFal = ReadNoFence(&Log->FirstAndLength);;) + { + LONG NewFal = OldFal; + FIRST_AND_LENGTH *Fal = (FIRST_AND_LENGTH *)&NewFal; + if (!Fal->Length) + goto wait; + Index = Fal->First & BUFFERED_LOG_ENTRIES_MASK; + Fal->First = (Fal->First + 1) & BUFFERED_LOG_ENTRIES_MASK; + --Fal->Length; + LONG CurFal = InterlockedCompareExchange(&Log->FirstAndLength, NewFal, OldFal); + if (CurFal == OldFal) + break; + OldFal = CurFal; + } + RtlCopyMemory(Line, Log->Entries[Index], MAX_LOG_LINE_LEN); + return STATUS_SUCCESS; + + wait: + Status = KeWaitForSingleObject(&Log->NewEntry, UserRequest, UserMode, TRUE, NULL); + if (Status != STATUS_SUCCESS) /* Intentionally not using NT_SUCCESS */ + break; + } + return STATUS_CANCELLED; +} + +enum +{ + RATELIMIT_INTERVAL = 5 * SYS_TIME_UNITS_PER_SEC, + RATELIMIT_BURST = 10, +}; + +_Use_decl_annotations_ +BOOLEAN +LogRingIsRatelimited(_Inout_ LOG_RING *Log) +{ + BOOLEAN Ret = TRUE; + if (InterlockedCompareExchange(&Log->RatelimitChecking, 1, 0) != 0) + return Ret; + ULONG64 Now = KeQueryInterruptTime(); + if (!Log->RatelimitStart) + Log->RatelimitStart = Now; + if ((LONG64)(Log->RatelimitStart + RATELIMIT_INTERVAL) - (LONG64)Now < 0) + { + if (Log->RatelimitMissed) + { + LogWarn( + CONTAINING_RECORD(Log, WG_DEVICE, Log), + "%u log lines swallowed by rate limiting", + Log->RatelimitMissed); + Log->RatelimitMissed = 0; + } + Log->RatelimitStart = Now; + Log->RatelimitPrinted = 0; + } + if (Log->RatelimitPrinted <= RATELIMIT_BURST) + { + ++Log->RatelimitPrinted; + Ret = FALSE; + } + else + ++Log->RatelimitMissed; + WriteNoFence(&Log->RatelimitChecking, 0); + return Ret; +} + +_Use_decl_annotations_ +VOID +SockaddrToString(PSTR Buffer, CONST SOCKADDR_INET *Addr) +{ + if (Addr->si_family == AF_INET) + { + ULONG Length = SOCKADDR_STR_MAX_LEN; + if (NT_SUCCESS(RtlIpv4AddressToStringExA(&Addr->Ipv4.sin_addr, Addr->Ipv4.sin_port, Buffer, &Length))) + return; + } + else if (Addr->si_family == AF_INET6) + { + ULONG Length = SOCKADDR_STR_MAX_LEN; + if (NT_SUCCESS(RtlIpv6AddressToStringExA( + &Addr->Ipv6.sin6_addr, Addr->Ipv6.sin6_scope_id, Addr->Ipv6.sin6_port, Buffer, &Length))) + return; + } + RtlStringCbCopyA(Buffer, SOCKADDR_STR_MAX_LEN, "no address"); +} + +#if DBG +_Use_decl_annotations_ +VOID +DumpNetBuffer(NET_BUFFER *Nb, CHAR *Prefix) +{ + ULONG Len = NET_BUFFER_DATA_LENGTH(Nb); + CHAR Format[128]; + DbgPrintEx(DPFLTR_IHVNETWORK_ID, 1, LOG_DRIVER_PREFIX "%s (%u):\n", Prefix, Len); + ULONG MdlOffset = NET_BUFFER_CURRENT_MDL_OFFSET(Nb), i = 0, k = 0; + for (MDL *Mdl = NET_BUFFER_CURRENT_MDL(Nb); Mdl; Mdl = Mdl->Next) + { + if (!Len) + break; + ULONG MdlLen = min(MmGetMdlByteCount(Mdl) - MdlOffset, Len); + UCHAR *Data = MmGetSystemAddressForMdlSafe(Mdl, NormalPagePriority | MdlMappingNoExecute | MdlMappingNoWrite); + for (ULONG j = 0; j < MdlLen; ++j, ++k) + { + i += Data ? _snprintf_s( + Format + i, + sizeof(Format) - i, + _TRUNCATE, + "%02x%c", + Data[j + MdlOffset], + (k % 16 == 15) ? '\n' : ' ') + : _snprintf_s(Format + i, sizeof(Format) - i, _TRUNCATE, "..%c", (k % 16 == 15) ? '\n' : ' '); + if (k % 16 == 15) + { + DbgPrintEx(DPFLTR_IHVNETWORK_ID, 1, "%s", Format); + i = 0; + Format[0] = 0; + } + } + Len -= MdlLen; + MdlOffset = 0; + } + if (i) + DbgPrintEx(DPFLTR_IHVNETWORK_ID, 1, "%s\n", Format); +} +#endif diff --git a/driver/logging.h b/driver/logging.h new file mode 100644 index 0000000..39e6d39 --- /dev/null +++ b/driver/logging.h @@ -0,0 +1,120 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +#include <ntifs.h> /* Must be included before <wdm.h> */ +#include <wdm.h> +#include <ndis.h> +#include <wsk.h> + +#define LOG_DRIVER_PREFIX "wireguard: " +#define LOG_DEVICE_PREFIX "%u: " + +#if DBG +# define LogErr(Device, Fmt, ...) \ + do \ + { \ + DbgPrintEx( \ + DPFLTR_IHVNETWORK_ID, \ + 1, \ + LOG_DRIVER_PREFIX LOG_DEVICE_PREFIX Fmt "\n", \ + (Device)->InterfaceIndex, \ + ##__VA_ARGS__); \ + LogRingWrite(&(Device)->Log, "1" Fmt, ##__VA_ARGS__); \ + } while (0) +# define LogWarn(Device, Fmt, ...) \ + do \ + { \ + DbgPrintEx( \ + DPFLTR_IHVNETWORK_ID, \ + 2, \ + LOG_DRIVER_PREFIX LOG_DEVICE_PREFIX Fmt "\n", \ + (Device)->InterfaceIndex, \ + ##__VA_ARGS__); \ + LogRingWrite(&(Device)->Log, "2" Fmt, ##__VA_ARGS__); \ + } while (0) +# define LogInfo(Device, Fmt, ...) \ + do \ + { \ + DbgPrintEx( \ + DPFLTR_IHVNETWORK_ID, \ + 3, \ + LOG_DRIVER_PREFIX LOG_DEVICE_PREFIX Fmt "\n", \ + (Device)->InterfaceIndex, \ + ##__VA_ARGS__); \ + LogRingWrite(&(Device)->Log, "3" Fmt, ##__VA_ARGS__); \ + } while (0) +# define LogDebug(Fmt, ...) DbgPrintEx(DPFLTR_IHVNETWORK_ID, 4, LOG_DRIVER_PREFIX Fmt "\n", ##__VA_ARGS__) +#else +# define LogErr(Device, Fmt, ...) LogRingWrite(&(Device)->Log, "1" Fmt, ##__VA_ARGS__) +# define LogWarn(Device, Fmt, ...) LogRingWrite(&(Device)->Log, "2" Fmt, ##__VA_ARGS__) +# define LogInfo(Device, Fmt, ...) LogRingWrite(&(Device)->Log, "3" Fmt, ##__VA_ARGS__) +# define LogDebug(Fmt, ...) +#endif + +#define LogInfoRatelimited(Device, Fmt, ...) \ + do \ + { \ + if (!LogRingIsRatelimited(&(Device)->Log)) \ + LogInfo(Device, Fmt, ##__VA_ARGS__); \ + } while (0) + +#define LogInfoNblRatelimited(Device, Fmt, Nbl, ...) \ + do \ + { \ + ENDPOINT __Endpoint; \ + CHAR __EndpointStr[SOCKADDR_STR_MAX_LEN]; \ + SocketEndpointFromNbl(&__Endpoint, Nbl); \ + SockaddrToString(__EndpointStr, &__Endpoint.Addr); \ + LogInfoRatelimited(Device, Fmt, __EndpointStr, ##__VA_ARGS__); \ + } while (0) + +enum +{ + MAX_LOG_LINE_LEN = 128, + BUFFERED_LOG_ENTRIES = 32, + BUFFERED_LOG_ENTRIES_MASK = BUFFERED_LOG_ENTRIES - 1, +}; + +typedef struct _LOG_RING +{ + CHAR Entries[BUFFERED_LOG_ENTRIES][MAX_LOG_LINE_LEN]; + LONG FirstAndLength; + LONG CurrentWriters; + KEVENT NewEntry; + LONG RatelimitChecking; + ULONG RatelimitPrinted, RatelimitMissed; + ULONG64 RatelimitStart; +} LOG_RING; + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +LogRingWrite(_Inout_ LOG_RING *Log, _In_z_ _Printf_format_string_ PCSTR Format, ...); + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +LogRingInit(_Inout_ LOG_RING *Log); + +_IRQL_requires_max_(APC_LEVEL) +NTSTATUS +LogRingRead( + _Inout_ LOG_RING *Log, + _Out_writes_bytes_all_(MAX_LOG_LINE_LEN) CHAR Line[MAX_LOG_LINE_LEN], + _In_ BOOLEAN *WhileFalse); + +_IRQL_requires_max_(DISPATCH_LEVEL) +BOOLEAN +LogRingIsRatelimited(_Inout_ LOG_RING *Log); + +#define SOCKADDR_STR_MAX_LEN max(INET_ADDRSTRLEN, INET6_ADDRSTRLEN) +VOID +SockaddrToString(_Out_z_cap_c_(SOCKADDR_STR_MAX_LEN) PSTR Buffer, _In_ CONST SOCKADDR_INET *Addr); + +#if DBG +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +DumpNetBuffer(_In_ NET_BUFFER *Nb, _In_z_ CHAR *Prefix); +#endif diff --git a/driver/main.c b/driver/main.c new file mode 100644 index 0000000..b2f305f --- /dev/null +++ b/driver/main.c @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include "device.h" +#include "noise.h" +#include "queueing.h" +#include "ratelimiter.h" +#include "rcu.h" +#include "socket.h" +#include "logging.h" +#include "crypto.h" +#include <wsk.h> +#include <ndis.h> + +DRIVER_INITIALIZE DriverEntry; +#ifdef ALLOC_PRAGMA +# pragma alloc_text(INIT, DriverEntry) +#endif +_Use_decl_annotations_ +NTSTATUS +DriverEntry(DRIVER_OBJECT *DriverObject, UNICODE_STRING *RegistryPath) +{ + NTSTATUS Ret; + + ExInitializeDriverRuntime(DrvRtPoolNxOptIn); + + CryptoDriverEntry(); + NoiseDriverEntry(); + + Ret = MemDriverEntry(); + if (!NT_SUCCESS(Ret)) + return Ret; + + Ret = RcuDriverEntry(); + if (!NT_SUCCESS(Ret)) + goto cleanupMem; + + Ret = AllowedIpsDriverEntry(); + if (!NT_SUCCESS(Ret)) + goto cleanupRcu; + + Ret = RatelimiterDriverEntry(); + if (!NT_SUCCESS(Ret)) + goto cleanupAllowedIps; + + Ret = PeerDriverEntry(); + if (!NT_SUCCESS(Ret)) + goto cleanupRatelimiter; + + Ret = DeviceDriverEntry(DriverObject, RegistryPath); + if (!NT_SUCCESS(Ret)) + goto cleanupPeer; + +#ifdef DBG + if (!CryptoSelftest() || !AllowedIpsSelftest() || !PacketCounterSelftest() || !RatelimiterSelftest()) + { + Ret = STATUS_INTERNAL_ERROR; + goto cleanupDevice; + } +#endif + return 0; + +#ifdef DBG +cleanupDevice: + DeviceUnload(); +#endif +cleanupPeer: + PeerUnload(); +cleanupRatelimiter: + RatelimiterUnload(); +cleanupAllowedIps: + AllowedIpsUnload(); +cleanupRcu: + RcuUnload(); +cleanupMem: + MemUnload(); + return Ret; +} + +MINIPORT_UNLOAD Unload; +_Use_decl_annotations_ +VOID +Unload(PDRIVER_OBJECT DriverObject) +{ + DeviceUnload(); + WskUnload(); + PeerUnload(); + RatelimiterUnload(); + AllowedIpsUnload(); + RcuUnload(); + MemUnload(); +} diff --git a/driver/memory.c b/driver/memory.c new file mode 100644 index 0000000..4718909 --- /dev/null +++ b/driver/memory.c @@ -0,0 +1,213 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include "memory.h" +#include "messages.h" + +static CONST ULONG PacketCacheSizes[] = { 192, 512, 1024, 1500, 9000 }; +static LOOKASIDE_ALIGN LOOKASIDE_LIST_EX PacketCaches[ARRAYSIZE(PacketCacheSizes)]; + +_Must_inspect_result_ +_IRQL_requires_max_(DISPATCH_LEVEL) +_Return_type_success_(return != NULL) +static __drv_allocatesMem(Mem) +VOID * +MemAllocateFromPacketCaches(_In_ ULONG BufferSize) +{ + for (ULONG i = 0; i < ARRAYSIZE(PacketCacheSizes); ++i) + { + if (PacketCacheSizes[i] >= BufferSize) + return ExAllocateFromLookasideListEx(&PacketCaches[i]); + } + return MemAllocate(BufferSize); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +static VOID +MemFreeToPacketCaches(_In_ ULONG BufferSize, _In_ __drv_freesMem(Mem) VOID *Memory) +{ + for (ULONG i = 0; i < ARRAYSIZE(PacketCacheSizes); ++i) + { + if (PacketCacheSizes[i] >= BufferSize) + { + ExFreeToLookasideListEx(&PacketCaches[i], Memory); + return; + } + } + MemFree(Memory); +} + +#pragma warning(suppress : 28195) /* IoAllocateMdl allocates, even if missing the SAL annotation. */ +_Use_decl_annotations_ +MDL * +MemAllocateDataAndMdlChain(ULONG BufferSize) +{ + NT_ASSERT(BufferSize <= (MAXULONG - PAGE_SIZE)); + VOID *Memory = MemAllocateFromPacketCaches(BufferSize); + if (!Memory) + return NULL; + MDL *Mdl = IoAllocateMdl(Memory, BufferSize, FALSE, FALSE, NULL); + if (!Mdl) + { + MemFreeToPacketCaches(BufferSize, Memory); + return NULL; + } + MmBuildMdlForNonPagedPool(Mdl); + return Mdl; +} + +#pragma warning(suppress : 6014) /* IoFreeMdl frees, even if missing the SAL annotation. */ +_Use_decl_annotations_ +VOID +MemFreeDataAndMdlChain(MDL *Mdl) +{ + while (Mdl) + { + MDL *Next = Mdl->Next; + ULONG BufferCount = MmGetMdlByteCount(Mdl); + VOID *Memory = MmGetMdlVirtualAddress(Mdl); + IoFreeMdl(Mdl); + MemFreeToPacketCaches(BufferCount, Memory); + Mdl = Next; + } +} + +_Use_decl_annotations_ +NET_BUFFER_LIST * +MemAllocateNetBufferList(NDIS_HANDLE NblPool, NDIS_HANDLE NbPool, ULONG SpaceBefore, ULONG Size, ULONG SpaceAfter) +{ + ULONG Sum = Size; + if (!NT_SUCCESS(RtlULongAdd(Sum, SpaceBefore, &Sum) || !NT_SUCCESS(RtlULongAdd(Sum, SpaceAfter, &Sum))) || + Sum > MTU_MAX) + return NULL; +#pragma warning(suppress : 6014) /* MDL is aliased in Nbl or freed on failure. */ + MDL *Mdl = MemAllocateDataAndMdlChain(Sum); + if (!Mdl) + return NULL; + NET_BUFFER *Nb = NdisAllocateNetBuffer(NbPool, Mdl, SpaceBefore, Size); + if (!Nb) + goto cleanupMdl; + NET_BUFFER_LIST *Nbl = NdisAllocateNetBufferList(NblPool, 0, 0); + if (!Nbl) + goto cleanupNb; + NET_BUFFER_LIST_FIRST_NB(Nbl) = Nb; + return Nbl; + +cleanupNb: + NdisFreeNetBuffer(Nb); +cleanupMdl: + MemFreeDataAndMdlChain(Mdl); + return NULL; +} + +_Use_decl_annotations_ +VOID +MemFreeNetBufferList(NET_BUFFER_LIST *Nbl) +{ + while (NET_BUFFER_LIST_FIRST_NB(Nbl)) + { + NET_BUFFER *Nb = NET_BUFFER_LIST_FIRST_NB(Nbl); + NET_BUFFER_LIST_FIRST_NB(Nbl) = NET_BUFFER_NEXT_NB(NET_BUFFER_LIST_FIRST_NB(Nbl)); + MemFreeDataAndMdlChain(NET_BUFFER_FIRST_MDL(Nb)); + NdisFreeNetBuffer(Nb); + } + NdisFreeNetBufferList(Nbl); +} + +#pragma warning(suppress : 28195) /* NdisAllocateNetBufferList & co allocate. */ +_Use_decl_annotations_ +NET_BUFFER_LIST * +MemAllocateNetBufferListWithClonedGeometry( + NDIS_HANDLE NblPool, + NDIS_HANDLE NbPool, + NET_BUFFER_LIST *Original, + ULONG AdditionalBytesPerNb) +{ + NET_BUFFER_LIST *Clone = NdisAllocateNetBufferList(NblPool, 0, 0); + if (!Clone) + return NULL; + NET_BUFFER_LIST_INFO(Clone, NetBufferListProtocolId) = NET_BUFFER_LIST_INFO(Original, NetBufferListProtocolId); + NET_BUFFER **CloneNb = &NET_BUFFER_LIST_FIRST_NB(Clone); + for (NET_BUFFER *Nb = NET_BUFFER_LIST_FIRST_NB(Original); Nb; Nb = NET_BUFFER_NEXT_NB(Nb)) + { + ULONG Length; + if (NET_BUFFER_DATA_LENGTH(Nb) > MTU_MAX || + !NT_SUCCESS(RtlULongAdd(NET_BUFFER_DATA_LENGTH(Nb), AdditionalBytesPerNb, &Length))) + goto cleanupClone; +#pragma warning(suppress : 6014) /* `CloneMdl` is aliased in NdisAllocateNetBuffer or freed on failure. */ + MDL *CloneMdl = MemAllocateDataAndMdlChain(Length); + if (!CloneMdl) + goto cleanupClone; + *CloneNb = NdisAllocateNetBuffer(NbPool, CloneMdl, 0, 0); + if (!*CloneNb) + { + MemFreeDataAndMdlChain(CloneMdl); + goto cleanupClone; + } + CloneNb = &NET_BUFFER_NEXT_NB(*CloneNb); + } + Clone->ParentNetBufferList = Original; + return Clone; + +cleanupClone: + MemFreeNetBufferList(Clone); + return NULL; +} + +_Use_decl_annotations_ +NTSTATUS +MemCopyFromMdl(VOID *Dst, MDL *Src, ULONG Offset, ULONG Size) +{ + if (!Src) + return STATUS_BUFFER_TOO_SMALL; + UCHAR *DstBuf = Dst; + while (Offset >= MmGetMdlByteCount(Src)) + { + Offset -= MmGetMdlByteCount(Src); + Src = Src->Next; + if (!Src) + return STATUS_BUFFER_TOO_SMALL; + } + for (ULONG CurSize; Size; Src = Src->Next, Size -= CurSize, DstBuf += CurSize) + { + if (!Src) + return STATUS_BUFFER_TOO_SMALL; + UCHAR *SrcBuf = MmGetSystemAddressForMdlSafe(Src, NormalPagePriority | MdlMappingNoExecute | MdlMappingNoWrite); + if (!SrcBuf) + return STATUS_INSUFFICIENT_RESOURCES; + CurSize = min(MmGetMdlByteCount(Src) - Offset, Size); + RtlCopyMemory(DstBuf, SrcBuf + Offset, CurSize); + Offset = 0; + } + return STATUS_SUCCESS; +} + +#ifdef ALLOC_PRAGMA +# pragma alloc_text(INIT, MemDriverEntry) +#endif +_Use_decl_annotations_ +NTSTATUS +MemDriverEntry(VOID) +{ + for (ULONG i = 0; i < ARRAYSIZE(PacketCacheSizes); ++i) + { + NTSTATUS Status = ExInitializeLookasideListEx( + &PacketCaches[i], NULL, NULL, NonPagedPool, 0, PacketCacheSizes[i], MEMORY_TAG, 0); + if (!NT_SUCCESS(Status)) + { + for (ULONG j = 0; j < i; ++j) + ExDeleteLookasideListEx(&PacketCaches[j]); + return Status; + } + } + return STATUS_SUCCESS; +} + +_Use_decl_annotations_ +VOID MemUnload(VOID) +{ + for (ULONG i = 0; i < ARRAYSIZE(PacketCacheSizes); ++i) + ExDeleteLookasideListEx(&PacketCaches[i]); +} diff --git a/driver/memory.h b/driver/memory.h new file mode 100644 index 0000000..99a691e --- /dev/null +++ b/driver/memory.h @@ -0,0 +1,165 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +#include "arithmetic.h" +#include <ntifs.h> /* Must be included before <wdm.h> */ +#include <wdm.h> +#include <ndis.h> + +#define MEMORY_TAG Be32ToCpu('wgnt') + +/* Source analysis has issues with ExAllocatePool... annotations raising false alerts. */ +#pragma warning(push) +#pragma warning(disable : 28118) +#pragma warning(disable : 28160) + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Post_maybenull_ +_Must_inspect_result_ +_Post_writable_byte_size_(NumberOfBytes) +_Return_type_success_(return != NULL) +static inline __drv_allocatesMem(Mem) +VOID * +MemAllocate(_In_ SIZE_T NumberOfBytes) +{ + return ExAllocatePoolUninitialized(NonPagedPool, NumberOfBytes, MEMORY_TAG); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Post_maybenull_ +_Must_inspect_result_ +_Post_writable_byte_size_(NumberOfBytes) +_Return_type_success_(return != NULL) +_At_buffer_((UCHAR *)return, _Iter_, NumberOfBytes, _Post_satisfies_(((UCHAR *)return )[_Iter_] == 0)) +static inline __drv_allocatesMem(Mem) +VOID * +MemAllocateAndZero(_In_ SIZE_T NumberOfBytes) +{ + return ExAllocatePoolZero(NonPagedPool, NumberOfBytes, MEMORY_TAG); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Post_maybenull_ +_Must_inspect_result_ +_Post_writable_byte_size_((NumberOfElements) * (SizeOfOneElement)) +_Return_type_success_(return != NULL) +static inline __drv_allocatesMem(Mem) +VOID * +MemAllocateArray(_In_ SIZE_T NumberOfElements, _In_ SIZE_T SizeOfOneElement) +{ + SIZE_T Size; + if (!NT_SUCCESS(RtlSIZETMult(NumberOfElements, SizeOfOneElement, &Size))) + return NULL; + return ExAllocatePoolUninitialized(NonPagedPool, Size, MEMORY_TAG); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Post_maybenull_ +_Must_inspect_result_ +_Post_writable_byte_size_((NumberOfElements) * (SizeOfOneElement)) +_Return_type_success_(return != NULL) +_At_buffer_( + (UCHAR *)return, + _Iter_, + NumberOfElements *SizeOfOneElement, + _Post_satisfies_(((UCHAR *)return )[_Iter_] == 0)) +static inline __drv_allocatesMem(Mem) +VOID * +MemAllocateArrayAndZero(_In_ SIZE_T NumberOfElements, _In_ SIZE_T SizeOfOneElement) +{ + SIZE_T Size; + if (!NT_SUCCESS(RtlSIZETMult(NumberOfElements, SizeOfOneElement, &Size))) + return NULL; + return ExAllocatePoolZero(NonPagedPool, Size, MEMORY_TAG); +} + +#pragma warning(pop) + +_IRQL_requires_max_(DISPATCH_LEVEL) +static inline VOID +MemFree(_Pre_maybenull_ __drv_freesMem(Mem) VOID *Ptr) +{ + if (Ptr) + ExFreePoolWithTag(Ptr, MEMORY_TAG); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +static inline VOID +MemFreeSensitive(_Pre_maybenull_ __drv_freesMem(Mem) VOID *Ptr, _In_ SIZE_T Size) +{ + if (Ptr) + { + RtlSecureZeroMemory(Ptr, Size); + ExFreePoolWithTag(Ptr, MEMORY_TAG); + } +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +MemFreeDataAndMdlChain(_In_opt_ __drv_freesMem(Mem) MDL *Mdl); + +_Must_inspect_result_ +_IRQL_requires_max_(DISPATCH_LEVEL) +_Return_type_success_(return != NULL) +__drv_allocatesMem(Mem) +MDL * +MemAllocateDataAndMdlChain(_In_ ULONG Size); + +_Must_inspect_result_ +_IRQL_requires_max_(DISPATCH_LEVEL) +_Return_type_success_(return != NULL) +__drv_allocatesMem(mem) +NET_BUFFER_LIST * +MemAllocateNetBufferList( + _In_ NDIS_HANDLE NblPool, + _In_ NDIS_HANDLE NbPool, + _In_ ULONG SpaceBefore, + _In_ ULONG Size, + _In_ ULONG SpaceAfter); + +_IRQL_requires_max_(DISPATCH_LEVEL) +__drv_allocatesMem(mem) +NET_BUFFER_LIST * +MemAllocateNetBufferListWithClonedGeometry( + _In_ NDIS_HANDLE NblPool, + _In_ NDIS_HANDLE NbPool, + _In_ NET_BUFFER_LIST *Original, + _In_ ULONG AdditionalBytesPerNb); + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +MemFreeNetBufferList(__drv_freesMem(mem) _In_ NET_BUFFER_LIST *Nbl); + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Post_notnull_ +static inline VOID * +MemGetValidatedNetBufferData(_In_ CONST NET_BUFFER *Nb) +{ + /* We intentionally neglect to add NET_BUFFER_CURRENT_MDL_OFFSET(Nb) here, as this really + * only applies to NBs that we create ourselves. + */ + return (UCHAR *)MmGetMdlVirtualAddress(NET_BUFFER_CURRENT_MDL(Nb)); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Post_notnull_ +static inline VOID * +MemGetValidatedNetBufferListData(_In_ CONST NET_BUFFER_LIST *Nbl) +{ + return MemGetValidatedNetBufferData(NET_BUFFER_LIST_FIRST_NB(Nbl)); +} + +_Must_inspect_result_ +NTSTATUS +MemCopyFromMdl(_Out_writes_bytes_all_(Size) VOID *Dst, _In_ MDL *Src, _In_ ULONG Offset, _In_ ULONG Size); + +_IRQL_requires_max_(PASSIVE_LEVEL) +NTSTATUS +MemDriverEntry(VOID); + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID MemUnload(VOID); diff --git a/driver/messages.h b/driver/messages.h new file mode 100644 index 0000000..6c12ef4 --- /dev/null +++ b/driver/messages.h @@ -0,0 +1,158 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +#include "arithmetic.h" +#include "crypto.h" + +enum NOISE_LENGTHS +{ + NOISE_PUBLIC_KEY_LEN = CURVE25519_KEY_SIZE, + NOISE_SYMMETRIC_KEY_LEN = CHACHA20POLY1305_KEY_SIZE, + NOISE_TIMESTAMP_LEN = sizeof(UINT64) + sizeof(UINT32), + NOISE_AUTHTAG_LEN = CHACHA20POLY1305_AUTHTAG_SIZE, + NOISE_HASH_LEN = BLAKE2S_HASH_SIZE +}; + +#define NoiseEncryptedLen(PlainLen) ((PlainLen) + NOISE_AUTHTAG_LEN) + +enum COOKIE_VALUES +{ + COOKIE_SECRET_MAX_AGE = 2 * 60, + COOKIE_SECRET_LATENCY = 5, + COOKIE_NONCE_LEN = XCHACHA20POLY1305_NONCE_SIZE, + COOKIE_LEN = 16 +}; + +enum COUNTER_VALUES +{ + COUNTER_BITS_TOTAL = 8192, + COUNTER_REDUNDANT_BITS = BITS_PER_POINTER, + COUNTER_WINDOW_SIZE = COUNTER_BITS_TOTAL - COUNTER_REDUNDANT_BITS +}; + +#define REKEY_AFTER_MESSAGES (1ULL << 60) +#define REJECT_AFTER_MESSAGES (~0ULL - COUNTER_WINDOW_SIZE - 1) +#define REKEY_TIMEOUT 5 +#define REKEY_TIMEOUT_JITTER_MAX_SYS_TIME_UNITS (SYS_TIME_UNITS_PER_SEC / 3) +#define REKEY_AFTER_TIME 120 +#define REJECT_AFTER_TIME 180 +#define INITIATIONS_PER_SECOND 50 +#define MAX_PEERS_PER_DEVICE (1U << 20) +#define KEEPALIVE_TIMEOUT 10 +#define MAX_TIMER_HANDSHAKES (90 / REKEY_TIMEOUT) + +enum L4_LENGTHS +{ + UDP_HEADER_LEN = 8, +}; + +enum MESSAGE_TYPE +{ + MESSAGE_TYPE_INVALID = 0, + MESSAGE_TYPE_HANDSHAKE_INITIATION = 1, + MESSAGE_TYPE_HANDSHAKE_RESPONSE = 2, + MESSAGE_TYPE_HANDSHAKE_COOKIE = 3, + MESSAGE_TYPE_DATA = 4 +}; + +typedef struct _IPV4HDR +{ +#if REG_DWORD == REG_DWORD_LITTLE_ENDIAN + UINT8 Ihl : 4, Version : 4; +#elif REG_DWORD == REG_DWORD_BIG_ENDIAN + UINT8 Version : 4, Ihl : 4; +#endif + UINT8 Tos; + UINT16_BE TotLen; + UINT16_BE Id; + UINT16_BE FragOff; + UINT8 Ttl; + UINT8 Protocol; + UINT16_BE Check; + UINT32_BE Saddr; + UINT32_BE Daddr; +} IPV4HDR; + +typedef struct _IPV6HDR +{ +#if REG_DWORD == REG_DWORD_LITTLE_ENDIAN + UINT8 Priority : 4, Version : 4; +#elif REG_DWORD == REG_DWORD_BIG_ENDIAN + UINT8 Version : 4, Priority : 4; +#endif + UINT8 FlowLbl[3]; + UINT16_BE PayloadLen; + UINT8 Nexthdr; + UINT8 HopLimit; + IN6_ADDR Saddr; + IN6_ADDR Daddr; +} IPV6HDR; + +typedef struct _MESSAGE_HEADER +{ + /* The actual layout of this that we want is: + * u8 type + * u8 reserved_zero[3] + * + * But it turns out that by encoding this as little endian, + * we achieve the same thing, and it makes checking faster. + */ + UINT32_LE Type; +} MESSAGE_HEADER; + +typedef struct _MESSAGE_MACS +{ + UINT8 Mac1[COOKIE_LEN]; + UINT8 Mac2[COOKIE_LEN]; +} MESSAGE_MACS; + +typedef struct _MESSAGE_HANDSHAKE_INITIATION +{ + MESSAGE_HEADER Header; + UINT32_LE SenderIndex; + UINT8 UnencryptedEphemeral[NOISE_PUBLIC_KEY_LEN]; + UINT8 EncryptedStatic[NoiseEncryptedLen(NOISE_PUBLIC_KEY_LEN)]; + UINT8 EncryptedTimestamp[NoiseEncryptedLen(NOISE_TIMESTAMP_LEN)]; + MESSAGE_MACS Macs; +} MESSAGE_HANDSHAKE_INITIATION; + +typedef struct _MESSAGE_HANDSHAKE_RESPONSE +{ + MESSAGE_HEADER Header; + UINT32_LE SenderIndex; + UINT32_LE ReceiverIndex; + UINT8 UnencryptedEphemeral[NOISE_PUBLIC_KEY_LEN]; + UINT8 EncryptedNothing[NoiseEncryptedLen(0)]; + MESSAGE_MACS Macs; +} MESSAGE_HANDSHAKE_RESPONSE; + +typedef struct _MESSAGE_HANDSHAKE_COOKIE +{ + MESSAGE_HEADER Header; + UINT32_LE ReceiverIndex; + UINT8 Nonce[COOKIE_NONCE_LEN]; + UINT8 EncryptedCookie[NoiseEncryptedLen(COOKIE_LEN)]; +} MESSAGE_HANDSHAKE_COOKIE; + +typedef struct _MESSAGE_DATA +{ + MESSAGE_HEADER Header; + UINT32_LE KeyIdx; + UINT64_LE Counter; + UINT8 EncryptedData[]; +} MESSAGE_DATA; + +#define MessageDataLen(PlainLen) (NoiseEncryptedLen(PlainLen) + sizeof(MESSAGE_DATA)) + +enum MESSAGE_ALIGNMENTS +{ + MESSAGE_PADDING_MULTIPLE = 16, + MESSAGE_MINIMUM_LENGTH = MessageDataLen(0) +}; + +#define DATA_PACKET_MINIMUM_LENGTH (max(sizeof(IPV4HDR), sizeof(IPV6HDR)) + UDP_HEADER_LEN + MESSAGE_MINIMUM_LENGTH) +#define MTU_MAX (ALIGN_DOWN_BY_T(SIZE_T, MAXLONG, MESSAGE_PADDING_MULTIPLE) - DATA_PACKET_MINIMUM_LENGTH) diff --git a/driver/noise.c b/driver/noise.c new file mode 100644 index 0000000..7c8d2a1 --- /dev/null +++ b/driver/noise.c @@ -0,0 +1,865 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include "interlocked.h" +#include "crypto.h" +#include "device.h" +#include "messages.h" +#include "noise.h" +#include "peer.h" +#include "peerlookup.h" +#include "queueing.h" +#include "logging.h" + +#pragma warning(disable : 4295) /* array is too small to include a terminating null character */ + +/* This implements Noise_IKpsk2: + * + * <- s + * ****** + * -> e, es, s, ss, {t} + * <- e, ee, se, psk, {} + */ + +static CONST UINT8 HandshakeName[37] = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; +static CONST UINT8 IdentifierName[34] = "WireGuard v1 zx2c4 Jason@zx2c4.com"; +static UINT8 HandshakeInitHash[NOISE_HASH_LEN]; +static UINT8 HandshakeInitChainingKey[NOISE_HASH_LEN]; +static LONG64 KeypairCounter = 0; + +#ifdef ALLOC_PRAGMA +# pragma alloc_text(INIT, NoiseDriverEntry) +#endif +VOID NoiseDriverEntry(VOID) +{ + BLAKE2S_STATE Blake; + + Blake2s(HandshakeInitChainingKey, HandshakeName, NULL, NOISE_HASH_LEN, sizeof(HandshakeName), 0); + Blake2sInit(&Blake, NOISE_HASH_LEN); + Blake2sUpdate(&Blake, HandshakeInitChainingKey, NOISE_HASH_LEN); + Blake2sUpdate(&Blake, IdentifierName, sizeof(IdentifierName)); + Blake2sFinal(&Blake, HandshakeInitHash); +} + +_Use_decl_annotations_ +VOID +NoisePrecomputeStaticStatic(WG_PEER *Peer) +{ + MuAcquirePushLockExclusive(&Peer->Handshake.Lock); + if (!Peer->Handshake.StaticIdentity->HasIdentity || !Curve25519( + Peer->Handshake.PrecomputedStaticStatic, + Peer->Handshake.StaticIdentity->StaticPrivate, + Peer->Handshake.RemoteStatic)) + RtlZeroMemory(Peer->Handshake.PrecomputedStaticStatic, NOISE_PUBLIC_KEY_LEN); + MuReleasePushLockExclusive(&Peer->Handshake.Lock); +} + +_Use_decl_annotations_ +VOID +NoiseHandshakeInit( + NOISE_HANDSHAKE *Handshake, + NOISE_STATIC_IDENTITY *StaticIdentity, + CONST UINT8 PeerPublicKey[NOISE_PUBLIC_KEY_LEN], + CONST UINT8 PeerPresharedKey[NOISE_SYMMETRIC_KEY_LEN], + WG_PEER *Peer) +{ + RtlZeroMemory(Handshake, sizeof(*Handshake)); + MuInitializePushLock(&Handshake->Lock); + Handshake->Entry.Type = INDEX_HASHTABLE_HANDSHAKE; + Handshake->Entry.Peer = Peer; + RtlCopyMemory(Handshake->RemoteStatic, PeerPublicKey, NOISE_PUBLIC_KEY_LEN); + if (PeerPresharedKey) + RtlCopyMemory(Handshake->PresharedKey, PeerPresharedKey, NOISE_SYMMETRIC_KEY_LEN); + Handshake->StaticIdentity = StaticIdentity; + Handshake->State = HANDSHAKE_ZEROED; + NoisePrecomputeStaticStatic(Peer); +} + +static VOID +HandshakeZero(_Out_ NOISE_HANDSHAKE *Handshake) +{ + RtlZeroMemory(&Handshake->EphemeralPrivate, NOISE_PUBLIC_KEY_LEN); + RtlZeroMemory(&Handshake->RemoteEphemeral, NOISE_PUBLIC_KEY_LEN); + RtlZeroMemory(&Handshake->Hash, NOISE_HASH_LEN); + RtlZeroMemory(&Handshake->ChainingKey, NOISE_HASH_LEN); + Handshake->RemoteIndex = 0; + Handshake->State = HANDSHAKE_ZEROED; +} + +_Use_decl_annotations_ +VOID +NoiseHandshakeClear(NOISE_HANDSHAKE *Handshake) +{ + MuAcquirePushLockExclusive(&Handshake->Lock); + IndexHashtableRemove(Handshake->Entry.Peer->Device->IndexHashtable, &Handshake->Entry); + HandshakeZero(Handshake); + MuReleasePushLockExclusive(&Handshake->Lock); +} + +_Must_inspect_result_ +_Post_maybenull_ +_Return_type_success_(return != NULL) +static __drv_allocatesMem(Mem) NOISE_KEYPAIR * +KeypairCreate(_In_ WG_PEER *Peer) +{ + NOISE_KEYPAIR *Keypair = MemAllocateAndZero(sizeof(*Keypair)); + + if (!Keypair) + return NULL; + KeInitializeSpinLock(&Keypair->ReceivingCounter.Lock); + Keypair->InternalId = InterlockedIncrement64(&KeypairCounter); + Keypair->Entry.Type = INDEX_HASHTABLE_KEYPAIR; + Keypair->Entry.Peer = Peer; + KrefInit(&Keypair->Refcount); + return Keypair; +} + +static RCU_CALLBACK_FN KeypairFreeRcu; +_Use_decl_annotations_ +static VOID +KeypairFreeRcu(RCU_CALLBACK *Rcu) +{ + MemFreeSensitive(CONTAINING_RECORD(Rcu, NOISE_KEYPAIR, Rcu), sizeof(NOISE_KEYPAIR)); +} + +static VOID +KeypairFreeKref(_In_ KREF *Kref) +{ + NOISE_KEYPAIR *Keypair = CONTAINING_RECORD(Kref, NOISE_KEYPAIR, Refcount); + + LogInfoRatelimited( + Keypair->Entry.Peer->Device, + "Keypair %llu destroyed for peer %llu", + Keypair->InternalId, + Keypair->Entry.Peer->InternalId); + IndexHashtableRemove(Keypair->Entry.Peer->Device->IndexHashtable, &Keypair->Entry); + RcuCall(&Keypair->Rcu, KeypairFreeRcu); +} + +_Use_decl_annotations_ +VOID +NoiseKeypairPut(NOISE_KEYPAIR *Keypair, BOOLEAN UnreferenceNow) +{ + if (!Keypair) + return; + if (UnreferenceNow) + IndexHashtableRemove(Keypair->Entry.Peer->Device->IndexHashtable, &Keypair->Entry); + KrefPut(&Keypair->Refcount, KeypairFreeKref); +} + +_Use_decl_annotations_ +NOISE_KEYPAIR * +NoiseKeypairGet(NOISE_KEYPAIR *Keypair) +{ + if (!Keypair || !KrefGetUnlessZero(&Keypair->Refcount)) + return NULL; + return Keypair; +} + +_Use_decl_annotations_ +VOID +NoiseKeypairsClear(NOISE_KEYPAIRS *Keypairs) +{ + NOISE_KEYPAIR *Old; + KIRQL Irql; + + KeAcquireSpinLock(&Keypairs->KeypairUpdateLock, &Irql); + + /* We zero the next_keypair before zeroing the others, so that + * wg_noise_received_with_keypair returns early before subsequent ones + * are zeroed. + */ + Old = RcuDereferenceProtected(NOISE_KEYPAIR, Keypairs->NextKeypair, &Keypairs->KeypairUpdateLock); + RcuInitPointer(Keypairs->NextKeypair, NULL); + NoiseKeypairPut(Old, TRUE); + + Old = RcuDereferenceProtected(NOISE_KEYPAIR, Keypairs->PreviousKeypair, &Keypairs->KeypairUpdateLock); + RcuInitPointer(Keypairs->PreviousKeypair, NULL); + NoiseKeypairPut(Old, TRUE); + + Old = RcuDereferenceProtected(NOISE_KEYPAIR, Keypairs->CurrentKeypair, &Keypairs->KeypairUpdateLock); + RcuInitPointer(Keypairs->CurrentKeypair, NULL); + NoiseKeypairPut(Old, TRUE); + + KeReleaseSpinLock(&Keypairs->KeypairUpdateLock, Irql); +} + +_Use_decl_annotations_ +VOID +NoiseExpireCurrentPeerKeypairs(WG_PEER *Peer) +{ + NOISE_KEYPAIR *Keypair; + KIRQL Irql; + + NoiseHandshakeClear(&Peer->Handshake); + NoiseResetLastSentHandshake(&Peer->LastSentHandshake); + + KeAcquireSpinLock(&Peer->Keypairs.KeypairUpdateLock, &Irql); + Keypair = RcuDereferenceProtected(NOISE_KEYPAIR, Peer->Keypairs.NextKeypair, &Peer->Keypairs.KeypairUpdateLock); + if (Keypair) + Keypair->Sending.IsValid = FALSE; + Keypair = RcuDereferenceProtected(NOISE_KEYPAIR, Peer->Keypairs.CurrentKeypair, &Peer->Keypairs.KeypairUpdateLock); + if (Keypair) + Keypair->Sending.IsValid = FALSE; + KeReleaseSpinLock(&Peer->Keypairs.KeypairUpdateLock, Irql); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Requires_lock_not_held_(Keypairs->KeypairUpdateLock) +static VOID +AddNewKeypair(_Inout_ NOISE_KEYPAIRS *Keypairs, _In_ __drv_aliasesMem NOISE_KEYPAIR *NewKeypair) +{ + NOISE_KEYPAIR *PreviousKeypair, *NextKeypair, *CurrentKeypair; + KIRQL Irql; + + KeAcquireSpinLock(&Keypairs->KeypairUpdateLock, &Irql); + PreviousKeypair = RcuDereferenceProtected(NOISE_KEYPAIR, Keypairs->PreviousKeypair, &Keypairs->KeypairUpdateLock); + NextKeypair = RcuDereferenceProtected(NOISE_KEYPAIR, Keypairs->NextKeypair, &Keypairs->KeypairUpdateLock); + CurrentKeypair = RcuDereferenceProtected(NOISE_KEYPAIR, Keypairs->CurrentKeypair, &Keypairs->KeypairUpdateLock); + if (NewKeypair->IAmTheInitiator) + { + /* If we're the initiator, it means we've sent a handshake, and + * received a confirmation response, which means this new + * keypair can now be used. + */ + if (NextKeypair) + { + /* If there already was a next keypair pending, we + * demote it to be the previous keypair, and free the + * existing current. Note that this means KCI can result + * in this transition. It would perhaps be more sound to + * always just get rid of the unused next keypair + * instead of putting it in the previous slot, but this + * might be a bit less robust. Something to think about + * for the future. + */ + RcuInitPointer(Keypairs->NextKeypair, NULL); + RcuAssignPointer(Keypairs->PreviousKeypair, NextKeypair); + NoiseKeypairPut(CurrentKeypair, TRUE); + } + else /* If there wasn't an existing next keypair, we replace + * the previous with the current one. + */ + RcuAssignPointer(Keypairs->PreviousKeypair, CurrentKeypair); + /* At this point we can get rid of the old previous keypair, and + * set up the new keypair. + */ + NoiseKeypairPut(PreviousKeypair, TRUE); + RcuAssignPointer(Keypairs->CurrentKeypair, NewKeypair); + } + else + { + /* If we're the responder, it means we can't use the new keypair + * until we receive confirmation via the first data packet, so + * we get rid of the existing previous one, the possibly + * existing next one, and slide in the new next one. + */ + RcuAssignPointer(Keypairs->NextKeypair, NewKeypair); + NoiseKeypairPut(NextKeypair, TRUE); + RcuInitPointer(Keypairs->PreviousKeypair, NULL); + NoiseKeypairPut(PreviousKeypair, TRUE); + } + KeReleaseSpinLock(&Keypairs->KeypairUpdateLock, Irql); +} + +_Use_decl_annotations_ +BOOLEAN +NoiseReceivedWithKeypair(NOISE_KEYPAIRS *Keypairs, NOISE_KEYPAIR *ReceivedKeypair) +{ + NOISE_KEYPAIR *OldKeypair; + BOOLEAN KeyIsNew; + KIRQL Irql; + + /* We first check without taking the spinlock. */ + KeyIsNew = ReceivedKeypair == RcuAccessPointer(Keypairs->NextKeypair); + if (!KeyIsNew) + return FALSE; + + KeAcquireSpinLock(&Keypairs->KeypairUpdateLock, &Irql); + /* After locking, we double check that things didn't change from + * beneath us. + */ + if (ReceivedKeypair != RcuDereferenceProtected(NOISE_KEYPAIR, Keypairs->NextKeypair, &Keypairs->KeypairUpdateLock)) + { + KeReleaseSpinLock(&Keypairs->KeypairUpdateLock, Irql); + return FALSE; + } + + /* When we've finally received the confirmation, we slide the next + * into the current, the current into the previous, and get rid of + * the old previous. + */ + OldKeypair = RcuDereferenceProtected(NOISE_KEYPAIR, Keypairs->PreviousKeypair, &Keypairs->KeypairUpdateLock); + RcuAssignPointer( + Keypairs->PreviousKeypair, + RcuDereferenceProtected(NOISE_KEYPAIR, Keypairs->CurrentKeypair, &Keypairs->KeypairUpdateLock)); + NoiseKeypairPut(OldKeypair, TRUE); + RcuAssignPointer(Keypairs->CurrentKeypair, ReceivedKeypair); + RcuInitPointer(Keypairs->NextKeypair, NULL); + + KeReleaseSpinLock(&Keypairs->KeypairUpdateLock, Irql); + return TRUE; +} + +_Use_decl_annotations_ +VOID +NoiseSetStaticIdentityPrivateKey(NOISE_STATIC_IDENTITY *StaticIdentity, CONST UINT8 PrivateKey[NOISE_PUBLIC_KEY_LEN]) +{ + RtlCopyMemory(StaticIdentity->StaticPrivate, PrivateKey, NOISE_PUBLIC_KEY_LEN); + Curve25519ClampSecret(StaticIdentity->StaticPrivate); + StaticIdentity->HasIdentity = Curve25519GeneratePublic(StaticIdentity->StaticPublic, PrivateKey); +} + +_Use_decl_annotations_ +VOID +NoiseStaticIdentityClear(NOISE_STATIC_IDENTITY *StaticIdentity) +{ + MuAcquirePushLockExclusive(&StaticIdentity->Lock); + RtlSecureZeroMemory(&StaticIdentity->StaticPublic, NOISE_PUBLIC_KEY_LEN); + RtlSecureZeroMemory(&StaticIdentity->StaticPrivate, NOISE_PUBLIC_KEY_LEN); + StaticIdentity->HasIdentity = FALSE; + MuReleasePushLockExclusive(&StaticIdentity->Lock); +} + +/* This is Hugo Krawczyk's HKDF: + * - https://eprint.iacr.org/2010/264.pdf + * - https://tools.ietf.org/html/rfc5869 + */ +static VOID +Kdf(_Out_writes_bytes_all_opt_(FirstLen) UINT8 *FirstDst, + _Out_writes_bytes_all_opt_(SecondLen) UINT8 *SecondDst, + _Out_writes_bytes_all_opt_(ThirdLen) UINT8 *ThirdDst, + _In_reads_bytes_(DataLen) CONST UINT8 *Data, + _In_ CONST SIZE_T FirstLen, + _In_ CONST SIZE_T SecondLen, + _In_ CONST SIZE_T ThirdLen, + _In_ CONST SIZE_T DataLen, + _In_ CONST UINT8 ChainingKey[NOISE_HASH_LEN]) +{ + UINT8 Output[BLAKE2S_HASH_SIZE + 1]; + UINT8 Secret[BLAKE2S_HASH_SIZE]; + NT_ASSERT( + !(FirstLen > BLAKE2S_HASH_SIZE || SecondLen > BLAKE2S_HASH_SIZE || ThirdLen > BLAKE2S_HASH_SIZE || + ((SecondLen || SecondDst || ThirdLen || ThirdDst) && (!FirstLen || !FirstDst)) || + ((ThirdLen || ThirdDst) && (!SecondLen || !SecondDst)))); + _Analysis_assume_(RtlFillMemory(Output, sizeof(Output), 'A')); + + /* Extract entropy from data into secret */ + Blake2s256Hmac(Secret, Data, ChainingKey, DataLen, NOISE_HASH_LEN); + + if (!FirstDst || !FirstLen) + goto out; + + /* Expand first key: key = secret, data = 0x1 */ + Output[0] = 1; + Blake2s256Hmac(Output, Output, Secret, 1, BLAKE2S_HASH_SIZE); + RtlCopyMemory(FirstDst, Output, FirstLen); + + if (!SecondDst || !SecondLen) + goto out; + + /* Expand second key: key = secret, data = first-key || 0x2 */ + Output[BLAKE2S_HASH_SIZE] = 2; + Blake2s256Hmac(Output, Output, Secret, BLAKE2S_HASH_SIZE + 1, BLAKE2S_HASH_SIZE); + RtlCopyMemory(SecondDst, Output, SecondLen); + + if (!ThirdDst || !ThirdLen) + goto out; + + /* Expand third key: key = secret, data = second-key || 0x3 */ + Output[BLAKE2S_HASH_SIZE] = 3; + Blake2s256Hmac(Output, Output, Secret, BLAKE2S_HASH_SIZE + 1, BLAKE2S_HASH_SIZE); + RtlCopyMemory(ThirdDst, Output, ThirdLen); + +out: + /* Clear sensitive data from stack */ + RtlSecureZeroMemory(Secret, BLAKE2S_HASH_SIZE); + RtlSecureZeroMemory(Output, BLAKE2S_HASH_SIZE + 1); +} + +static VOID +DeriveKeys( + _Out_ NOISE_SYMMETRIC_KEY *FirstDst, + _Out_ NOISE_SYMMETRIC_KEY *SecondDst, + _In_ CONST UINT8 ChainingKey[NOISE_HASH_LEN]) +{ + UINT64 Birthdate = KeQueryInterruptTime(); + Kdf(FirstDst->Key, SecondDst->Key, NULL, NULL, NOISE_SYMMETRIC_KEY_LEN, NOISE_SYMMETRIC_KEY_LEN, 0, 0, ChainingKey); + FirstDst->Birthdate = SecondDst->Birthdate = Birthdate; + FirstDst->IsValid = SecondDst->IsValid = TRUE; +} + +_Must_inspect_result_ +_Return_type_success_(return != FALSE) +static BOOLEAN +MixDh( + _Inout_updates_all_(NOISE_HASH_LEN) UINT8 ChainingKey[NOISE_HASH_LEN], + _Out_writes_bytes_all_opt_(NOISE_SYMMETRIC_KEY_LEN) UINT8 Key[NOISE_SYMMETRIC_KEY_LEN], + _In_count_(NOISE_PUBLIC_KEY_LEN) CONST UINT8 Private[NOISE_PUBLIC_KEY_LEN], + _In_count_(NOISE_PUBLIC_KEY_LEN) CONST UINT8 Public[NOISE_PUBLIC_KEY_LEN]) +{ + UINT8 DhCalculation[NOISE_PUBLIC_KEY_LEN]; + + if (!Curve25519(DhCalculation, Private, Public)) + return FALSE; + Kdf(ChainingKey, + Key, + NULL, + DhCalculation, + NOISE_HASH_LEN, + NOISE_SYMMETRIC_KEY_LEN, + 0, + NOISE_PUBLIC_KEY_LEN, + ChainingKey); + RtlSecureZeroMemory(DhCalculation, NOISE_PUBLIC_KEY_LEN); + return TRUE; +} + +_Must_inspect_result_ +_Return_type_success_(return != FALSE) +static BOOLEAN +MixPrecomputedDh( + _Inout_updates_all_(NOISE_HASH_LEN) UINT8 ChainingKey[NOISE_HASH_LEN], + _Out_writes_bytes_all_opt_(NOISE_SYMMETRIC_KEY_LEN) UINT8 Key[NOISE_SYMMETRIC_KEY_LEN], + _In_reads_bytes_(NOISE_PUBLIC_KEY_LEN) CONST UINT8 Precomputed[NOISE_PUBLIC_KEY_LEN]) +{ + if (Curve25519IsNull(Precomputed)) + return FALSE; + Kdf(ChainingKey, + Key, + NULL, + Precomputed, + NOISE_HASH_LEN, + NOISE_SYMMETRIC_KEY_LEN, + 0, + NOISE_PUBLIC_KEY_LEN, + ChainingKey); + return TRUE; +} + +static VOID +MixHash( + _Inout_updates_all_(NOISE_HASH_LEN) UINT8 Hash[NOISE_HASH_LEN], + _In_reads_bytes_(SrcLen) CONST UINT8 *Src, + _In_ SIZE_T SrcLen) +{ + BLAKE2S_STATE Blake; + + Blake2sInit(&Blake, NOISE_HASH_LEN); + Blake2sUpdate(&Blake, Hash, NOISE_HASH_LEN); + Blake2sUpdate(&Blake, Src, SrcLen); + Blake2sFinal(&Blake, Hash); +} + +static VOID +MixPsk( + _Inout_updates_all_(NOISE_HASH_LEN) UINT8 ChainingKey[NOISE_HASH_LEN], + _Inout_updates_all_(NOISE_HASH_LEN) UINT8 Hash[NOISE_HASH_LEN], + _Out_writes_bytes_all_opt_(NOISE_SYMMETRIC_KEY_LEN) UINT8 Key[NOISE_SYMMETRIC_KEY_LEN], + _In_reads_bytes_(NOISE_SYMMETRIC_KEY_LEN) CONST UINT8 Psk[NOISE_SYMMETRIC_KEY_LEN]) +{ + UINT8 TempHash[NOISE_HASH_LEN]; + + Kdf(ChainingKey, + TempHash, + Key, + Psk, + NOISE_HASH_LEN, + NOISE_HASH_LEN, + NOISE_SYMMETRIC_KEY_LEN, + NOISE_SYMMETRIC_KEY_LEN, + ChainingKey); + MixHash(Hash, TempHash, NOISE_HASH_LEN); + RtlSecureZeroMemory(TempHash, NOISE_HASH_LEN); +} + +static VOID +HandshakeInit( + _Out_writes_bytes_all_(NOISE_HASH_LEN) UINT8 ChainingKey[NOISE_HASH_LEN], + _Out_writes_bytes_all_(NOISE_HASH_LEN) UINT8 Hash[NOISE_HASH_LEN], + _In_reads_bytes_(NOISE_PUBLIC_KEY_LEN) CONST UINT8 RemoteStatic[NOISE_PUBLIC_KEY_LEN]) +{ + RtlCopyMemory(Hash, HandshakeInitHash, NOISE_HASH_LEN); + RtlCopyMemory(ChainingKey, HandshakeInitChainingKey, NOISE_HASH_LEN); + MixHash(Hash, RemoteStatic, NOISE_PUBLIC_KEY_LEN); +} + +static VOID +MessageEncrypt( + _Out_writes_bytes_all_(SrcLen + CHACHA20POLY1305_AUTHTAG_SIZE) UINT8 *DstCiphertext, + _In_reads_bytes_(SrcLen) CONST UINT8 *SrcPlaintext, + _In_ SIZE_T SrcLen, + _In_ CONST UINT8 Key[NOISE_SYMMETRIC_KEY_LEN], + _Inout_updates_all_(NOISE_HASH_LEN) UINT8 Hash[NOISE_HASH_LEN]) +{ + ChaCha20Poly1305Encrypt( + DstCiphertext, SrcPlaintext, SrcLen, Hash, NOISE_HASH_LEN, 0 /* Always zero for Noise_IK */, Key); + MixHash(Hash, DstCiphertext, NoiseEncryptedLen(SrcLen)); +} + +_Must_inspect_result_ +_Return_type_success_(return != FALSE) +static BOOLEAN +MessageDecrypt( + _Out_writes_bytes_all_(SrcLen - CHACHA20POLY1305_AUTHTAG_SIZE) UINT8 *DstPlaintext, + _In_reads_bytes_(SrcLen) CONST UINT8 *SrcCiphertext, + _In_ SIZE_T SrcLen, + _In_ CONST UINT8 Key[NOISE_SYMMETRIC_KEY_LEN], + _Inout_updates_all_(NOISE_HASH_LEN) UINT8 Hash[NOISE_HASH_LEN]) +{ + if (!ChaCha20Poly1305Decrypt( + DstPlaintext, SrcCiphertext, SrcLen, Hash, NOISE_HASH_LEN, 0 /* Always zero for Noise_IK */, Key)) + return FALSE; + MixHash(Hash, SrcCiphertext, SrcLen); + return TRUE; +} + +static VOID +MessageEphemeral( + _Out_writes_bytes_all_(NOISE_PUBLIC_KEY_LEN) UINT8 EphemeralDst[NOISE_PUBLIC_KEY_LEN], + _In_reads_bytes_(NOISE_PUBLIC_KEY_LEN) CONST UINT8 EphemeralSrc[NOISE_PUBLIC_KEY_LEN], + _Inout_updates_all_(NOISE_HASH_LEN) UINT8 ChainingKey[NOISE_HASH_LEN], + _Inout_updates_all_(NOISE_HASH_LEN) UINT8 Hash[NOISE_HASH_LEN]) +{ + if (EphemeralDst != EphemeralSrc) + RtlCopyMemory(EphemeralDst, EphemeralSrc, NOISE_PUBLIC_KEY_LEN); + else + _Analysis_assume_((RtlCopyMemory(EphemeralDst, EphemeralSrc, NOISE_PUBLIC_KEY_LEN), TRUE)); + MixHash(Hash, EphemeralSrc, NOISE_PUBLIC_KEY_LEN); + Kdf(ChainingKey, NULL, NULL, EphemeralSrc, NOISE_HASH_LEN, 0, 0, NOISE_PUBLIC_KEY_LEN, ChainingKey); +} + +static VOID +Tai64nNow(_Out_writes_bytes_all_(NOISE_TIMESTAMP_LEN) UINT8 Output[NOISE_TIMESTAMP_LEN]) +{ + LARGE_INTEGER Now; + INT64 Sec; + LONG Nsec; + + KeQuerySystemTime(&Now); + Now.QuadPart -= 11644473600LL * SYS_TIME_UNITS_PER_SEC; + + /* In order to prevent some sort of infoleak from precise timers, we + * round down the time to the closest rounded-down power of + * two to the maximum initiations per second allowed anyway by the + * implementation. + */ + Now.QuadPart = + ALIGN_DOWN_BY_T(INT64, Now.QuadPart, RounddownPowOfTwo(SYS_TIME_UNITS_PER_SEC / INITIATIONS_PER_SECOND)); + Sec = Now.QuadPart / SYS_TIME_UNITS_PER_SEC; + Nsec = (LONG)(Now.QuadPart % SYS_TIME_UNITS_PER_SEC) * (1000000000 / SYS_TIME_UNITS_PER_SEC); + + /* https://cr.yp.to/libtai/tai64.html */ + *(UINT64_BE *)Output = CpuToBe64(0x400000000000000aULL + Sec); + *(UINT32_BE *)(Output + sizeof(UINT64_BE)) = CpuToBe32(Nsec); +} + +_Use_decl_annotations_ +BOOLEAN +NoiseHandshakeCreateInitiation(MESSAGE_HANDSHAKE_INITIATION *Dst, NOISE_HANDSHAKE *Handshake) +{ + UINT8 Timestamp[NOISE_TIMESTAMP_LEN]; + UINT8 Key[NOISE_SYMMETRIC_KEY_LEN]; + BOOLEAN Ret = FALSE; + + MuAcquirePushLockShared(&Handshake->StaticIdentity->Lock); + MuAcquirePushLockExclusive(&Handshake->Lock); + + if (!Handshake->StaticIdentity->HasIdentity) + goto out; + + Dst->Header.Type = CpuToLe32(MESSAGE_TYPE_HANDSHAKE_INITIATION); + + HandshakeInit(Handshake->ChainingKey, Handshake->Hash, Handshake->RemoteStatic); + + /* e */ + Curve25519GenerateSecret(Handshake->EphemeralPrivate); + if (!Curve25519GeneratePublic(Dst->UnencryptedEphemeral, Handshake->EphemeralPrivate)) + goto out; + MessageEphemeral(Dst->UnencryptedEphemeral, Dst->UnencryptedEphemeral, Handshake->ChainingKey, Handshake->Hash); + + /* es */ + if (!MixDh(Handshake->ChainingKey, Key, Handshake->EphemeralPrivate, Handshake->RemoteStatic)) + goto out; + + /* s */ + MessageEncrypt( + Dst->EncryptedStatic, Handshake->StaticIdentity->StaticPublic, NOISE_PUBLIC_KEY_LEN, Key, Handshake->Hash); + + /* ss */ + if (!MixPrecomputedDh(Handshake->ChainingKey, Key, Handshake->PrecomputedStaticStatic)) + goto out; + + /* {t} */ + Tai64nNow(Timestamp); + MessageEncrypt(Dst->EncryptedTimestamp, Timestamp, NOISE_TIMESTAMP_LEN, Key, Handshake->Hash); + + Dst->SenderIndex = IndexHashtableInsert(Handshake->Entry.Peer->Device->IndexHashtable, &Handshake->Entry); + + Handshake->State = HANDSHAKE_CREATED_INITIATION; + Ret = TRUE; + +out: + MuReleasePushLockExclusive(&Handshake->Lock); + MuReleasePushLockShared(&Handshake->StaticIdentity->Lock); + RtlSecureZeroMemory(Key, NOISE_SYMMETRIC_KEY_LEN); + return Ret; +} + +_Use_decl_annotations_ +WG_PEER * +NoiseHandshakeConsumeInitiation(CONST MESSAGE_HANDSHAKE_INITIATION *Src, WG_DEVICE *Wg) +{ + WG_PEER *Peer = NULL, *RetPeer = NULL; + NOISE_HANDSHAKE *Handshake; + BOOLEAN ReplayAttack, FloodAttack; + UINT8 Key[NOISE_SYMMETRIC_KEY_LEN]; + UINT8 ChainingKey[NOISE_HASH_LEN]; + UINT8 Hash[NOISE_HASH_LEN]; + UINT8 S[NOISE_PUBLIC_KEY_LEN]; + UINT8 E[NOISE_PUBLIC_KEY_LEN]; + UINT8 T[NOISE_TIMESTAMP_LEN]; + UINT64 InitiationConsumption; + + MuAcquirePushLockShared(&Wg->StaticIdentity.Lock); + if (!Wg->StaticIdentity.HasIdentity) + goto out; + + HandshakeInit(ChainingKey, Hash, Wg->StaticIdentity.StaticPublic); + + /* e */ + MessageEphemeral(E, Src->UnencryptedEphemeral, ChainingKey, Hash); + + /* es */ + if (!MixDh(ChainingKey, Key, Wg->StaticIdentity.StaticPrivate, E)) + goto out; + + /* s */ + if (!MessageDecrypt(S, Src->EncryptedStatic, sizeof(Src->EncryptedStatic), Key, Hash)) + goto out; + + /* Lookup which peer we're actually talking to */ + Peer = PubkeyHashtableLookup(Wg->PeerHashtable, S); + if (!Peer) + goto out; + Handshake = &Peer->Handshake; + + /* ss */ + if (!MixPrecomputedDh(ChainingKey, Key, Handshake->PrecomputedStaticStatic)) + goto out; + + /* {t} */ + if (!MessageDecrypt(T, Src->EncryptedTimestamp, sizeof(Src->EncryptedTimestamp), Key, Hash)) + goto out; + + MuAcquirePushLockShared(&Handshake->Lock); + ReplayAttack = memcmp(T, Handshake->LatestTimestamp, NOISE_TIMESTAMP_LEN) <= 0; + FloodAttack = (INT64)Handshake->LastInitiationConsumption + SYS_TIME_UNITS_PER_SEC / INITIATIONS_PER_SECOND > + (INT64)KeQueryInterruptTime(); + MuReleasePushLockShared(&Handshake->Lock); + if (ReplayAttack || FloodAttack) + goto out; + + /* Success! Copy everything to peer */ + MuAcquirePushLockExclusive(&Handshake->Lock); + RtlCopyMemory(Handshake->RemoteEphemeral, E, NOISE_PUBLIC_KEY_LEN); + if (memcmp(T, Handshake->LatestTimestamp, NOISE_TIMESTAMP_LEN) > 0) + RtlCopyMemory(Handshake->LatestTimestamp, T, NOISE_TIMESTAMP_LEN); + RtlCopyMemory(Handshake->Hash, Hash, NOISE_HASH_LEN); + RtlCopyMemory(Handshake->ChainingKey, ChainingKey, NOISE_HASH_LEN); + Handshake->RemoteIndex = Src->SenderIndex; + InitiationConsumption = KeQueryInterruptTime(); + if ((INT64)(Handshake->LastInitiationConsumption - InitiationConsumption) < 0) + Handshake->LastInitiationConsumption = InitiationConsumption; + Handshake->State = HANDSHAKE_CONSUMED_INITIATION; + MuReleasePushLockExclusive(&Handshake->Lock); + RetPeer = Peer; + +out: + RtlSecureZeroMemory(Key, NOISE_SYMMETRIC_KEY_LEN); + RtlSecureZeroMemory(Hash, NOISE_HASH_LEN); + RtlSecureZeroMemory(ChainingKey, NOISE_HASH_LEN); + MuReleasePushLockShared(&Wg->StaticIdentity.Lock); + if (!RetPeer) + PeerPut(Peer); + return RetPeer; +} + +_Use_decl_annotations_ +BOOLEAN +NoiseHandshakeCreateResponse(MESSAGE_HANDSHAKE_RESPONSE *Dst, NOISE_HANDSHAKE *Handshake) +{ + UINT8 Key[NOISE_SYMMETRIC_KEY_LEN]; + BOOLEAN Ret = FALSE; + + MuAcquirePushLockShared(&Handshake->StaticIdentity->Lock); + MuAcquirePushLockExclusive(&Handshake->Lock); + + if (Handshake->State != HANDSHAKE_CONSUMED_INITIATION) + goto out; + + Dst->Header.Type = CpuToLe32(MESSAGE_TYPE_HANDSHAKE_RESPONSE); + Dst->ReceiverIndex = Handshake->RemoteIndex; + + /* e */ + Curve25519GenerateSecret(Handshake->EphemeralPrivate); + if (!Curve25519GeneratePublic(Dst->UnencryptedEphemeral, Handshake->EphemeralPrivate)) + goto out; + MessageEphemeral(Dst->UnencryptedEphemeral, Dst->UnencryptedEphemeral, Handshake->ChainingKey, Handshake->Hash); + + /* ee */ + if (!MixDh(Handshake->ChainingKey, NULL, Handshake->EphemeralPrivate, Handshake->RemoteEphemeral)) + goto out; + + /* se */ + if (!MixDh(Handshake->ChainingKey, NULL, Handshake->EphemeralPrivate, Handshake->RemoteStatic)) + goto out; + + /* psk */ + MixPsk(Handshake->ChainingKey, Handshake->Hash, Key, Handshake->PresharedKey); + + /* {} */ + MessageEncrypt(Dst->EncryptedNothing, NULL, 0, Key, Handshake->Hash); + + Dst->SenderIndex = IndexHashtableInsert(Handshake->Entry.Peer->Device->IndexHashtable, &Handshake->Entry); + + Handshake->State = HANDSHAKE_CREATED_RESPONSE; + Ret = TRUE; + +out: + MuReleasePushLockExclusive(&Handshake->Lock); + MuReleasePushLockShared(&Handshake->StaticIdentity->Lock); + RtlSecureZeroMemory(Key, NOISE_SYMMETRIC_KEY_LEN); + return Ret; +} + +_Use_decl_annotations_ +WG_PEER * +NoiseHandshakeConsumeResponse(CONST MESSAGE_HANDSHAKE_RESPONSE *Src, WG_DEVICE *Wg) +{ + NOISE_HANDSHAKE_STATE State = HANDSHAKE_ZEROED; + WG_PEER *Peer = NULL, *RetPeer = NULL; + NOISE_HANDSHAKE *Handshake; + UINT8 Key[NOISE_SYMMETRIC_KEY_LEN]; + UINT8 Hash[NOISE_HASH_LEN]; + UINT8 ChainingKey[NOISE_HASH_LEN]; + UINT8 E[NOISE_PUBLIC_KEY_LEN]; + UINT8 EphemeralPrivate[NOISE_PUBLIC_KEY_LEN]; + UINT8 StaticPrivate[NOISE_PUBLIC_KEY_LEN]; + UINT8 PresharedKey[NOISE_SYMMETRIC_KEY_LEN]; + + MuAcquirePushLockShared(&Wg->StaticIdentity.Lock); + + if (!Wg->StaticIdentity.HasIdentity) + goto out; + + Handshake = (NOISE_HANDSHAKE *)IndexHashtableLookup( + Wg->IndexHashtable, INDEX_HASHTABLE_HANDSHAKE, Src->ReceiverIndex, &Peer); + if (!Handshake) + goto out; + + MuAcquirePushLockShared(&Handshake->Lock); + State = Handshake->State; + RtlCopyMemory(Hash, Handshake->Hash, NOISE_HASH_LEN); + RtlCopyMemory(ChainingKey, Handshake->ChainingKey, NOISE_HASH_LEN); + RtlCopyMemory(EphemeralPrivate, Handshake->EphemeralPrivate, NOISE_PUBLIC_KEY_LEN); + RtlCopyMemory(PresharedKey, Handshake->PresharedKey, NOISE_SYMMETRIC_KEY_LEN); + MuReleasePushLockShared(&Handshake->Lock); + + if (State != HANDSHAKE_CREATED_INITIATION) + goto fail; + + /* e */ + MessageEphemeral(E, Src->UnencryptedEphemeral, ChainingKey, Hash); + + /* ee */ + if (!MixDh(ChainingKey, NULL, EphemeralPrivate, E)) + goto fail; + + /* se */ + if (!MixDh(ChainingKey, NULL, Wg->StaticIdentity.StaticPrivate, E)) + goto fail; + + /* psk */ + MixPsk(ChainingKey, Hash, Key, PresharedKey); + + /* {} */ + if (!MessageDecrypt(NULL, Src->EncryptedNothing, sizeof(Src->EncryptedNothing), Key, Hash)) + goto fail; + + /* Success! Copy everything to peer */ + MuAcquirePushLockExclusive(&Handshake->Lock); + /* It's important to check that the state is still the same, while we + * have an exclusive lock. + */ + if (Handshake->State != State) + { + MuReleasePushLockExclusive(&Handshake->Lock); + goto fail; + } + RtlCopyMemory(Handshake->RemoteEphemeral, E, NOISE_PUBLIC_KEY_LEN); + RtlCopyMemory(Handshake->Hash, Hash, NOISE_HASH_LEN); + RtlCopyMemory(Handshake->ChainingKey, ChainingKey, NOISE_HASH_LEN); + Handshake->RemoteIndex = Src->SenderIndex; + Handshake->State = HANDSHAKE_CONSUMED_RESPONSE; + MuReleasePushLockExclusive(&Handshake->Lock); + RetPeer = Peer; + goto out; + +fail: + PeerPut(Peer); +out: + RtlSecureZeroMemory(Key, NOISE_SYMMETRIC_KEY_LEN); + RtlSecureZeroMemory(Hash, NOISE_HASH_LEN); + RtlSecureZeroMemory(ChainingKey, NOISE_HASH_LEN); + RtlSecureZeroMemory(EphemeralPrivate, NOISE_PUBLIC_KEY_LEN); + RtlSecureZeroMemory(StaticPrivate, NOISE_PUBLIC_KEY_LEN); + RtlSecureZeroMemory(PresharedKey, NOISE_SYMMETRIC_KEY_LEN); + MuReleasePushLockShared(&Wg->StaticIdentity.Lock); + return RetPeer; +} + +_Use_decl_annotations_ +BOOLEAN +NoiseHandshakeBeginSession(NOISE_HANDSHAKE *Handshake, NOISE_KEYPAIRS *Keypairs) +{ + NOISE_KEYPAIR *NewKeypair; + BOOLEAN Ret = FALSE; + + MuAcquirePushLockExclusive(&Handshake->Lock); + if (Handshake->State != HANDSHAKE_CREATED_RESPONSE && Handshake->State != HANDSHAKE_CONSUMED_RESPONSE) + goto out; + + NewKeypair = KeypairCreate(Handshake->Entry.Peer); + if (!NewKeypair) + goto out; + NewKeypair->IAmTheInitiator = Handshake->State == HANDSHAKE_CONSUMED_RESPONSE; + NewKeypair->RemoteIndex = Handshake->RemoteIndex; + + if (NewKeypair->IAmTheInitiator) + DeriveKeys(&NewKeypair->Sending, &NewKeypair->Receiving, Handshake->ChainingKey); + else + DeriveKeys(&NewKeypair->Receiving, &NewKeypair->Sending, Handshake->ChainingKey); + + HandshakeZero(Handshake); + if (ExAcquireRundownProtection(&CONTAINING_RECORD(Handshake, WG_PEER, Handshake)->InUse)) + { + AddNewKeypair(Keypairs, NewKeypair); + LogInfoRatelimited( + Handshake->Entry.Peer->Device, + "Keypair %llu created for peer %llu", + NewKeypair->InternalId, + Handshake->Entry.Peer->InternalId); + Ret = + IndexHashtableReplace(Handshake->Entry.Peer->Device->IndexHashtable, &Handshake->Entry, &NewKeypair->Entry); + ExReleaseRundownProtection(&CONTAINING_RECORD(Handshake, WG_PEER, Handshake)->InUse); + } + else + MemFreeSensitive(NewKeypair, sizeof(NOISE_KEYPAIR)); + +out: + MuReleasePushLockExclusive(&Handshake->Lock); + return Ret; +} diff --git a/driver/noise.h b/driver/noise.h new file mode 100644 index 0000000..6da41fd --- /dev/null +++ b/driver/noise.h @@ -0,0 +1,196 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +#include "interlocked.h" +#include "messages.h" +#include "peerlookup.h" +#include "timers.h" +#include "rcu.h" + +typedef struct _NOISE_REPLAY_COUNTER +{ + UINT64 Counter; + KSPIN_LOCK Lock; + ULONG_PTR Backtrack[COUNTER_BITS_TOTAL / BITS_PER_POINTER]; +} NOISE_REPLAY_COUNTER; + +typedef struct _NOISE_SYMMETRIC_KEY +{ + UINT8 Key[NOISE_SYMMETRIC_KEY_LEN]; + UINT64 Birthdate; + BOOLEAN IsValid; +} NOISE_SYMMETRIC_KEY; + +typedef struct _NOISE_KEYPAIR +{ + INDEX_HASHTABLE_ENTRY Entry; + NOISE_SYMMETRIC_KEY Sending; + LONG64 SendingCounter; + NOISE_SYMMETRIC_KEY Receiving; + NOISE_REPLAY_COUNTER ReceivingCounter; + UINT32_LE RemoteIndex; + BOOLEAN IAmTheInitiator; + KREF Refcount; + RCU_CALLBACK Rcu; + UINT64 InternalId; +} NOISE_KEYPAIR; + +typedef struct _NOISE_KEYPAIRS +{ + NOISE_KEYPAIR __rcu *CurrentKeypair; + NOISE_KEYPAIR __rcu *PreviousKeypair; + NOISE_KEYPAIR __rcu *NextKeypair; + KSPIN_LOCK KeypairUpdateLock; +} NOISE_KEYPAIRS; + +typedef struct _NOISE_STATIC_IDENTITY +{ + UINT8 StaticPublic[NOISE_PUBLIC_KEY_LEN]; + UINT8 StaticPrivate[NOISE_PUBLIC_KEY_LEN]; + EX_PUSH_LOCK Lock; + BOOLEAN HasIdentity; +} NOISE_STATIC_IDENTITY; + +typedef enum _NOISE_HANDSHAKE_STATE +{ + HANDSHAKE_ZEROED, + HANDSHAKE_CREATED_INITIATION, + HANDSHAKE_CONSUMED_INITIATION, + HANDSHAKE_CREATED_RESPONSE, + HANDSHAKE_CONSUMED_RESPONSE +} NOISE_HANDSHAKE_STATE; + +typedef struct _NOISE_HANDSHAKE +{ + INDEX_HASHTABLE_ENTRY Entry; + + NOISE_HANDSHAKE_STATE State; + UINT64 LastInitiationConsumption; + + NOISE_STATIC_IDENTITY *StaticIdentity; + + UINT8 EphemeralPrivate[NOISE_PUBLIC_KEY_LEN]; + UINT8 RemoteStatic[NOISE_PUBLIC_KEY_LEN]; + UINT8 RemoteEphemeral[NOISE_PUBLIC_KEY_LEN]; + UINT8 PrecomputedStaticStatic[NOISE_PUBLIC_KEY_LEN]; + + UINT8 PresharedKey[NOISE_SYMMETRIC_KEY_LEN]; + + UINT8 Hash[NOISE_HASH_LEN]; + UINT8 ChainingKey[NOISE_HASH_LEN]; + + UINT8 LatestTimestamp[NOISE_TIMESTAMP_LEN]; + UINT32_LE RemoteIndex; + + /* Protects all members except the immutable (after noise_handshake_ + * init): remote_static, precomputed_static_static, static_identity. + */ + EX_PUSH_LOCK Lock; +} NOISE_HANDSHAKE; + +typedef struct _WG_DEVICE WG_DEVICE; + +VOID NoiseDriverEntry(VOID); + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_held_(Peer->Device->DeviceUpdateLock) +VOID +NoiseHandshakeInit( + _Out_ NOISE_HANDSHAKE *Handshake, + _In_ NOISE_STATIC_IDENTITY *StaticIdentity, + _In_ CONST UINT8 PeerPublicKey[NOISE_PUBLIC_KEY_LEN], + _In_ CONST UINT8 PeerPresharedKey[NOISE_SYMMETRIC_KEY_LEN], + _In_ WG_PEER *Peer); + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_not_held_(Handshake->Lock) +VOID +NoiseHandshakeClear(_Inout_ NOISE_HANDSHAKE *Handshake); + +static inline VOID +NoiseResetLastSentHandshake(_Out_ _Interlocked_operand_ LONG64 *HandshakeSysTimeUnits) +{ + WriteNoFence64( + HandshakeSysTimeUnits, KeQueryInterruptTime() - (UINT64)(REKEY_TIMEOUT + 1) * SYS_TIME_UNITS_PER_SEC); +} + +VOID +NoiseKeypairPut(_In_opt_ NOISE_KEYPAIR *Keypair, _In_ BOOLEAN UnreferenceNow); + +_Must_inspect_result_ +_Post_maybenull_ +NOISE_KEYPAIR * +NoiseKeypairGet(_Inout_opt_ NOISE_KEYPAIR *Keypair); + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Requires_lock_not_held_(Keypairs->KeypairUpdateLock) +VOID +NoiseKeypairsClear(_Inout_ NOISE_KEYPAIRS *Keypairs); + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Requires_lock_not_held_(Keypairs->KeypairUpdateLock) +BOOLEAN +NoiseReceivedWithKeypair(_Inout_ NOISE_KEYPAIRS *Keypairs, _In_ NOISE_KEYPAIR *ReceivedKeypair); + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_not_held_(Peer->Keypairs.KeypairUpdateLock) +VOID +NoiseExpireCurrentPeerKeypairs(_Inout_ WG_PEER *Peer); + +_Requires_exclusive_lock_held_(StaticIdentity->Lock) +VOID +NoiseSetStaticIdentityPrivateKey( + _Inout_ NOISE_STATIC_IDENTITY *StaticIdentity, + _In_ CONST UINT8 PrivateKey[NOISE_PUBLIC_KEY_LEN]); + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_not_held_(StaticIdentity->Lock) +VOID +NoiseStaticIdentityClear(_Inout_ NOISE_STATIC_IDENTITY *StaticIdentity); + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_held_(Peer->Device->DeviceUpdateLock) +_Requires_lock_not_held_(Peer->Handshake.Lock) +VOID +NoisePrecomputeStaticStatic(_Inout_ WG_PEER *Peer); + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_not_held_(Handshake->StaticIdentity->Lock) +_Requires_lock_not_held_(Handshake->Lock) +_Must_inspect_result_ +_Return_type_success_(return != FALSE) +BOOLEAN +NoiseHandshakeCreateInitiation(_Out_ MESSAGE_HANDSHAKE_INITIATION *Dst, _Inout_ NOISE_HANDSHAKE *Handshake); + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_not_held_(Wg->StaticIdentity.Lock) +_Must_inspect_result_ +_Return_type_success_(return != NULL) +WG_PEER * +NoiseHandshakeConsumeInitiation(_In_ CONST MESSAGE_HANDSHAKE_INITIATION *Src, _Inout_ WG_DEVICE *Wg); + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_not_held_(Handshake->StaticIdentity->Lock) +_Requires_lock_not_held_(Handshake->Lock) +_Must_inspect_result_ +_Return_type_success_(return != FALSE) +BOOLEAN +NoiseHandshakeCreateResponse(_Out_ MESSAGE_HANDSHAKE_RESPONSE *Dst, _Inout_ NOISE_HANDSHAKE *Handshake); + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_not_held_(Wg->StaticIdentity.Lock) +_Must_inspect_result_ +_Return_type_success_(return != NULL) +WG_PEER * +NoiseHandshakeConsumeResponse(_In_ CONST MESSAGE_HANDSHAKE_RESPONSE *Src, _Inout_ WG_DEVICE *Wg); + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_not_held_(Handshake->Lock) +_Must_inspect_result_ +_Return_type_success_(return != FALSE) +BOOLEAN +NoiseHandshakeBeginSession(_Inout_ NOISE_HANDSHAKE *Handshake, _Inout_ NOISE_KEYPAIRS *Keypairs); diff --git a/driver/peer.c b/driver/peer.c new file mode 100644 index 0000000..690e211 --- /dev/null +++ b/driver/peer.c @@ -0,0 +1,170 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include "interlocked.h" +#include "containers.h" +#include "device.h" +#include "noise.h" +#include "peer.h" +#include "peerlookup.h" +#include "queueing.h" +#include "timers.h" +#include "logging.h" + +static LOOKASIDE_ALIGN LOOKASIDE_LIST_EX PeerCache; +static LONG64 PeerCounter = 0; + +_Use_decl_annotations_ +NTSTATUS +PeerCreate( + WG_DEVICE *Wg, + CONST UINT8 PublicKey[NOISE_PUBLIC_KEY_LEN], + CONST UINT8 PresharedKey[NOISE_SYMMETRIC_KEY_LEN], + WG_PEER **Peer) +{ + if (Wg->NumPeers >= MAX_PEERS_PER_DEVICE) + return STATUS_TOO_MANY_NODES; + + *Peer = ExAllocateFromLookasideListEx(&PeerCache); + if (!*Peer) + return STATUS_INSUFFICIENT_RESOURCES; + RtlZeroMemory(*Peer, sizeof(**Peer)); + + (*Peer)->Device = Wg; + NoiseHandshakeInit(&(*Peer)->Handshake, &Wg->StaticIdentity, PublicKey, PresharedKey, *Peer); + (*Peer)->InternalId = InterlockedIncrement64(&PeerCounter); + CookieInit(&(*Peer)->LatestCookie); + TimersInit(*Peer); + CookieCheckerPrecomputePeerKeys(*Peer); + KeInitializeSpinLock(&(*Peer)->Keypairs.KeypairUpdateLock); + PrevQueueInit(&(*Peer)->TxQueue); + PrevQueueInit(&(*Peer)->RxQueue); + KrefInit(&(*Peer)->Refcount); + NetBufferListInitQueue(&(*Peer)->StagedPacketQueue); + ExInitializeRundownProtection(&(*Peer)->InUse); + NoiseResetLastSentHandshake(&(*Peer)->LastSentHandshake); + InsertTailList(&Wg->PeerList, &(*Peer)->PeerList); + InitializeListHead(&(*Peer)->AllowedIpsList); + PubkeyHashtableAdd(Wg->PeerHashtable, *Peer); + ++Wg->NumPeers; + LogInfo(Wg, "Peer %llu created", (*Peer)->InternalId); + return STATUS_SUCCESS; +} + +_Use_decl_annotations_ +WG_PEER * +PeerGetMaybeZero(WG_PEER *Peer) +{ + if (!Peer || !KrefGetUnlessZero(&Peer->Refcount)) + return NULL; + return Peer; +} + +/* We have a separate "remove" function make sure that all active places where + * a peer is currently operating will eventually come to an end and not pass + * their reference onto another context. + */ +_Use_decl_annotations_ +VOID +PeerRemove(WG_PEER *Peer) +{ + if (!Peer) + return; + + /* Remove from configuration-time lookup structures. */ + RemoveEntryList(&Peer->PeerList); + InitializeListHead(&Peer->PeerList); + AllowedIpsRemoveByPeer(&Peer->Device->PeerAllowedIps, Peer, &Peer->Device->DeviceUpdateLock); + PubkeyHashtableRemove(Peer->Device->PeerHashtable, Peer); + NoiseKeypairsClear(&Peer->Keypairs); + /* Disable creation of new references and wait for old ones to go away. */ + ExWaitForRundownProtectionRelease(&Peer->InUse); + /* Destroy all ongoing timers that were in-flight at the beginning of this function. */ + TimersStop(Peer); + + --Peer->Device->NumPeers; + PeerPut(Peer); +} + +_Use_decl_annotations_ +VOID +PeerRemoveAll(WG_DEVICE *Wg) +{ + WG_PEER *Peer, *Temp; + + /* Avoid having to traverse individually for each one. */ + AllowedIpsFree(&Wg->PeerAllowedIps, &Wg->DeviceUpdateLock); + + LIST_FOR_EACH_ENTRY_SAFE (Peer, Temp, &Wg->PeerList, WG_PEER, PeerList) + { + _Analysis_assume_same_lock_(Peer->Device->DeviceUpdateLock, Wg->DeviceUpdateLock); + PeerRemove(Peer); + } + RcuSynchronize(); +} + +static RCU_CALLBACK_FN RcuRelease; +_Use_decl_annotations_ +static VOID +RcuRelease(RCU_CALLBACK *Rcu) +{ + WG_PEER *Peer = CONTAINING_RECORD(Rcu, WG_PEER, Rcu); + + NT_ASSERT(!PrevQueuePeek(&Peer->TxQueue) && !PrevQueuePeek(&Peer->RxQueue)); + + /* The final zeroing takes care of clearing any remaining handshake key + * material and other potentially sensitive information. + */ + RtlSecureZeroMemory(Peer, sizeof(*Peer)); + ExFreeToLookasideListEx(&PeerCache, Peer); +} + +static VOID +KrefRelease(_In_ KREF *Refcount) +{ + WG_PEER *Peer = CONTAINING_RECORD(Refcount, WG_PEER, Refcount); + + CHAR EndpointName[SOCKADDR_STR_MAX_LEN]; + SockaddrToString(EndpointName, &Peer->Endpoint.Addr); + LogInfo(Peer->Device, "Peer %llu (%s) destroyed", Peer->InternalId, EndpointName); + + /* Remove ourself from dynamic runtime lookup structures, now that the + * last reference is gone. + */ + IndexHashtableRemove(Peer->Device->IndexHashtable, &Peer->Handshake.Entry); + + /* Remove any lingering packets that didn't have a chance to be + * transmitted. + */ + PacketPurgeStagedPackets(Peer); + + /* Free the memory used. */ + RcuCall(&Peer->Rcu, RcuRelease); +} + +_Use_decl_annotations_ +VOID +PeerPut(WG_PEER *Peer) +{ + if (!Peer) + return; + KrefPut(&Peer->Refcount, KrefRelease); +} + +#ifdef ALLOC_PRAGMA +# pragma alloc_text(INIT, PeerDriverEntry) +#endif +_Use_decl_annotations_ +NTSTATUS +PeerDriverEntry(VOID) +{ + return ExInitializeLookasideListEx(&PeerCache, NULL, NULL, NonPagedPool, 0, sizeof(WG_PEER), MEMORY_TAG, 0); +} + +_Use_decl_annotations_ +VOID PeerUnload(VOID) +{ + ExDeleteLookasideListEx(&PeerCache); +} diff --git a/driver/peer.h b/driver/peer.h new file mode 100644 index 0000000..5a80971 --- /dev/null +++ b/driver/peer.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +#include "interlocked.h" +#include "cookie.h" +#include "device.h" +#include "noise.h" +#include "rcu.h" +#include "timers.h" + +typedef struct _ENDPOINT +{ + SOCKADDR_INET Addr; + struct + { + union + { + WSACMSGHDR SrcCmsghdr; + CHAR __Alignment[WSA_CMSGDATA_ALIGN(sizeof(WSACMSGHDR))]; + }; + union + { + IN_PKTINFO Src4; + IN6_PKTINFO Src6; + }; + }; + UINT32 RoutingGeneration; + UINT32 UpdateGeneration; +} ENDPOINT; + +typedef enum _HANDSHAKE_TX_ACTION +{ + HANDSHAKE_TX_NONE = 0, + HANDSHAKE_TX_CLEAR, + HANDSHAKE_TX_SEND +} HANDSHAKE_TX_ACTION; + +typedef struct _WG_PEER +{ + WG_DEVICE *Device; + PREV_QUEUE TxQueue, RxQueue; + PEER_SERIAL_ENTRY TxSerialEntry, RxSerialEntry, HandshakeTxSerialEntry; + NET_BUFFER_LIST_QUEUE StagedPacketQueue; + EX_RUNDOWN_REF InUse; + NOISE_KEYPAIRS Keypairs; + ENDPOINT Endpoint; + EX_SPIN_LOCK EndpointLock; + NOISE_HANDSHAKE Handshake; + LONG64 LastSentHandshake; + COOKIE LatestCookie; + HLIST_NODE PubkeyHash; + UINT64 RxBytes, TxBytes; + TIMER TimerRetransmitHandshake, TimerSendKeepalive; + TIMER TimerNewHandshake, TimerZeroKeyMaterial; + TIMER TimerPersistentKeepalive; + ULONG TimerHandshakeAttempts; + UINT16 PersistentKeepaliveInterval; + SHORT HandshakeTxAction; + BOOLEAN TimerNeedAnotherKeepalive; + BOOLEAN SentLastminuteHandshake; + LARGE_INTEGER WalltimeLastHandshake; + KREF Refcount; + RCU_CALLBACK Rcu; + LIST_ENTRY PeerList; + LIST_ENTRY AllowedIpsList; + UINT64 InternalId; +} WG_PEER; + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Requires_lock_held_(Wg->DeviceUpdateLock) +_Must_inspect_result_ +NTSTATUS +PeerCreate( + _In_ WG_DEVICE *Wg, + _In_ CONST UINT8 PublicKey[NOISE_PUBLIC_KEY_LEN], + _In_ CONST UINT8 PresharedKey[NOISE_SYMMETRIC_KEY_LEN], + _Out_ WG_PEER **Peer); + +_Must_inspect_result_ +_Post_maybenull_ +WG_PEER * +PeerGetMaybeZero(_In_opt_ WG_PEER *Peer); + +_Post_notnull_ +static inline WG_PEER * +PeerGet(_In_ WG_PEER *Peer) +{ + KrefGet(&Peer->Refcount); + return Peer; +} + +VOID +PeerPut(_In_opt_ WG_PEER *Peer); + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_held_(Peer->Device->DeviceUpdateLock) +VOID +PeerRemove(_In_opt_ WG_PEER *Peer); + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_held_(Wg->DeviceUpdateLock) +VOID +PeerRemoveAll(_Inout_ WG_DEVICE *Wg); + +_IRQL_requires_max_(DISPATCH_LEVEL) +NTSTATUS +PeerDriverEntry(VOID); + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID PeerUnload(VOID); diff --git a/driver/peerlookup.c b/driver/peerlookup.c new file mode 100644 index 0000000..812060f --- /dev/null +++ b/driver/peerlookup.c @@ -0,0 +1,235 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include "containers.h" +#include "noise.h" +#include "peer.h" +#include "peerlookup.h" + +_Post_notnull_ +static HLIST_HEAD * +PubkeyBucket(_In_ PUBKEY_HASHTABLE *Table, _In_ CONST UINT8 Pubkey[NOISE_PUBLIC_KEY_LEN]) +{ + /* siphash gives us a secure 64bit number based on a random key. Since + * the bits are uniformly distributed, we can then mask off to get the + * bits we need. + */ + CONST UINT64 Hash = Siphash(Pubkey, NOISE_PUBLIC_KEY_LEN, &Table->Key); + + return &Table->Hashtable[Hash & (HASH_SIZE(Table->Hashtable) - 1)]; +} + +_Use_decl_annotations_ +PUBKEY_HASHTABLE *PubkeyHashtableAlloc(VOID) +{ + PUBKEY_HASHTABLE *Table = MemAllocate(sizeof(*Table)); + if (!Table) + return NULL; + + CryptoRandom(&Table->Key, sizeof(Table->Key)); + HashInit(Table->Hashtable); + MuInitializePushLock(&Table->Lock); + return Table; +} + +_Use_decl_annotations_ +VOID +PubkeyHashtableAdd(PUBKEY_HASHTABLE *Table, WG_PEER *Peer) +{ + MuAcquirePushLockExclusive(&Table->Lock); + HlistAddHeadRcu(&Peer->PubkeyHash, PubkeyBucket(Table, Peer->Handshake.RemoteStatic)); + MuReleasePushLockExclusive(&Table->Lock); +} + +_Use_decl_annotations_ +VOID +PubkeyHashtableRemove(PUBKEY_HASHTABLE *Table, WG_PEER *Peer) +{ + MuAcquirePushLockExclusive(&Table->Lock); + HlistDelInitRcu(&Peer->PubkeyHash); + MuReleasePushLockExclusive(&Table->Lock); +} + +_Use_decl_annotations_ +WG_PEER * +PubkeyHashtableLookup(PUBKEY_HASHTABLE *Table, CONST UINT8 Pubkey[NOISE_PUBLIC_KEY_LEN]) +{ + WG_PEER *IterPeer, *Peer = NULL; + KIRQL Irql; + + Irql = RcuReadLock(); + HLIST_FOR_EACH_ENTRY_RCU (IterPeer, PubkeyBucket(Table, Pubkey), WG_PEER, PubkeyHash) + { + if (RtlEqualMemory(Pubkey, IterPeer->Handshake.RemoteStatic, NOISE_PUBLIC_KEY_LEN)) + { + Peer = IterPeer; + break; + } + } + Peer = PeerGetMaybeZero(Peer); + RcuReadUnlock(Irql); + return Peer; +} + +_Post_notnull_ +static HLIST_HEAD * +IndexBucket(_In_ INDEX_HASHTABLE *Table, _In_ CONST UINT32_LE Index) +{ + /* Since the indices are random and thus all bits are uniformly + * distributed, we can find its bucket simply by masking. + */ + return &Table->Hashtable[(UINT32)Index & (HASH_SIZE(Table->Hashtable) - 1)]; +} + +_Use_decl_annotations_ +INDEX_HASHTABLE *IndexHashtableAlloc(VOID) +{ + INDEX_HASHTABLE *Table = MemAllocate(sizeof(*Table)); + if (!Table) + return NULL; + + HashInit(Table->Hashtable); + KeInitializeSpinLock(&Table->Lock); + return Table; +} + +/* At the moment, we limit ourselves to 2^20 total peers, which generally might + * amount to 2^20*3 items in this hashtable. The algorithm below works by + * picking a random number and testing it. We can see that these limits mean we + * usually succeed pretty quickly: + * + * >>> def calculation(tries, size): + * ... return (size / 2**32)**(tries - 1) * (1 - (size / 2**32)) + * ... + * >>> calculation(1, 2**20 * 3) + * 0.999267578125 + * >>> calculation(2, 2**20 * 3) + * 0.0007318854331970215 + * >>> calculation(3, 2**20 * 3) + * 5.360489012673497e-07 + * >>> calculation(4, 2**20 * 3) + * 3.9261394135792216e-10 + * + * At the moment, we don't do any masking, so this algorithm isn't exactly + * constant time in either the random guessing or in the hash list lookup. We + * could require a minimum of 3 tries, which would successfully mask the + * guessing. this would not, however, help with the growing hash lengths, which + * is another thing to consider moving forward. + */ + +_Use_decl_annotations_ +UINT32_LE +IndexHashtableInsert(INDEX_HASHTABLE *Table, INDEX_HASHTABLE_ENTRY *Entry) +{ + INDEX_HASHTABLE_ENTRY *ExistingEntry; + KIRQL Irql; + + KeAcquireSpinLock(&Table->Lock, &Irql); + HlistDelInitRcu(&Entry->IndexHash); + KeReleaseSpinLock(&Table->Lock, Irql); + + Irql = RcuReadLock(); + +searchUnusedSlot: + /* First we try to find an unused slot, randomly, while unlocked. */ + CryptoRandom(&Entry->Index, sizeof(Entry->Index)); + HLIST_FOR_EACH_ENTRY_RCU (ExistingEntry, IndexBucket(Table, Entry->Index), INDEX_HASHTABLE_ENTRY, IndexHash) + { + if (ExistingEntry->Index == Entry->Index) + /* If it's already in use, we continue searching. */ + goto searchUnusedSlot; + } + + /* Once we've found an unused slot, we lock it, and then double-check + * that nobody else stole it from us. + */ + KeAcquireSpinLockAtDpcLevel(&Table->Lock); + HLIST_FOR_EACH_ENTRY_RCU (ExistingEntry, IndexBucket(Table, Entry->Index), INDEX_HASHTABLE_ENTRY, IndexHash) + { + if (ExistingEntry->Index == Entry->Index) + { + KeReleaseSpinLockFromDpcLevel(&Table->Lock); + /* If it was stolen, we start over. */ + goto searchUnusedSlot; + } + } + /* Otherwise, we know we have it exclusively (since we're locked), + * so we insert. + */ + HlistAddHeadRcu(&Entry->IndexHash, IndexBucket(Table, Entry->Index)); + KeReleaseSpinLockFromDpcLevel(&Table->Lock); + + RcuReadUnlock(Irql); + + return Entry->Index; +} + +_Use_decl_annotations_ +BOOLEAN +IndexHashtableReplace(INDEX_HASHTABLE *Table, INDEX_HASHTABLE_ENTRY *Old, INDEX_HASHTABLE_ENTRY *New) +{ + BOOLEAN Ret; + KIRQL Irql; + + KeAcquireSpinLock(&Table->Lock, &Irql); + Ret = !HlistUnhashed(&Old->IndexHash); + if (!Ret) + goto out; + + New->Index = Old->Index; + HlistReplaceRcu(&Old->IndexHash, &New->IndexHash); + + /* Calling init here NULLs out IndexHash, and in fact after this + * function returns, it's theoretically possible for this to get + * reinserted elsewhere. That means the RCU lookup below might either + * terminate early or jump between buckets, in which case the packet + * simply gets dropped, which isn't terrible. + */ + HlistInit(&Old->IndexHash); +out: + KeReleaseSpinLock(&Table->Lock, Irql); + return Ret; +} + +_Use_decl_annotations_ +VOID +IndexHashtableRemove(INDEX_HASHTABLE *Table, INDEX_HASHTABLE_ENTRY *Entry) +{ + KIRQL Irql; + + KeAcquireSpinLock(&Table->Lock, &Irql); + HlistDelInitRcu(&Entry->IndexHash); + KeReleaseSpinLock(&Table->Lock, Irql); +} + +/* Returns a strong reference to a entry->peer */ +_Use_decl_annotations_ +INDEX_HASHTABLE_ENTRY * +IndexHashtableLookup(INDEX_HASHTABLE *Table, CONST INDEX_HASHTABLE_TYPE TypeMask, CONST UINT32_LE Index, WG_PEER **Peer) +{ + INDEX_HASHTABLE_ENTRY *IterEntry, *Entry = NULL; + KIRQL Irql; + + Irql = RcuReadLock(); + HLIST_FOR_EACH_ENTRY_RCU (IterEntry, IndexBucket(Table, Index), INDEX_HASHTABLE_ENTRY, IndexHash) + { + if (IterEntry->Index == Index) + { + if (IterEntry->Type & TypeMask) + Entry = IterEntry; + break; + } + } + if (Entry) + { + Entry->Peer = PeerGetMaybeZero(Entry->Peer); + if (Entry->Peer) + *Peer = Entry->Peer; + else + Entry = NULL; + } + RcuReadUnlock(Irql); + return Entry; +} diff --git a/driver/peerlookup.h b/driver/peerlookup.h new file mode 100644 index 0000000..fcc1a17 --- /dev/null +++ b/driver/peerlookup.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +#include "containers.h" +#include "messages.h" +#include "crypto.h" + +typedef struct _WG_PEER WG_PEER; + +typedef struct _PUBKEY_HASHTABLE +{ + DECLARE_HASHTABLE(Hashtable, 11); + SIPHASH_KEY Key; + EX_PUSH_LOCK Lock; +} PUBKEY_HASHTABLE; + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Must_inspect_result_ +_Post_maybenull_ +_Return_type_success_(return != NULL) +__drv_allocatesMem(Mem) +PUBKEY_HASHTABLE *PubkeyHashtableAlloc(VOID); + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_not_held_(Table->Lock) +VOID +PubkeyHashtableAdd(_Inout_ PUBKEY_HASHTABLE *Table, _Inout_ WG_PEER *Peer); + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_not_held_(Table->Lock) +VOID +PubkeyHashtableRemove(_Inout_ PUBKEY_HASHTABLE *Table, _Inout_ WG_PEER *Peer); + +/* Returns a strong reference to a peer */ +_IRQL_requires_max_(DISPATCH_LEVEL) +_Must_inspect_result_ +_Post_maybenull_ +WG_PEER * +PubkeyHashtableLookup(_In_ PUBKEY_HASHTABLE *Table, _In_ CONST UINT8 Pubkey[NOISE_PUBLIC_KEY_LEN]); + +typedef struct _INDEX_HASHTABLE +{ + DECLARE_HASHTABLE(Hashtable, 13); + KSPIN_LOCK Lock; +} INDEX_HASHTABLE; + +typedef enum _INDEX_HASHTABLE_TYPE +{ + INDEX_HASHTABLE_HANDSHAKE = 1U << 0, + INDEX_HASHTABLE_KEYPAIR = 1U << 1 +} INDEX_HASHTABLE_TYPE; + +typedef struct _INDEX_HASHTABLE_ENTRY +{ + WG_PEER *Peer; + HLIST_NODE IndexHash; + INDEX_HASHTABLE_TYPE Type; + UINT32_LE Index; +} INDEX_HASHTABLE_ENTRY; + +_Must_inspect_result_ +_Post_maybenull_ +_Return_type_success_(return != NULL) +__drv_allocatesMem(Mem) +INDEX_HASHTABLE *IndexHashtableAlloc(VOID); + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Requires_lock_not_held_(Table->Lock) +UINT32_LE +IndexHashtableInsert(_Inout_ INDEX_HASHTABLE *Table, _Inout_ INDEX_HASHTABLE_ENTRY *Entry); + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Requires_lock_not_held_(Table->Lock) +BOOLEAN +IndexHashtableReplace( + _Inout_ INDEX_HASHTABLE *Table, + _Inout_ INDEX_HASHTABLE_ENTRY *Old, + _Inout_ INDEX_HASHTABLE_ENTRY *New); + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Requires_lock_not_held_(Table->Lock) +VOID +IndexHashtableRemove(_Inout_ INDEX_HASHTABLE *Table, _In_ INDEX_HASHTABLE_ENTRY *Entry); + +_Must_inspect_result_ +_Return_type_success_(return != NULL) +INDEX_HASHTABLE_ENTRY * +IndexHashtableLookup( + _In_ INDEX_HASHTABLE *Table, + _In_ CONST INDEX_HASHTABLE_TYPE TypeMask, + _In_ CONST UINT32_LE Index, + _Out_ WG_PEER **Peer); diff --git a/driver/queueing.c b/driver/queueing.c new file mode 100644 index 0000000..029f253 --- /dev/null +++ b/driver/queueing.c @@ -0,0 +1,225 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include "interlocked.h" +#include "queueing.h" + +static KSTART_ROUTINE WorkerThread; +_Use_decl_annotations_ +static VOID +WorkerThread(PVOID StartContext) +{ + MULTICORE_WORKTHREAD *WorkThread = StartContext; + MULTICORE_WORKQUEUE *WorkQueue = WorkThread->WorkQueue; + PMULTICORE_WORKQUEUE_ROUTINE Func = WorkQueue->Func; + GROUP_AFFINITY Affinity = { .Mask = (KAFFINITY)1 << WorkThread->Processor.Number, + .Group = WorkThread->Processor.Group }; + KeSetSystemGroupAffinityThread(&Affinity, NULL); + PVOID Handles[] = { &WorkQueue->NewWork, &WorkQueue->Dead }; + for (;;) + { + if (KeWaitForMultipleObjects(ARRAYSIZE(Handles), Handles, WaitAny, Executive, KernelMode, FALSE, NULL, NULL) != + STATUS_WAIT_0) + break; + Func(WorkQueue); + } +} + +static KSTART_ROUTINE NewThreadSpawner; +_Use_decl_annotations_ +static VOID +NewThreadSpawner(PVOID StartContext) +{ + MULTICORE_WORKQUEUE *WorkQueue = StartContext; + OBJECT_ATTRIBUTES ObjectAttributes; + InitializeObjectAttributes(&ObjectAttributes, NULL, OBJ_KERNEL_HANDLE, NULL, NULL); + PVOID Handles[] = { &WorkQueue->NewCpus, &WorkQueue->Dead }; + for (;;) + { + if (KeWaitForMultipleObjects(ARRAYSIZE(Handles), Handles, WaitAny, Executive, KernelMode, FALSE, NULL, NULL) != + STATUS_WAIT_0) + break; + for (MULTICORE_WORKTHREAD *Thread = ReadPointerAcquire(&WorkQueue->FirstThread); Thread && !Thread->Thread; + Thread = Thread->NextThread) + { + HANDLE Handle; + if (!NT_SUCCESS(PsCreateSystemThread( + &Handle, THREAD_ALL_ACCESS, &ObjectAttributes, NULL, NULL, WorkerThread, Thread))) + break; + ObReferenceObjectByHandle(Handle, SYNCHRONIZE, NULL, KernelMode, &Thread->Thread, NULL); + ZwClose(Handle); + } + } +} + +static PROCESSOR_CALLBACK_FUNCTION NewCpuArrival; +_Use_decl_annotations_ +static VOID +NewCpuArrival(PVOID CallbackContext, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT ChangeContext, PNTSTATUS OperationStatus) +{ + if (ChangeContext->State != KeProcessorAddCompleteNotify) + return; + MULTICORE_WORKQUEUE *WorkQueue = CallbackContext; + MULTICORE_WORKTHREAD *WorkThread = MemAllocateAndZero(sizeof(*WorkThread)); + if (!WorkThread) + return; + WorkThread->Processor = ChangeContext->ProcNumber; + WorkThread->NextThread = WorkQueue->FirstThread; + WorkThread->WorkQueue = WorkQueue; + WritePointerRelease(&WorkQueue->FirstThread, WorkThread); + KeSetEvent(&WorkQueue->NewCpus, IO_NETWORK_INCREMENT, FALSE); +} + +_Use_decl_annotations_ +NTSTATUS +MulticoreWorkQueueInit(MULTICORE_WORKQUEUE *WorkQueue, PMULTICORE_WORKQUEUE_ROUTINE Func) +{ + KeInitializeEvent(&WorkQueue->NewWork, SynchronizationEvent, FALSE); + KeInitializeEvent(&WorkQueue->NewCpus, SynchronizationEvent, FALSE); + KeInitializeEvent(&WorkQueue->Dead, NotificationEvent, FALSE); + WorkQueue->FirstThread = NULL; + WorkQueue->Func = Func; + OBJECT_ATTRIBUTES ObjectAttributes; + InitializeObjectAttributes(&ObjectAttributes, NULL, OBJ_KERNEL_HANDLE, NULL, NULL); + HANDLE Handle; + NTSTATUS Status = + PsCreateSystemThread(&Handle, THREAD_ALL_ACCESS, &ObjectAttributes, NULL, NULL, NewThreadSpawner, WorkQueue); + if (!NT_SUCCESS(Status)) + return Status; + ObReferenceObjectByHandle(Handle, SYNCHRONIZE, NULL, KernelMode, &WorkQueue->WorkerSpawnerThread, NULL); + ZwClose(Handle); + WorkQueue->NewCpuNotifier = + KeRegisterProcessorChangeCallback(NewCpuArrival, WorkQueue, KE_PROCESSOR_CHANGE_ADD_EXISTING); + Status = WorkQueue->NewCpuNotifier ? STATUS_SUCCESS : STATUS_INSUFFICIENT_RESOURCES; + if (!NT_SUCCESS(Status)) + MulticoreWorkQueueDestroy(WorkQueue); + return Status; +} + +_Use_decl_annotations_ +BOOLEAN +MulticoreWorkQueueBump(MULTICORE_WORKQUEUE *WorkQueue) +{ + return KeSetEvent(&WorkQueue->NewWork, IO_NETWORK_INCREMENT, FALSE) == 0; +} + +_Use_decl_annotations_ +VOID +MulticoreWorkQueueDestroy(MULTICORE_WORKQUEUE *WorkQueue) +{ + if (WorkQueue->NewCpuNotifier) + KeDeregisterProcessorChangeCallback(WorkQueue->NewCpuNotifier); + KeSetEvent(&WorkQueue->Dead, IO_NETWORK_INCREMENT, FALSE); + KeWaitForSingleObject(WorkQueue->WorkerSpawnerThread, Executive, KernelMode, FALSE, NULL); + ObDereferenceObject(WorkQueue->WorkerSpawnerThread); + + ULONG ThreadCount = 0; + for (MULTICORE_WORKTHREAD *Thread = WorkQueue->FirstThread; Thread; Thread = Thread->NextThread) + { + if (Thread->Thread) + ++ThreadCount; + } + if (!ThreadCount) + return; + PKTHREAD *Threads = MemAllocateArray(ThreadCount, sizeof(*Threads) + sizeof(KWAIT_BLOCK)); + if (Threads) + { + PKWAIT_BLOCK WaitBlock = (PKWAIT_BLOCK)((ULONG_PTR)Threads + (ThreadCount * sizeof(*Threads))); + ThreadCount = 0; + for (MULTICORE_WORKTHREAD *Thread = WorkQueue->FirstThread; Thread; Thread = Thread->NextThread) + { + if (Thread->Thread) + Threads[ThreadCount++] = Thread->Thread; + } + KeWaitForMultipleObjects(ThreadCount, Threads, WaitAll, Executive, KernelMode, FALSE, NULL, WaitBlock); + for (MULTICORE_WORKTHREAD *Thread = WorkQueue->FirstThread, *Next; Thread; Thread = Next) + { + Next = Thread->NextThread; + if (Thread->Thread) + ObDereferenceObject(Thread->Thread); + MemFree(Thread); + } + MemFree(Threads); + } + else + { + for (MULTICORE_WORKTHREAD *Thread = WorkQueue->FirstThread, *Next; Thread; Thread = Next) + { + Next = Thread->NextThread; + if (Thread->Thread) + { + KeWaitForSingleObject(Thread->Thread, Executive, KernelMode, FALSE, NULL); + ObDereferenceObject(Thread->Thread); + } + MemFree(Thread); + } + } +} + +#define NEXT(Nbl) NET_BUFFER_LIST_PER_PEER_LIST_LINK(Nbl) +#define STUB(Queue) (&(Queue)->Empty) + +_Use_decl_annotations_ +VOID +PrevQueueInit(PREV_QUEUE *Queue) +{ + NEXT(STUB(Queue)) = NULL; + Queue->Head = Queue->Tail = STUB(Queue); + Queue->Peeked = NULL; + WriteRaw(&Queue->Count, 0); +} + +static VOID +__PrevQueueEnqueue(_Inout_ PREV_QUEUE *Queue, _In_ __drv_aliasesMem NET_BUFFER_LIST *Nbl) +{ + WritePointerNoFence(&NEXT(Nbl), NULL); + WritePointerNoFence(&NEXT((NET_BUFFER_LIST *)InterlockedExchangePointerRelease(&Queue->Head, Nbl)), Nbl); +} + +_Use_decl_annotations_ +BOOLEAN +PrevQueueEnqueue(PREV_QUEUE *Queue, NET_BUFFER_LIST *Nbl) +{ + if (!InterlockedIncrementUnless(&Queue->Count, MAX_QUEUED_PACKETS)) + return FALSE; + __PrevQueueEnqueue(Queue, Nbl); + return TRUE; +} + +_Use_decl_annotations_ +NET_BUFFER_LIST * +PrevQueueDequeue(PREV_QUEUE *Queue) +{ + NET_BUFFER_LIST *Tail = Queue->Tail, *Next = ReadPointerAcquire(&NEXT(Tail)); + + if (Tail == STUB(Queue)) + { + if (!Next) + return NULL; + Queue->Tail = Next; + Tail = Next; + Next = ReadPointerAcquire(&NEXT(Next)); + } + if (Next) + { + Queue->Tail = Next; + InterlockedDecrement(&Queue->Count); + return Tail; + } + if (Tail != ReadPointerNoFence(&Queue->Head)) + return NULL; + __PrevQueueEnqueue(Queue, STUB(Queue)); + Next = ReadPointerAcquire(&NEXT(Tail)); + if (Next) + { + Queue->Tail = Next; + InterlockedDecrement(&Queue->Count); + return Tail; + } + return NULL; +} + +#undef NEXT +#undef STUB diff --git a/driver/queueing.h b/driver/queueing.h new file mode 100644 index 0000000..a0d856e --- /dev/null +++ b/driver/queueing.h @@ -0,0 +1,305 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +#include "peer.h" + +#define MAX_QUEUED_INCOMING_HANDSHAKES 4096 +#define MAX_STAGED_PACKETS 128 +#define MAX_QUEUED_PACKETS 1024 +#define PEER_XMIT_PACKETS_PER_ROUND 256 + +typedef struct _WG_DEVICE WG_DEVICE; +typedef struct _WG_PEER WG_PEER; +typedef struct _PREV_QUEUE PREV_QUEUE; + +/* queueing.c APIs: */ + +_IRQL_requires_max_(PASSIVE_LEVEL) +NTSTATUS +MulticoreWorkQueueInit(_Out_ MULTICORE_WORKQUEUE *WorkQueue, _In_ PMULTICORE_WORKQUEUE_ROUTINE Func); + +_IRQL_requires_max_(DISPATCH_LEVEL) +BOOLEAN +MulticoreWorkQueueBump(_Inout_ MULTICORE_WORKQUEUE *WorkQueue); + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +MulticoreWorkQueueDestroy(_Inout_ MULTICORE_WORKQUEUE *WorkQueue); + +#define BUSY_LINK ((PEER_SERIAL_ENTRY *)~(ULONG_PTR)0) + +static inline VOID +PeerSerialInit(_Out_ PEER_SERIAL *Serial) +{ + Serial->Last = &Serial->First; + Serial->First = BUSY_LINK; + KeInitializeSpinLock(&Serial->Lock); +} + +_Requires_lock_not_held_(Serial->Lock) +_IRQL_requires_max_(DISPATCH_LEVEL) +static inline BOOLEAN +PeerSerialEnqueueIfNotBusy(_Inout_ PEER_SERIAL *Serial, _Inout_ PEER_SERIAL_ENTRY *Item, _In_ BOOLEAN MaybeRequeue) +{ + BOOLEAN Added = FALSE; + KIRQL Irql; + KeAcquireSpinLock(&Serial->Lock, &Irql); + if (ReadPointerNoFence(&Item->Next)) + { + if (MaybeRequeue) + WriteNoFence16(&Item->Requeue, TRUE); + goto out; + } + WritePointerNoFence(&Item->Next, BUSY_LINK); + *Serial->Last = Item; + Serial->Last = &Item->Next; + Added = TRUE; +out: + KeReleaseSpinLock(&Serial->Lock, Irql); + return Added; +} + +_Requires_lock_not_held_(Serial->Lock) +_IRQL_requires_max_(DISPATCH_LEVEL) +static inline BOOLEAN +PeerSerialMaybeRetire(_Inout_ PEER_SERIAL *Serial, _Inout_ PEER_SERIAL_ENTRY *Item, _In_ BOOLEAN ForceMore) +{ + BOOLEAN Ret = FALSE; + KIRQL Irql; + KeAcquireSpinLock(&Serial->Lock, &Irql); + WritePointerNoFence(&Item->Next, NULL); + if (InterlockedCompareExchange16(&Item->Requeue, FALSE, TRUE) || ForceMore) + { + WritePointerNoFence(&Item->Next, BUSY_LINK); + *Serial->Last = Item; + Serial->Last = &Item->Next; + Ret = TRUE; + } + KeReleaseSpinLock(&Serial->Lock, Irql); + return Ret; +} + +_Requires_lock_not_held_(Serial->Lock) +_IRQL_requires_max_(DISPATCH_LEVEL) +_Must_inspect_result_ +_Post_maybenull_ +static inline PEER_SERIAL_ENTRY * +PeerSerialDequeue(_Inout_ PEER_SERIAL *Serial) +{ + if (ReadPointerNoFence(&Serial->First) == BUSY_LINK) + return NULL; + PEER_SERIAL_ENTRY *First = NULL; + KIRQL Irql; + KeAcquireSpinLock(&Serial->Lock, &Irql); + First = Serial->First; + if (First != BUSY_LINK && (Serial->First = First->Next) == BUSY_LINK) + Serial->Last = &Serial->First; + KeReleaseSpinLock(&Serial->Lock, Irql); + return First == BUSY_LINK ? NULL : First; +} + +#undef BUSY_LINK + +/* + * NBL[0] = crypt state + * NBL[1] = prev queue link + * NB[0-1] = nonce + * NB[2] = keypair + * NB[3] = <empty> + */ +#define NET_BUFFER_NONCE(Nb) (*(UINT64 *)&NET_BUFFER_MINIPORT_RESERVED(Nb)[0]) +#define NET_BUFFER_LIST_KEYPAIR(Nbl) \ + (*(NOISE_KEYPAIR **)&NET_BUFFER_MINIPORT_RESERVED(NET_BUFFER_LIST_FIRST_NB(Nbl))[2]) +#define NET_BUFFER_LIST_PEER(Nbl) (NET_BUFFER_LIST_KEYPAIR(Nbl)->Entry.Peer) +#define NET_BUFFER_LIST_CRYPT_STATE(Nbl) ((LONG *)&NET_BUFFER_LIST_MINIPORT_RESERVED(Nbl)[0]) +#define NET_BUFFER_LIST_PER_PEER_LIST_LINK(Nbl) (*(NET_BUFFER_LIST **)&NET_BUFFER_LIST_MINIPORT_RESERVED(Nbl)[1]) +#define NET_BUFFER_LIST_PROTOCOL(Nbl) ((UINT16_BE)(ULONG_PTR)NET_BUFFER_LIST_INFO(Nbl, NetBufferListProtocolId)) +#define NET_BUFFER_LIST_DATAGRAM_INDICATION(Nbl) (*(WSK_DATAGRAM_INDICATION **)&Nbl->ParentNetBufferList) + +/* receive.c APIs: */ +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +PacketReceive(_Inout_ WG_DEVICE *Wg, _In_ __drv_aliasesMem NET_BUFFER_LIST *First); + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +FreeReceiveNetBufferList(_Inout_ WG_DEVICE *Wg, __drv_freesMem(Mem) _In_ NET_BUFFER_LIST *First); + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +FreeIncomingHandshakes(_Inout_ WG_DEVICE *Wg); + +/* send.c APIs: */ +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +PacketSendQueuedHandshakeInitiation(_Inout_ WG_PEER *Peer, _In_ BOOLEAN IsRetry); + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_not_held_(Peer->Handshake.StaticIdentity->Lock) +_Requires_lock_not_held_(Peer->Handshake.Lock) +VOID +PacketSendHandshakeResponse(_Inout_ WG_PEER *Peer); + +_IRQL_requires_max_(APC_LEVEL) +VOID +PacketSendHandshakeCookie(_Inout_ WG_DEVICE *Wg, _In_ CONST NET_BUFFER_LIST *InitiatingNbl, _In_ UINT32_LE SenderIndex); + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +PacketSendKeepalive(_Inout_ WG_PEER *Peer); + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Requires_lock_not_held_(Peer->StagedPacketQueue.Lock) +VOID +PacketPurgeStagedPackets(_Inout_ WG_PEER *Peer); + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Requires_lock_not_held_(Peer->StagedPacketQueue.Lock) +VOID +PacketSendStagedPackets(_Inout_ WG_PEER *Peer); + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +FreeSendNetBufferList(_In_ WG_DEVICE *Wg, __drv_freesMem(Mem) _In_ NET_BUFFER_LIST *Nbl, _In_ ULONG SendCompleteFlags); + +MULTICORE_WORKQUEUE_ROUTINE PacketEncryptWorker, PacketDecryptWorker; +MULTICORE_WORKQUEUE_ROUTINE PacketTxWorker, PacketRxWorker; +MULTICORE_WORKQUEUE_ROUTINE PacketHandshakeTxWorker, PacketHandshakeRxWorker; + +typedef enum _PACKET_STATE +{ + PACKET_STATE_UNCRYPTED, + PACKET_STATE_CRYPTED, + PACKET_STATE_DEAD +} PACKET_STATE; + +_IRQL_requires_max_(DISPATCH_LEVEL) +static inline UINT16_BE +IpTunnelParseProtocol(_In_ CONST NET_BUFFER_LIST *Nbl) +{ + NET_BUFFER *Nb = NET_BUFFER_LIST_FIRST_NB(Nbl); + NT_ASSERT(Nb); + UCHAR *FirstByteOfHeader = NdisGetDataBuffer(Nb, sizeof(*FirstByteOfHeader), NULL, 1, 0); + if (!FirstByteOfHeader) + return 0; + if (NET_BUFFER_DATA_LENGTH(Nb) >= sizeof(IPV4HDR) && (*FirstByteOfHeader >> 4) == 4) + return Htons(NDIS_ETH_TYPE_IPV4); + if (NET_BUFFER_DATA_LENGTH(Nb) >= sizeof(IPV6HDR) && (*FirstByteOfHeader >> 4) == 6) + return Htons(NDIS_ETH_TYPE_IPV6); + return 0; +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +static inline BOOLEAN +CheckPacketProtocol(_In_ CONST NET_BUFFER_LIST *Nbl) +{ + UINT16_BE RealProtocol = IpTunnelParseProtocol(Nbl); + return RealProtocol && NET_BUFFER_LIST_PROTOCOL(Nbl) == RealProtocol; +} + +VOID +PrevQueueInit(_Out_ PREV_QUEUE *Queue); + +/* Multi producer */ +_Return_type_success_(return != FALSE) +BOOLEAN +PrevQueueEnqueue(_Inout_ PREV_QUEUE *Queue, _In_ __drv_aliasesMem NET_BUFFER_LIST *Nbl); + +/* Single consumer */ +_Must_inspect_result_ +_Post_maybenull_ +NET_BUFFER_LIST * +PrevQueueDequeue(_Inout_ PREV_QUEUE *Queue); + +/* Single consumer */ +_Must_inspect_result_ +_Post_maybenull_ +static inline NET_BUFFER_LIST * +PrevQueuePeek(_Inout_ PREV_QUEUE *Queue) +{ + if (Queue->Peeked) + return Queue->Peeked; + Queue->Peeked = PrevQueueDequeue(Queue); + return Queue->Peeked; +} + +/* Single consumer */ +static inline VOID +PrevQueueDropPeeked(_Out_ PREV_QUEUE *Queue) +{ + Queue->Peeked = NULL; +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +static inline BOOLEAN +QueueEnqueuePerDevice( + _Inout_ PTR_RING *DeviceQueue, + _Inout_ MULTICORE_WORKQUEUE *DeviceThreads, + _Inout_ NET_BUFFER_LIST *Nbl) +{ + if (!NT_SUCCESS(PtrRingProduce(DeviceQueue, Nbl))) + return FALSE; + MulticoreWorkQueueBump(DeviceThreads); + return TRUE; +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +static inline BOOLEAN +QueueInsertPerPeer(_Inout_ PREV_QUEUE *PeerQueue, _Inout_ NET_BUFFER_LIST *Nbl) +{ + WriteRelease(NET_BUFFER_LIST_CRYPT_STATE(Nbl), PACKET_STATE_UNCRYPTED); + /* We first queue this up for the peer ingestion, but the consumer + * will wait for the state to change to CRYPTED or DEAD before. + */ + return PrevQueueEnqueue(PeerQueue, Nbl); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +static inline NTSTATUS +QueueEnqueuePerDeviceAndPeer( + _Inout_ PTR_RING *DeviceQueue, + _Inout_ PREV_QUEUE *PeerQueue, + _Inout_ MULTICORE_WORKQUEUE *DeviceThreads, + _Inout_ NET_BUFFER_LIST *Nbl) +{ + if (!QueueInsertPerPeer(PeerQueue, Nbl)) + return STATUS_BUFFER_TOO_SMALL; + + /* Then we queue it up in the device queue, which consumes the + * packet as soon as it can. + */ + if (!QueueEnqueuePerDevice(DeviceQueue, DeviceThreads, Nbl)) + return STATUS_PIPE_BROKEN; + return STATUS_SUCCESS; +} + +_Requires_lock_not_held_(PeerQueue->Lock) +_IRQL_requires_max_(DISPATCH_LEVEL) +static inline VOID +QueueEnqueuePerPeer( + _Inout_ PEER_SERIAL *PeerQueue, + _Inout_ PEER_SERIAL_ENTRY *PeerSerialEntry, + _Inout_ MULTICORE_WORKQUEUE *PeerThreads, + _Inout_ __drv_aliasesMem NET_BUFFER_LIST *Nbl, + _In_ PACKET_STATE State) +{ + /* We take a reference, because as soon as we call WriteRelease, the + * peer can be freed from below us. + */ + WG_PEER *Peer = PeerGet(NET_BUFFER_LIST_PEER(Nbl)); + + WriteRelease(NET_BUFFER_LIST_CRYPT_STATE(Nbl), State); + if (PeerSerialEnqueueIfNotBusy(PeerQueue, PeerSerialEntry, TRUE)) + MulticoreWorkQueueBump(PeerThreads); + PeerPut(Peer); +} + +#ifdef DBG +_IRQL_requires_max_(PASSIVE_LEVEL) +BOOLEAN +PacketCounterSelftest(VOID); +#endif diff --git a/driver/ratelimiter.c b/driver/ratelimiter.c new file mode 100644 index 0000000..8d72afe --- /dev/null +++ b/driver/ratelimiter.c @@ -0,0 +1,211 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include "containers.h" +#include "ratelimiter.h" +#include "crypto.h" +#include "logging.h" +#include "timers.h" + +#define TABLE_SIZE 8192 +#define MAX_ENTRIES (TABLE_SIZE * 8) + +static LOOKASIDE_ALIGN LOOKASIDE_LIST_EX EntryCache; +static HSIPHASH_KEY Key; +static KSPIN_LOCK TableLock; +static LONG TotalEntries = 0; +static struct +{ + KEVENT Terminate; + PKTHREAD Thread; +} RatelimiterGcEntriesThread; +static HLIST_HEAD TableV4[TABLE_SIZE] = { 0 }, TableV6[TABLE_SIZE] = { 0 }; + +typedef struct _RATELIMITER_ENTRY +{ + UINT64 LastTime, Tokens, Ip; + KSPIN_LOCK Lock; + HLIST_NODE Hash; + RCU_CALLBACK Rcu; +} RATELIMITER_ENTRY; + +enum +{ + PACKETS_PER_SECOND = 20, + PACKETS_BURSTABLE = 5, + PACKET_COST = SYS_TIME_UNITS_PER_SEC / PACKETS_PER_SECOND, + TOKEN_MAX = PACKET_COST * PACKETS_BURSTABLE +}; + +static RCU_CALLBACK_FN EntryFree; +_Use_decl_annotations_ +static VOID +EntryFree(RCU_CALLBACK *Rcu) +{ + ExFreeToLookasideListEx(&EntryCache, CONTAINING_RECORD(Rcu, RATELIMITER_ENTRY, Rcu)); + InterlockedDecrement(&TotalEntries); +} + +static VOID +EntryUninit(_Inout_ RATELIMITER_ENTRY *Entry) +{ + HlistDelRcu(&Entry->Hash); + RcuCall(&Entry->Rcu, EntryFree); +} + +/* Calling this function with a NULL work uninits all entries. */ +_IRQL_requires_max_(PASSIVE_LEVEL) +_Function_class_(KSTART_ROUTINE) +static VOID +RatelimiterGcEntries(_In_opt_ PVOID StartContext) +{ + for (;;) + { + CONST UINT64 Now = KeQueryInterruptTime(); + RATELIMITER_ENTRY *Entry; + HLIST_NODE *Temp; + ULONG i; + KIRQL Irql; + + for (i = 0; i < TABLE_SIZE; ++i) + { + KeAcquireSpinLock(&TableLock, &Irql); + HLIST_FOR_EACH_ENTRY_SAFE (Entry, Temp, &TableV4[i], RATELIMITER_ENTRY, Hash) + { + if (!StartContext || Now - Entry->LastTime > SYS_TIME_UNITS_PER_SEC) + EntryUninit(Entry); + } + HLIST_FOR_EACH_ENTRY_SAFE (Entry, Temp, &TableV6[i], RATELIMITER_ENTRY, Hash) + { + if (!StartContext || Now - Entry->LastTime > SYS_TIME_UNITS_PER_SEC) + EntryUninit(Entry); + } + KeReleaseSpinLock(&TableLock, Irql); + } + if (!StartContext) + break; + if (KeWaitForSingleObject( + &RatelimiterGcEntriesThread.Terminate, + Executive, + KernelMode, + FALSE, + &(LARGE_INTEGER){ .QuadPart = -SYS_TIME_UNITS_PER_SEC }) == STATUS_SUCCESS) + break; + } +} + +_Use_decl_annotations_ +BOOLEAN +RatelimiterAllow(CONST SOCKADDR *Src) +{ + RATELIMITER_ENTRY *Entry; + HLIST_HEAD *Bucket; + UINT64 Ip; + KIRQL Irql; + + if (Src->sa_family == AF_INET) + { + Ip = (UINT64)((SOCKADDR_IN *)Src)->sin_addr.s_addr; + Bucket = &TableV4[Hsiphash1u32((UINT32)Ip, &Key) & (TABLE_SIZE - 1)]; + } + else if (Src->sa_family == AF_INET6) + { + /* Only use 64 bits, so as to ratelimit the whole /64. */ + RtlCopyMemory(&Ip, &((SOCKADDR_IN6 *)Src)->sin6_addr, sizeof(Ip)); + Bucket = &TableV6[Hsiphash2u32((UINT32)(Ip >> 32), (UINT32)Ip, &Key) & (TABLE_SIZE - 1)]; + } + else + return FALSE; + Irql = RcuReadLock(); + HLIST_FOR_EACH_ENTRY_RCU (Entry, Bucket, RATELIMITER_ENTRY, Hash) + { + if (Entry->Ip == Ip) + { + UINT64 Now, Tokens; + BOOLEAN Ret; + /* Quasi-inspired by nft_limit.c, but this is actually a + * slightly different algorithm. Namely, we incorporate + * the burst as part of the maximum tokens, rather than + * as part of the rate. + */ + KeAcquireSpinLockAtDpcLevel(&Entry->Lock); + Now = KeQueryInterruptTime(); + Tokens = min(TOKEN_MAX, Entry->Tokens + Now - Entry->LastTime); + Entry->LastTime = Now; + Ret = Tokens >= PACKET_COST; + Entry->Tokens = Ret ? Tokens - PACKET_COST : Tokens; + KeReleaseSpinLockFromDpcLevel(&Entry->Lock); + RcuReadUnlock(Irql); + return Ret; + } + } + RcuReadUnlock(Irql); + + if ((ULONG)InterlockedIncrement(&TotalEntries) > MAX_ENTRIES) + goto cleanupOom; + + Entry = ExAllocateFromLookasideListEx(&EntryCache); + if (!Entry) + goto cleanupOom; + + Entry->Ip = Ip; + HlistInit(&Entry->Hash); + KeInitializeSpinLock(&Entry->Lock); + Entry->LastTime = KeQueryInterruptTime(); + Entry->Tokens = TOKEN_MAX - PACKET_COST; + KeAcquireSpinLock(&TableLock, &Irql); + HlistAddHeadRcu(&Entry->Hash, Bucket); + KeReleaseSpinLock(&TableLock, Irql); + return TRUE; + +cleanupOom: + InterlockedDecrement(&TotalEntries); + return FALSE; +} + +#ifdef ALLOC_PRAGMA +# pragma alloc_text(INIT, RatelimiterDriverEntry) +#endif +_Use_decl_annotations_ +NTSTATUS +RatelimiterDriverEntry(VOID) +{ + NTSTATUS Status = + ExInitializeLookasideListEx(&EntryCache, NULL, NULL, NonPagedPool, 0, sizeof(RATELIMITER_ENTRY), MEMORY_TAG, 0); + if (!NT_SUCCESS(Status)) + return Status; + KeInitializeSpinLock(&TableLock); + KeInitializeEvent(&RatelimiterGcEntriesThread.Terminate, NotificationEvent, FALSE); + OBJECT_ATTRIBUTES ObjectAttributes; + InitializeObjectAttributes(&ObjectAttributes, NULL, OBJ_KERNEL_HANDLE, NULL, NULL); + HANDLE Handle; + Status = PsCreateSystemThread( + &Handle, THREAD_ALL_ACCESS, &ObjectAttributes, NULL, NULL, RatelimiterGcEntries, (PVOID)TRUE); + if (!NT_SUCCESS(Status)) + goto cleanupEntryCache; + ObReferenceObjectByHandle(Handle, SYNCHRONIZE, NULL, KernelMode, &RatelimiterGcEntriesThread.Thread, NULL); + ZwClose(Handle); + CryptoRandom(&Key, sizeof(Key)); + return STATUS_SUCCESS; +cleanupEntryCache: + ExDeleteLookasideListEx(&EntryCache); + return Status; +} + +_Use_decl_annotations_ +VOID RatelimiterUnload(VOID) +{ +#pragma warning(suppress : 28160) /* Acknowledge caution about Wait parameter. */ + KeSetEvent(&RatelimiterGcEntriesThread.Terminate, IO_NO_INCREMENT, TRUE); + KeWaitForSingleObject(RatelimiterGcEntriesThread.Thread, Executive, KernelMode, FALSE, NULL); + ObDereferenceObject(RatelimiterGcEntriesThread.Thread); + RatelimiterGcEntries(NULL); + RcuBarrier(); + ExDeleteLookasideListEx(&EntryCache); +} + +#ifdef DBG +# include "selftest/ratelimiter.c" +#endif diff --git a/driver/ratelimiter.h b/driver/ratelimiter.h new file mode 100644 index 0000000..01523a5 --- /dev/null +++ b/driver/ratelimiter.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +_IRQL_requires_max_(DISPATCH_LEVEL) +BOOLEAN +RatelimiterAllow(_In_ CONST SOCKADDR *Src); + +_IRQL_requires_max_(PASSIVE_LEVEL) +NTSTATUS +RatelimiterDriverEntry(VOID); + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID RatelimiterUnload(VOID); + +#ifdef DBG +_IRQL_requires_max_(PASSIVE_LEVEL) +BOOLEAN +RatelimiterSelftest(VOID); +#endif diff --git a/driver/rcu.c b/driver/rcu.c new file mode 100644 index 0000000..e0cd1e0 --- /dev/null +++ b/driver/rcu.c @@ -0,0 +1,176 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include "memory.h" +#include "rcu.h" + +static struct +{ + KDPC *PerCpuDpcs; + EX_PUSH_LOCK Lock; +} SyncState; + +static KDEFERRED_ROUTINE ProcessorTick; +_Use_decl_annotations_ +static VOID +ProcessorTick(KDPC *Dpc, PVOID DeferredContext, PVOID SystemArgument1, PVOID SystemArgument2) +{ + _Analysis_assume_(SystemArgument1 != NULL); + _Analysis_assume_(SystemArgument2 != NULL); + KEVENT *Event = SystemArgument1; + LONG *Refs = SystemArgument2; + if (!InterlockedDecrement(Refs)) + KeSetEvent(Event, IO_NO_INCREMENT, FALSE); +} + +/* `Refs`, although on stack, is shared between processors, and does require interlocked access. */ +#pragma warning(push) +#pragma warning(disable : 28112) +#pragma warning(disable : 28113) + +_Use_decl_annotations_ +VOID RcuSynchronize(VOID) +{ + MuAcquirePushLockExclusive(&SyncState.Lock); + KEVENT Event; + KeInitializeEvent(&Event, NotificationEvent, FALSE); + LONG Refs = 1; + ULONG NumProcessors = KeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS); + for (ULONG Processor = 0; Processor < NumProcessors; ++Processor) + { + PROCESSOR_NUMBER ProcessorNumber; + if (!NT_SUCCESS(KeGetProcessorNumberFromIndex(Processor, &ProcessorNumber))) + continue; + if (!NT_SUCCESS(KeSetTargetProcessorDpcEx(&SyncState.PerCpuDpcs[Processor], &ProcessorNumber))) + continue; + InterlockedIncrement(&Refs); + if (!KeInsertQueueDpc(&SyncState.PerCpuDpcs[Processor], &Event, &Refs)) + InterlockedDecrement(&Refs); + } + if (!InterlockedDecrement(&Refs)) + goto out; + KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL); +out: + MuReleasePushLockExclusive(&SyncState.Lock); +} + +#pragma warning(pop) + +static struct +{ + PKTHREAD Thread; + KEVENT WorkPending; + BOOLEAN Terminate; + RCU_CALLBACK *Head, *Tail; + KSPIN_LOCK Lock; +} Cleanup; + +static KSTART_ROUTINE CallerThread; +_Use_decl_annotations_ +static VOID +CallerThread(PVOID StartContext) +{ + for (;;) + { + KeWaitForSingleObject(&Cleanup.WorkPending, Executive, KernelMode, FALSE, NULL); + KLOCK_QUEUE_HANDLE LockHandle; + KeAcquireInStackQueuedSpinLock(&Cleanup.Lock, &LockHandle); + RCU_CALLBACK *Head = Cleanup.Head; + Cleanup.Head = NULL; + Cleanup.Tail = NULL; + KeClearEvent(&Cleanup.WorkPending); + KeReleaseInStackQueuedSpinLock(&LockHandle); + RcuSynchronize(); + while (Head) + { + RCU_CALLBACK *Next = Head->Next; + switch (Head->Type) + { + case RCU_CALLBACK_CALL: + Head->Func(Head); + break; + case RCU_CALLBACK_FREE: + MemFree((UCHAR *)Head - Head->Offset); + break; + case RCU_CALLBACK_SYNC: + KeSetEvent(&Head->Done, IO_NETWORK_INCREMENT, FALSE); + break; + } + Head = Next; + } + if (ReadBooleanNoFence(&Cleanup.Terminate)) + break; + } +} + +_Use_decl_annotations_ +VOID +__RcuCall(_Inout_ RCU_CALLBACK *Head) +{ + Head->Next = NULL; + KLOCK_QUEUE_HANDLE LockHandle; + KeAcquireInStackQueuedSpinLock(&Cleanup.Lock, &LockHandle); + *(Cleanup.Tail ? &Cleanup.Tail->Next : &Cleanup.Head) = Head; + Cleanup.Tail = Head; + KeSetEvent(&Cleanup.WorkPending, IO_NO_INCREMENT, FALSE); + KeReleaseInStackQueuedSpinLock(&LockHandle); +} + +_Use_decl_annotations_ +VOID RcuBarrier(VOID) +{ + RCU_CALLBACK Work; + Work.Type = RCU_CALLBACK_SYNC; + KeInitializeEvent(&Work.Done, SynchronizationEvent, FALSE); + __RcuCall(&Work); + KeWaitForSingleObject(&Work.Done, Executive, KernelMode, FALSE, NULL); +} + +#ifdef ALLOC_PRAGMA +# pragma alloc_text(INIT, RcuDriverEntry) +#endif +_Use_decl_annotations_ +NTSTATUS +RcuDriverEntry(VOID) +{ + ULONG NumProcessors = KeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS); + SyncState.PerCpuDpcs = MemAllocateArray(NumProcessors, sizeof(*SyncState.PerCpuDpcs)); + if (!SyncState.PerCpuDpcs) + return STATUS_INSUFFICIENT_RESOURCES; + MuInitializePushLock(&SyncState.Lock); + for (ULONG Processor = 0; Processor < NumProcessors; ++Processor) + { + KeInitializeDpc(&SyncState.PerCpuDpcs[Processor], ProcessorTick, NULL); + KeSetImportanceDpc(&SyncState.PerCpuDpcs[Processor], LowImportance); + } + + KeInitializeEvent(&Cleanup.WorkPending, NotificationEvent, FALSE); + KeInitializeSpinLock(&Cleanup.Lock); + OBJECT_ATTRIBUTES ObjectAttributes; + InitializeObjectAttributes(&ObjectAttributes, NULL, OBJ_KERNEL_HANDLE, NULL, NULL); + HANDLE Handle; + NTSTATUS Status = + PsCreateSystemThread(&Handle, THREAD_ALL_ACCESS, &ObjectAttributes, NULL, NULL, CallerThread, (PVOID)TRUE); + if (!NT_SUCCESS(Status)) + goto cleanupDpcs; + ObReferenceObjectByHandle(Handle, SYNCHRONIZE, NULL, KernelMode, &Cleanup.Thread, NULL); + ZwClose(Handle); + return STATUS_SUCCESS; + +cleanupDpcs: + MemFree(SyncState.PerCpuDpcs); + return Status; +} + +_Use_decl_annotations_ +VOID RcuUnload(VOID) +{ + RcuBarrier(); + WriteBooleanNoFence(&Cleanup.Terminate, TRUE); + KeSetEvent(&Cleanup.WorkPending, IO_NO_INCREMENT, FALSE); + KeWaitForSingleObject(Cleanup.Thread, Executive, KernelMode, FALSE, NULL); + ObDereferenceObject(Cleanup.Thread); + MemFree(SyncState.PerCpuDpcs); +} diff --git a/driver/rcu.h b/driver/rcu.h new file mode 100644 index 0000000..1c9e153 --- /dev/null +++ b/driver/rcu.h @@ -0,0 +1,125 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +#include <ntifs.h> /* Must be included before <wdm.h> */ +#include <wdm.h> +#include "interlocked.h" + +extern int _Global_rcu_read_lock_; +#define _Requires_rcu_held_ _Requires_lock_held_(_Global_rcu_read_lock_) +#define _Acquires_rcu_ _Acquires_lock_(_Global_rcu_read_lock_) +#define _Releases_rcu_ _Releases_lock_(_Global_rcu_read_lock_) +#define _Analysis_assume_rcu_held_ _Analysis_assume_lock_held_(_Global_rcu_read_lock_); +#define _Analysis_assume_rcu_not_held_ _Analysis_assume_lock_not_held_(_Global_rcu_read_lock_); +#define _Analysis_assume_rcu_acquired_ _Analysis_assume_lock_acquired_(_Global_rcu_read_lock_); +#define _Analysis_assume_rcu_released_ _Analysis_assume_lock_released_(_Global_rcu_read_lock_); + +typedef enum _RCU_CALLBACK_TYPE +{ + RCU_CALLBACK_CALL = 1, + RCU_CALLBACK_FREE, + RCU_CALLBACK_SYNC, +} RCU_CALLBACK_TYPE; +typedef struct _RCU_CALLBACK RCU_CALLBACK; +typedef _Function_class_(RCU_CALLBACK_FN) +_IRQL_requires_max_(PASSIVE_LEVEL) +_IRQL_requires_same_ +VOID +RCU_CALLBACK_FN(_In_ RCU_CALLBACK *); +typedef RCU_CALLBACK_FN *PRCU_CALLBACK_FN; + +struct _RCU_CALLBACK +{ + RCU_CALLBACK *Next; + RCU_CALLBACK_TYPE Type; + union + { + PRCU_CALLBACK_FN Func; + ULONG_PTR Offset; + KEVENT Done; + }; +}; + +_IRQL_requires_max_(DISPATCH_LEVEL) +_IRQL_saves_ +_IRQL_raises_(DISPATCH_LEVEL) +_Acquires_rcu_ +static inline KIRQL RcuReadLock(VOID) +{ + _Analysis_assume_rcu_acquired_; + return KeRaiseIrqlToDpcLevel(); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Releases_rcu_ +static inline VOID +RcuReadUnlock(_In_ _Notliteral_ _IRQL_restores_ KIRQL Irql) +{ + _Analysis_assume_rcu_released_; + KeLowerIrql(Irql); +} + +// TODO: replace __rcu with proper SAL annotations perhaps? +#define __rcu + +#define RcuInitPointer(P, V) WritePointerNoFence(&(P), V) +#define RcuAssignPointer(P, V) WritePointerRelease(&(P), V) +#define RcuAccessPointer(P) ReadPointerNoFence(&(P)) + +_Requires_rcu_held_ +static inline PVOID +__RcuDereference(_In_ _Interlocked_operand_ PVOID CONST volatile *Source) +{ + return ReadPointerNoFence(Source); +} +#define RcuDereference(Type, P) ((Type *)__RcuDereference(&(P))) + +_Requires_lock_held_(Lock) +static inline PVOID +__RcuDereferenceProtected(_In_ PVOID *Address, _In_ PVOID Lock) +{ + return *Address; +} +#define RcuDereferenceProtected(Type, P, Lock) ((Type *)__RcuDereferenceProtected(&(P), Lock)) + +_IRQL_requires_max_(APC_LEVEL) +VOID RcuSynchronize(VOID); + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +__RcuCall(_Inout_ RCU_CALLBACK *Head); + +_IRQL_requires_max_(DISPATCH_LEVEL) +static inline VOID +RcuCall(_Out_ RCU_CALLBACK *Head, _In_ RCU_CALLBACK_FN Func) +{ + Head->Type = RCU_CALLBACK_CALL; + Head->Func = Func; + __RcuCall(Head); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +static inline VOID +__RcuFree(_Out_ RCU_CALLBACK *Head, _In_ SIZE_T Offset) +{ + Head->Type = RCU_CALLBACK_FREE; + Head->Offset = Offset; + __RcuCall(Head); +} + +#define RcuFree(Type, Head, Member) __RcuFree(&((Type *)(Head))->Member, FIELD_OFFSET(Type, Member)) + +_IRQL_requires_max_(APC_LEVEL) +VOID RcuBarrier(VOID); + +_IRQL_requires_max_(PASSIVE_LEVEL) +_Must_inspect_result_ +NTSTATUS +RcuDriverEntry(VOID); + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID RcuUnload(VOID); diff --git a/driver/receive.c b/driver/receive.c new file mode 100644 index 0000000..34ad8b6 --- /dev/null +++ b/driver/receive.c @@ -0,0 +1,634 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include "interlocked.h" +#include "cookie.h" +#include "device.h" +#include "messages.h" +#include "peer.h" +#include "queueing.h" +#include "rcu.h" +#include "socket.h" +#include "timers.h" +#include "logging.h" + +static VOID +UpdateRxStats(_Inout_ WG_PEER *Peer, _In_ CONST ULONG Len) +{ + Peer->RxBytes += Len; + Peer->Device->Statistics.ifHCInOctets += Len; + Peer->Device->Statistics.ifHCInUcastOctets += Len; + ++Peer->Device->Statistics.ifHCInUcastPkts; +} + +#define NBL_TYPE_LE32(Nbl) (((MESSAGE_HEADER *)MemGetValidatedNetBufferListData(Nbl))->Type) + +_IRQL_requires_max_(DISPATCH_LEVEL) +static VOID +ReceiveHandshakePacket(_Inout_ WG_DEVICE *Wg, _In_ NET_BUFFER_LIST *Nbl) +{ + COOKIE_MAC_STATE MacState; + WG_PEER *Peer = NULL; + /* This is global, so that our load calculation applies to the whole + * system. We don't care about races with it at all. + */ + static UINT64 LastUnderLoad; + BOOLEAN PacketNeedsCookie; + BOOLEAN UnderLoad; + UINT32_LE NblType = NBL_TYPE_LE32(Nbl); + NET_BUFFER *Nb = NET_BUFFER_LIST_FIRST_NB(Nbl); + CHAR EndpointName[SOCKADDR_STR_MAX_LEN]; + + if (NblType == CpuToLe32(MESSAGE_TYPE_HANDSHAKE_COOKIE)) + { + LogInfoNblRatelimited(Wg, "Receiving cookie response from %s", Nbl); + CookieMessageConsume(MemGetValidatedNetBufferListData(Nbl), Wg); + return; + } + + UnderLoad = NetBufferListQueueLength(&Wg->HandshakeRxQueue) >= MAX_QUEUED_INCOMING_HANDSHAKES / 8; + if (UnderLoad) + { + LastUnderLoad = KeQueryInterruptTime(); + } + else if (LastUnderLoad) + { + UnderLoad = !BirthdateHasExpired(LastUnderLoad, 1); + if (!UnderLoad) + LastUnderLoad = 0; + } + MacState = CookieValidatePacket(&Wg->CookieChecker, Nbl, UnderLoad); + if ((UnderLoad && MacState == VALID_MAC_WITH_COOKIE) || (!UnderLoad && MacState == VALID_MAC_BUT_NO_COOKIE)) + { + PacketNeedsCookie = FALSE; + } + else if (UnderLoad && MacState == VALID_MAC_BUT_NO_COOKIE) + { + PacketNeedsCookie = TRUE; + } + else + { + LogInfoNblRatelimited(Wg, "Invalid MAC of handshake, dropping packet from %s", Nbl); + return; + } + + switch (NblType) + { + case CpuToLe32(MESSAGE_TYPE_HANDSHAKE_INITIATION): { + MESSAGE_HANDSHAKE_INITIATION *Message = MemGetValidatedNetBufferListData(Nbl); + + if (PacketNeedsCookie) + { + PacketSendHandshakeCookie(Wg, Nbl, Message->SenderIndex); + return; + } + Peer = NoiseHandshakeConsumeInitiation(Message, Wg); + if (!Peer) + { + LogInfoNblRatelimited(Wg, "Invalid handshake initiation from %s", Nbl); + return; + } + SocketSetPeerEndpointFromNbl(Peer, Nbl); + SockaddrToString(EndpointName, &Peer->Endpoint.Addr); + LogInfoRatelimited(Wg, "Receiving handshake initiation from peer %llu (%s)", Peer->InternalId, EndpointName); + PacketSendHandshakeResponse(Peer); + break; + } + case CpuToLe32(MESSAGE_TYPE_HANDSHAKE_RESPONSE): { + MESSAGE_HANDSHAKE_RESPONSE *Message = MemGetValidatedNetBufferListData(Nbl); + + if (PacketNeedsCookie) + { + PacketSendHandshakeCookie(Wg, Nbl, Message->SenderIndex); + return; + } + Peer = NoiseHandshakeConsumeResponse(Message, Wg); + if (!Peer) + { + LogInfoNblRatelimited(Wg, "Invalid handshake response from %s", Nbl); + return; + } + SocketSetPeerEndpointFromNbl(Peer, Nbl); + SockaddrToString(EndpointName, &Peer->Endpoint.Addr); + LogInfoRatelimited(Wg, "Receiving handshake response from peer %llu (%s)", Peer->InternalId, EndpointName); + if (NoiseHandshakeBeginSession(&Peer->Handshake, &Peer->Keypairs)) + { + TimersSessionDerived(Peer); + TimersHandshakeComplete(Peer); + /* Calling this function will either send any existing + * packets in the queue and not send a keepalive, which + * is the best case, Or, if there's nothing in the + * queue, it will send a keepalive, in order to give + * immediate confirmation of the session. + */ + PacketSendKeepalive(Peer); + } + break; + } + } + + if (!Peer) + { + NT_ASSERTMSG("Somehow a wrong type of packet wound up in the handshake queue!", 0); + return; + } + + UpdateRxStats(Peer, NET_BUFFER_DATA_LENGTH(Nb)); + + TimersAnyAuthenticatedPacketReceived(Peer); + TimersAnyAuthenticatedPacketTraversal(Peer); + PeerPut(Peer); +} + +_Use_decl_annotations_ +VOID +PacketHandshakeRxWorker(MULTICORE_WORKQUEUE *WorkQueue) +{ + WG_DEVICE *Wg = CONTAINING_RECORD(WorkQueue, WG_DEVICE, HandshakeRxThreads); + NET_BUFFER_LIST *Nbl; + + while ((Nbl = NetBufferListInterlockedDequeue(&Wg->HandshakeRxQueue)) != NULL) + { + ReceiveHandshakePacket(Wg, Nbl); + FreeReceiveNetBufferList(Wg, Nbl); + } +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +static VOID +KeepKeyFresh(_Inout_ WG_PEER *Peer) +{ + NOISE_KEYPAIR *Keypair; + BOOLEAN Send; + KIRQL Irql; + + if (Peer->SentLastminuteHandshake) + return; + + Irql = RcuReadLock(); + Keypair = RcuDereference(NOISE_KEYPAIR, Peer->Keypairs.CurrentKeypair); + Send = Keypair && ReadBooleanNoFence(&Keypair->Sending.IsValid) && Keypair->IAmTheInitiator && + BirthdateHasExpired(Keypair->Sending.Birthdate, REJECT_AFTER_TIME - KEEPALIVE_TIMEOUT - REKEY_TIMEOUT); + RcuReadUnlock(Irql); + + if (Send) + { + Peer->SentLastminuteHandshake = TRUE; + PacketSendQueuedHandshakeInitiation(Peer, FALSE); + } +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Must_inspect_result_ +_Return_type_success_(return != FALSE) +static BOOLEAN +DecryptPacket(_In_ CONST SIMD_STATE *Simd, _Inout_ NET_BUFFER_LIST *Nbl, _Inout_opt_ NOISE_KEYPAIR *Keypair) +{ + if (!Keypair) + return FALSE; + + if (!ReadBooleanNoFence(&Keypair->Receiving.IsValid) || + BirthdateHasExpired(Keypair->Receiving.Birthdate, REJECT_AFTER_TIME) || + Keypair->ReceivingCounter.Counter >= REJECT_AFTER_MESSAGES) + { + WriteBooleanNoFence(&Keypair->Receiving.IsValid, FALSE); + return FALSE; + } + + NET_BUFFER *Nb = NET_BUFFER_LIST_FIRST_NB(Nbl); + WSK_BUF *Buffer = &NET_BUFFER_LIST_DATAGRAM_INDICATION(Nbl)->Buffer; + MESSAGE_DATA *Message = MemGetValidatedNetBufferListData(Nbl); + UINT64 Nonce = Le64ToCpu(Message->Counter); + NET_BUFFER_NONCE(Nb) = Nonce; + NET_BUFFER_DATA_LENGTH(Nb) = (ULONG)Buffer->Length - MessageDataLen(0); + return ChaCha20Poly1305DecryptMdl( + MemGetValidatedNetBufferListData(Nbl), + Buffer->Mdl, + (ULONG)Buffer->Length - sizeof(*Message), + Buffer->Offset + sizeof(*Message), + NULL, + 0, + Nonce, + Keypair->Receiving.Key, + Simd); +} + +/* This is RFC6479, a replay detection bitmap algorithm that avoids bitshifts */ +_IRQL_requires_max_(DISPATCH_LEVEL) +_Requires_lock_not_held_(Counter->Lock) +_Must_inspect_result_ +_Return_type_success_(return != FALSE) +static BOOLEAN +CounterValidate(_Inout_ NOISE_REPLAY_COUNTER *Counter, _In_ UINT64 TheirCounter) +{ + ULONG_PTR Index, IndexCurrent, Top, i; + KIRQL Irql; + BOOLEAN Ret = FALSE; + + KeAcquireSpinLock(&Counter->Lock, &Irql); + + if (Counter->Counter >= REJECT_AFTER_MESSAGES + 1 || TheirCounter >= REJECT_AFTER_MESSAGES) + goto out; + + ++TheirCounter; + + if ((COUNTER_WINDOW_SIZE + TheirCounter) < Counter->Counter) + goto out; + + Index = (ULONG_PTR)(TheirCounter >> BITS_PER_POINTER_SHIFT); + + if (TheirCounter > Counter->Counter) + { + IndexCurrent = (ULONG_PTR)(Counter->Counter >> BITS_PER_POINTER_SHIFT); + Top = min(Index - IndexCurrent, COUNTER_BITS_TOTAL / BITS_PER_POINTER); + for (i = 1; i <= Top; ++i) + Counter->Backtrack[(i + IndexCurrent) & ((COUNTER_BITS_TOTAL / BITS_PER_POINTER) - 1)] = 0; + Counter->Counter = TheirCounter; + } + + Index &= (COUNTER_BITS_TOTAL / BITS_PER_POINTER) - 1; + Ret = !InterlockedBitTestAndSetPtr((LONG_PTR *)&Counter->Backtrack[Index], TheirCounter & (BITS_PER_POINTER - 1)); + +out: + KeReleaseSpinLock(&Counter->Lock, Irql); + return Ret; +} + +#ifdef DBG +# include "selftest/counter.c" +#endif + +_IRQL_requires_max_(DISPATCH_LEVEL) +static BOOLEAN +PacketConsumeDataDone(_Inout_ WG_PEER *Peer, _Inout_ NET_BUFFER_LIST *Nbl, _In_ CONST ENDPOINT *Endpoint) +{ + ULONG Len, LenBeforeTrim; + WG_PEER *RoutedPeer; + NET_BUFFER *Nb = NET_BUFFER_LIST_FIRST_NB(Nbl); + UINT16_BE Proto; + VOID *Hdr; + CHAR EndpointName[SOCKADDR_STR_MAX_LEN], SrcStr[46] = ""; + + SocketSetPeerEndpoint(Peer, Endpoint); + + if (NoiseReceivedWithKeypair(&Peer->Keypairs, NET_BUFFER_LIST_KEYPAIR(Nbl))) + { + TimersHandshakeComplete(Peer); + PacketSendStagedPackets(Peer); + } + + KeepKeyFresh(Peer); + + TimersAnyAuthenticatedPacketReceived(Peer); + TimersAnyAuthenticatedPacketTraversal(Peer); + + /* A packet with length 0 is a keepalive packet */ + if (!NET_BUFFER_DATA_LENGTH(Nb)) + { + UpdateRxStats(Peer, MessageDataLen(0)); + SockaddrToString(EndpointName, &Peer->Endpoint.Addr); + LogInfoRatelimited( + Peer->Device, "Receiving keepalive packet from peer %llu (%s)", Peer->InternalId, EndpointName); + goto packetProcessed; + } + + TimersDataReceived(Peer); + + Nbl->SourceHandle = Peer->Device->MiniportAdapterHandle; + /* We've already verified the Poly1305 auth tag, which means this packet + * was not modified in transit. We can therefore tell the networking + * stack that all checksums of every layer of encapsulation have already + * been checked "by the hardware" and therefore is unnecessary to check + * again in software. + */ + NDIS_TCP_IP_CHECKSUM_NET_BUFFER_LIST_INFO *TcpIpChecksumNblInfo = + (NDIS_TCP_IP_CHECKSUM_NET_BUFFER_LIST_INFO *)&NET_BUFFER_LIST_INFO(Nbl, TcpIpChecksumNetBufferListInfo); + TcpIpChecksumNblInfo->Receive.TcpChecksumFailed = 0; + TcpIpChecksumNblInfo->Receive.TcpChecksumValueInvalid = 0; + TcpIpChecksumNblInfo->Receive.TcpChecksumSucceeded = 1; + TcpIpChecksumNblInfo->Receive.UdpChecksumFailed = 0; + TcpIpChecksumNblInfo->Receive.UdpChecksumSucceeded = 1; + TcpIpChecksumNblInfo->Receive.IpChecksumFailed = 0; + TcpIpChecksumNblInfo->Receive.IpChecksumValueInvalid = 0; + TcpIpChecksumNblInfo->Receive.IpChecksumSucceeded = 1; + Proto = IpTunnelParseProtocol(Nbl); + if (Proto == Htons(NDIS_ETH_TYPE_IPV4) && (Hdr = NdisGetDataBuffer(Nb, sizeof(IPV4HDR), NULL, 1, 0)) != NULL) + { + Len = Ntohs(((IPV4HDR *)Hdr)->TotLen); + if (Len < sizeof(IPV4HDR)) + goto dishonestPacketSize; + NdisSetNblFlag(Nbl, NDIS_NBL_FLAGS_IS_IPV4); + } + else if (Proto == Htons(NDIS_ETH_TYPE_IPV6) && (Hdr = NdisGetDataBuffer(Nb, sizeof(IPV6HDR), NULL, 1, 0)) != NULL) + { + Len = Ntohs(((IPV6HDR *)Hdr)->PayloadLen) + sizeof(IPV6HDR); + NdisSetNblFlag(Nbl, NDIS_NBL_FLAGS_IS_IPV6); + } + else + goto dishonestPacketType; + NET_BUFFER_LIST_INFO(Nbl, NetBufferListProtocolId) = (VOID *)Proto; + + if (Len > NET_BUFFER_DATA_LENGTH(Nb)) + goto dishonestPacketSize; + LenBeforeTrim = NET_BUFFER_DATA_LENGTH(Nb); + NET_BUFFER_DATA_LENGTH(Nb) = Len; + + RoutedPeer = AllowedIpsLookupSrc(&Peer->Device->PeerAllowedIps, Proto, Hdr); + PeerPut(RoutedPeer); /* We don't need the extra reference. */ + + if (RoutedPeer != Peer) + goto dishonestPacketPeer; + + NET_BUFFER_LIST_STATUS(Nbl) = NDIS_STATUS_SUCCESS; + UpdateRxStats(Peer, MessageDataLen(LenBeforeTrim)); + return TRUE; + +dishonestPacketPeer: + SockaddrToString(EndpointName, &Peer->Endpoint.Addr); + if (Proto == Htons(NDIS_ETH_TYPE_IPV4)) + RtlIpv4AddressToStringA((IN_ADDR *)&((IPV4HDR *)Hdr)->Saddr, SrcStr); + else if (Proto == Htons(NDIS_ETH_TYPE_IPV6)) + RtlIpv6AddressToStringA(&((IPV6HDR *)Hdr)->Saddr, SrcStr); + LogInfoRatelimited( + Peer->Device, "Packet has unallowed src IP (%s) from peer %llu (%s)", SrcStr, Peer->InternalId, EndpointName); + goto falsePacket; +dishonestPacketType: + SockaddrToString(EndpointName, &Peer->Endpoint.Addr); + LogInfoRatelimited( + Peer->Device, "Packet is neither ipv4 nor ipv6 from peer %llu (%s)", Peer->InternalId, EndpointName); + goto falsePacket; +dishonestPacketSize: + SockaddrToString(EndpointName, &Peer->Endpoint.Addr); + LogInfoRatelimited(Peer->Device, "Packet has incorrect size from peer %llu (%s)", Peer->InternalId, EndpointName); + goto falsePacket; +falsePacket: + ++Peer->Device->Statistics.ifInErrors; + ++Peer->Device->Statistics.ifInDiscards; +packetProcessed: + FreeReceiveNetBufferList(Peer->Device, Nbl); + return FALSE; +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +static BOOLEAN +PacketPeerRxWork(_Inout_ WG_PEER *Peer, _In_ ULONG Budget) +{ + NOISE_KEYPAIR *Keypair; + ENDPOINT Endpoint; + PACKET_STATE State; + NET_BUFFER_LIST *Nbl, *First = NULL, **Next = &First; + BOOLEAN Free, MoreProcessing = FALSE; + ULONG NumNbls = 0; + + while ((Nbl = PrevQueuePeek(&Peer->RxQueue)) != NULL && + (State = ReadAcquire(NET_BUFFER_LIST_CRYPT_STATE(Nbl))) != PACKET_STATE_UNCRYPTED) + { + if (!Budget--) + { + MoreProcessing = TRUE; + break; + } + PrevQueueDropPeeked(&Peer->RxQueue); + Keypair = NET_BUFFER_LIST_KEYPAIR(Nbl); + Free = TRUE; + + if (State != PACKET_STATE_CRYPTED) + goto next; + + UINT64 Nonce = NET_BUFFER_NONCE(NET_BUFFER_LIST_FIRST_NB(Nbl)); + if (!CounterValidate(&Keypair->ReceivingCounter, Nonce)) + { + LogInfoRatelimited( + Peer->Device, "Packet has invalid nonce %llu (max %llu)", Nonce, Keypair->ReceivingCounter.Counter); + goto next; + } + + if (!NT_SUCCESS(SocketEndpointFromNbl(&Endpoint, Nbl))) + goto next; + + if (PacketConsumeDataDone(Peer, Nbl, &Endpoint)) + { + *Next = Nbl; + Next = &NET_BUFFER_LIST_NEXT_NBL(Nbl); + ++NumNbls; + } + Free = FALSE; + + next: + NoiseKeypairPut(Keypair, FALSE); + if (Free) + FreeReceiveNetBufferList(Peer->Device, Nbl); + ExReleaseRundownProtection(&Peer->InUse); + PeerPut(Peer); + } + if (First) + NdisMIndicateReceiveNetBufferLists(First->SourceHandle, First, NDIS_DEFAULT_PORT_NUMBER, NumNbls, 0); + return MoreProcessing; +} + +_Use_decl_annotations_ +VOID +PacketRxWorker(MULTICORE_WORKQUEUE *WorkQueue) +{ + WG_DEVICE *Wg = CONTAINING_RECORD(WorkQueue, WG_DEVICE, RxThreads); + PEER_SERIAL_ENTRY *Entry; + while ((Entry = PeerSerialDequeue(&Wg->RxQueue)) != NULL) + PeerSerialMaybeRetire( + &Wg->RxQueue, + Entry, + PacketPeerRxWork(CONTAINING_RECORD(Entry, WG_PEER, RxSerialEntry), PEER_XMIT_PACKETS_PER_ROUND)); +} + +_Use_decl_annotations_ +VOID +PacketDecryptWorker(MULTICORE_WORKQUEUE *WorkQueue) +{ + WG_DEVICE *Wg = CONTAINING_RECORD(WorkQueue, WG_DEVICE, DecryptThreads); + PTR_RING *Ring = &Wg->DecryptQueue; + NET_BUFFER_LIST *First; + SIMD_STATE Simd; + + SimdGet(&Simd); + while ((First = PtrRingConsume(Ring)) != NULL) + { + for (NET_BUFFER_LIST *Nbl = First, *NextNbl; Nbl; Nbl = NextNbl) + { + WG_PEER *Peer = NET_BUFFER_LIST_PEER(Nbl); + NextNbl = NET_BUFFER_LIST_NEXT_NBL(Nbl); + NET_BUFFER_LIST_NEXT_NBL(Nbl) = NULL; + PACKET_STATE State = + DecryptPacket(&Simd, Nbl, NET_BUFFER_LIST_KEYPAIR(Nbl)) ? PACKET_STATE_CRYPTED : PACKET_STATE_DEAD; + QueueEnqueuePerPeer(&Peer->Device->RxQueue, &Peer->RxSerialEntry, &Peer->Device->RxThreads, Nbl, State); + } + } + SimdPut(&Simd); +} + +#pragma warning(suppress : 28194) /* `Nbl` is aliased in QueueEnqueuePerDeviceAndPeer, or QueueEnqueuePerPeer or freed \ + in FreeReceiveNetBufferList. */ +_IRQL_requires_max_(DISPATCH_LEVEL) +static VOID +PacketConsumeData(_Inout_ WG_DEVICE *Wg, _Inout_ __drv_aliasesMem NET_BUFFER_LIST *First) +{ + NET_BUFFER_LIST *FirstForDevice = NULL, **Link = &FirstForDevice; + for (NET_BUFFER_LIST *Nbl = First, *NextNbl; Nbl; Nbl = NextNbl) + { + NextNbl = NET_BUFFER_LIST_NEXT_NBL(Nbl); + NET_BUFFER_LIST_NEXT_NBL(Nbl) = NULL; + + MESSAGE_DATA *Message = MemGetValidatedNetBufferListData(Nbl); + WG_PEER *Peer = NULL; + NOISE_KEYPAIR *Keypair; + + KIRQL Irql = RcuReadLock(); + NET_BUFFER_LIST_KEYPAIR(Nbl) = Keypair = NoiseKeypairGet( + (NOISE_KEYPAIR *)IndexHashtableLookup(Wg->IndexHashtable, INDEX_HASHTABLE_KEYPAIR, Message->KeyIdx, &Peer)); + RcuReadUnlock(Irql); + if (!Keypair) + goto cleanupNbl; + + if (!ExAcquireRundownProtection(&Peer->InUse)) + goto cleanupKeypair; + if (!QueueInsertPerPeer(&Peer->RxQueue, Nbl)) + goto cleanupInUse; + *Link = Nbl; + Link = &NET_BUFFER_LIST_NEXT_NBL(Nbl); + continue; + + cleanupInUse: + ExReleaseRundownProtection(&Peer->InUse); + cleanupKeypair: + NoiseKeypairPut(Keypair, FALSE); + cleanupNbl: + FreeReceiveNetBufferList(Wg, Nbl); + PeerPut(Peer); + } + if (FirstForDevice && !QueueEnqueuePerDevice(&Wg->DecryptQueue, &Wg->DecryptThreads, FirstForDevice)) + { + for (NET_BUFFER_LIST *Nbl = FirstForDevice, *NextNbl; Nbl; Nbl = NextNbl) + { + WG_PEER *Peer = NET_BUFFER_LIST_PEER(Nbl); + NextNbl = NET_BUFFER_LIST_NEXT_NBL(Nbl); + NET_BUFFER_LIST_NEXT_NBL(Nbl) = NULL; + QueueEnqueuePerPeer( + &Peer->Device->RxQueue, &Peer->RxSerialEntry, &Peer->Device->RxThreads, Nbl, PACKET_STATE_DEAD); + } + } +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Return_type_success_(return == TRUE) +_Must_inspect_result_ +static BOOLEAN +PrepareNetBufferListHeader(_Inout_ NET_BUFFER_LIST *Nbl) +{ + WSK_BUF *Buffer = &NET_BUFFER_LIST_DATAGRAM_INDICATION(Nbl)->Buffer; + if (Buffer->Length < sizeof(MESSAGE_HEADER)) + return FALSE; + ULONG MdlLen = MmGetMdlByteCount(Buffer->Mdl); + if (Buffer->Offset > MdlLen) + return FALSE; + MdlLen -= Buffer->Offset; + /* We lazily require that the full header is in the first MDL. + * Later we can switch to something more complex if this turns + * out to be an actual problem with real indications. + */ + if (MdlLen < sizeof(MESSAGE_HEADER)) + return FALSE; + UCHAR *Src = + MmGetSystemAddressForMdlSafe(Buffer->Mdl, NormalPagePriority | MdlMappingNoExecute | MdlMappingNoWrite); + if (!Src) + return FALSE; + Src += Buffer->Offset; + MESSAGE_HEADER *Header = MemGetValidatedNetBufferListData(Nbl); + RtlCopyMemory(Header, Src, sizeof(*Header)); + ULONG HeaderLen, RequiredLen; + if (Header->Type == CpuToLe32(MESSAGE_TYPE_DATA)) + HeaderLen = sizeof(MESSAGE_DATA), RequiredLen = MESSAGE_MINIMUM_LENGTH; + else if (Header->Type == CpuToLe32(MESSAGE_TYPE_HANDSHAKE_INITIATION)) + RequiredLen = HeaderLen = sizeof(MESSAGE_HANDSHAKE_INITIATION); + else if (Header->Type == CpuToLe32(MESSAGE_TYPE_HANDSHAKE_RESPONSE)) + RequiredLen = HeaderLen = sizeof(MESSAGE_HANDSHAKE_RESPONSE); + else if (Header->Type == CpuToLe32(MESSAGE_TYPE_HANDSHAKE_COOKIE)) + RequiredLen = HeaderLen = sizeof(MESSAGE_HANDSHAKE_COOKIE); + else + return FALSE; + if (Buffer->Length < RequiredLen || MdlLen < HeaderLen) + return FALSE; + RtlCopyMemory(Header + 1, Src + sizeof(*Header), HeaderLen - sizeof(*Header)); + return TRUE; +} + +#pragma warning(suppress : 28194) /* `Nbl` is aliased in NetBufferListInterlockedEnqueue, or PacketConsumeData, \ + or freed in FreeReceiveNetBufferList. */ +_Use_decl_annotations_ +VOID +PacketReceive(WG_DEVICE *Wg, NET_BUFFER_LIST *First) +{ + NET_BUFFER_LIST *FirstData = NULL, **Link = &FirstData; + for (NET_BUFFER_LIST *Nbl = First, *NextNbl; Nbl; Nbl = NextNbl) + { + NextNbl = NET_BUFFER_LIST_NEXT_NBL(Nbl); + NET_BUFFER_LIST_NEXT_NBL(Nbl) = NULL; + + if (!PrepareNetBufferListHeader(Nbl)) + goto cleanup; + switch (NBL_TYPE_LE32(Nbl)) + { + case CpuToLe32(MESSAGE_TYPE_HANDSHAKE_INITIATION): + case CpuToLe32(MESSAGE_TYPE_HANDSHAKE_RESPONSE): + case CpuToLe32(MESSAGE_TYPE_HANDSHAKE_COOKIE): { + if (NetBufferListQueueLength(&Wg->HandshakeRxQueue) > MAX_QUEUED_INCOMING_HANDSHAKES) + { + LogInfoNblRatelimited(Wg, "Dropping handshake packet from %s", Nbl); + goto cleanup; + } + NetBufferListInterlockedEnqueue(&Wg->HandshakeRxQueue, Nbl); + MulticoreWorkQueueBump(&Wg->HandshakeRxThreads); + break; + } + case CpuToLe32(MESSAGE_TYPE_DATA): + *Link = Nbl; + Link = &NET_BUFFER_LIST_NEXT_NBL(Nbl); + break; + default: + goto cleanup; + } + continue; + + cleanup: + FreeReceiveNetBufferList(Wg, Nbl); + } + + if (FirstData) + PacketConsumeData(Wg, FirstData); +} + +_Use_decl_annotations_ +VOID +FreeReceiveNetBufferList(WG_DEVICE *Wg, NET_BUFFER_LIST *First) +{ + for (NET_BUFFER_LIST *Nbl = First, *NextNbl; Nbl; Nbl = NextNbl) + { + NextNbl = NET_BUFFER_LIST_NEXT_NBL(Nbl); + NET_BUFFER_LIST_NEXT_NBL(Nbl) = NULL; + WSK_DATAGRAM_INDICATION *DatagramIndication = NET_BUFFER_LIST_DATAGRAM_INDICATION(Nbl); + NET_BUFFER_LIST_DATAGRAM_INDICATION(Nbl) = NULL; + SOCKET *Socket = (SOCKET *)DatagramIndication->Next; + DatagramIndication->Next = NULL; + ((WSK_PROVIDER_DATAGRAM_DISPATCH *)Socket->Sock->Dispatch)->WskRelease(Socket->Sock, DatagramIndication); + MemFreeNetBufferList(Nbl); + ExReleaseRundownProtection(&Socket->ItemsInFlight); + } +} + +_Use_decl_annotations_ +VOID +FreeIncomingHandshakes(WG_DEVICE *Wg) +{ + NET_BUFFER_LIST *Nbl; + while ((Nbl = NetBufferListInterlockedDequeue(&Wg->HandshakeRxQueue)) != NULL) + FreeReceiveNetBufferList(Wg, Nbl); +} diff --git a/driver/selftest/allowedips.c b/driver/selftest/allowedips.c new file mode 100644 index 0000000..a6b57ef --- /dev/null +++ b/driver/selftest/allowedips.c @@ -0,0 +1,269 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +static inline IN_ADDR * +Ip4(UINT8 A, UINT8 B, UINT8 C, UINT8 D); +static inline IN6_ADDR * +Ip6(UINT32 A, UINT32 B, UINT32 C, UINT32 D); +static WG_PEER *InitPeer(VOID); + +#ifdef ALLOC_PRAGMA +# pragma alloc_text(INIT, Ip4) +# pragma alloc_text(INIT, Ip6) +# pragma alloc_text(INIT, InitPeer) +# pragma alloc_text(INIT, AllowedIpsSelftest) +#endif + +static inline IN_ADDR * +Ip4(UINT8 A, UINT8 B, UINT8 C, UINT8 D) +{ + static IN_ADDR Ip; + UINT8 *Split = (UINT8 *)&Ip; + + Split[0] = A; + Split[1] = B; + Split[2] = C; + Split[3] = D; + return &Ip; +} + +static inline IN6_ADDR * +Ip6(UINT32 A, UINT32 B, UINT32 C, UINT32 D) +{ + static IN6_ADDR Ip; + UINT32_BE *Split = (UINT32_BE *)&Ip; + + Split[0] = CpuToBe32(A); + Split[1] = CpuToBe32(B); + Split[2] = CpuToBe32(C); + Split[3] = CpuToBe32(D); + return &Ip; +} + +static WG_PEER *InitPeer(VOID) +{ + WG_PEER *Peer = MemAllocateAndZero(sizeof(*Peer)); + + if (!Peer) + return NULL; + KrefInit(&Peer->Refcount); + InitializeListHead(&Peer->AllowedIpsList); + return Peer; +} + +#define Insert(Version, Mem, Ipa, Ipb, Ipc, Ipd, Cidr) \ + AllowedIpsInsertV##Version(&t, Ip##Version(Ipa, Ipb, Ipc, Ipd), Cidr, Mem, &Mutex) + +#define MaybeFail() \ + do \ + { \ + ++i; \ + if (!_s) \ + { \ + LogDebug("allowedips self-test %zu: FAIL", i); \ + Success = FALSE; \ + } \ + } while (0) + +#define Test(Version, Mem, Ipa, Ipb, Ipc, Ipd) \ + do \ + { \ + BOOLEAN _s = Lookup(t.Root##Version, (Version) == 4 ? 32 : 128, Ip##Version(Ipa, Ipb, Ipc, Ipd)) == (Mem); \ + MaybeFail(); \ + } while (0) + +#define TestNegative(Version, Mem, Ipa, Ipb, Ipc, Ipd) \ + do \ + { \ + BOOLEAN _s = Lookup(t.Root##Version, (Version) == 4 ? 32 : 128, Ip##Version(Ipa, Ipb, Ipc, Ipd)) != (Mem); \ + MaybeFail(); \ + } while (0) + +#define TestBoolean(Cond) \ + do \ + { \ + BOOLEAN _s = (Cond); \ + MaybeFail(); \ + } while (0) + +_Use_decl_annotations_ +BOOLEAN +AllowedIpsSelftest(VOID) +{ + BOOLEAN FoundA = FALSE, FoundB = FALSE, FoundC = FALSE, FoundD = FALSE, FoundE = FALSE, FoundOther = FALSE; + WG_PEER *A = InitPeer(), *B = InitPeer(), *C = InitPeer(), *D = InitPeer(), *E = InitPeer(), *F = InitPeer(), + *G = InitPeer(), *H = InitPeer(); + ALLOWEDIPS_NODE *IterNode; + BOOLEAN Success = FALSE; + ALLOWEDIPS_TABLE t; + EX_PUSH_LOCK Mutex; + SIZE_T i = 0, Count = 0; + UINT64_BE Part; + + MuInitializePushLock(&Mutex); + MuAcquirePushLockExclusive(&Mutex); + AllowedIpsInit(&t); + + if (!A || !B || !C || !D || !E || !F || !G || !H) + { + LogDebug("allowedips self-test malloc: FAIL"); + goto free; + } + + Insert(4, A, 192, 168, 4, 0, 24); + Insert(4, B, 192, 168, 4, 4, 32); + Insert(4, C, 192, 168, 0, 0, 16); + Insert(4, D, 192, 95, 5, 64, 27); + /* replaces previous entry, and maskself is required */ + Insert(4, C, 192, 95, 5, 65, 27); + Insert(6, D, 0x26075300, 0x60006b00, 0, 0xc05f0543, 128); + Insert(6, C, 0x26075300, 0x60006b00, 0, 0, 64); + Insert(4, E, 0, 0, 0, 0, 0); + Insert(6, E, 0, 0, 0, 0, 0); + /* replaces previous entry */ + Insert(6, F, 0, 0, 0, 0, 0); + Insert(6, G, 0x24046800, 0, 0, 0, 32); + /* maskself is required */ + Insert(6, H, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef, 64); + Insert(6, A, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef, 128); + Insert(6, C, 0x24446800, 0x40e40800, 0xdeaebeef, 0xdefbeef, 128); + Insert(6, B, 0x24446800, 0xf0e40800, 0xeeaebeef, 0, 98); + Insert(4, G, 64, 15, 112, 0, 20); + /* maskself is required */ + Insert(4, H, 64, 15, 123, 211, 25); + Insert(4, A, 10, 0, 0, 0, 25); + Insert(4, B, 10, 0, 0, 128, 25); + Insert(4, A, 10, 1, 0, 0, 30); + Insert(4, B, 10, 1, 0, 4, 30); + Insert(4, C, 10, 1, 0, 8, 29); + Insert(4, D, 10, 1, 0, 16, 29); + + Success = TRUE; + + Test(4, A, 192, 168, 4, 20); + Test(4, A, 192, 168, 4, 0); + Test(4, B, 192, 168, 4, 4); + Test(4, C, 192, 168, 200, 182); + Test(4, C, 192, 95, 5, 68); + Test(4, E, 192, 95, 5, 96); + Test(6, D, 0x26075300, 0x60006b00, 0, 0xc05f0543); + Test(6, C, 0x26075300, 0x60006b00, 0, 0xc02e01ee); + Test(6, F, 0x26075300, 0x60006b01, 0, 0); + Test(6, G, 0x24046800, 0x40040806, 0, 0x1006); + Test(6, G, 0x24046800, 0x40040806, 0x1234, 0x5678); + Test(6, F, 0x240467ff, 0x40040806, 0x1234, 0x5678); + Test(6, F, 0x24046801, 0x40040806, 0x1234, 0x5678); + Test(6, H, 0x24046800, 0x40040800, 0x1234, 0x5678); + Test(6, H, 0x24046800, 0x40040800, 0, 0); + Test(6, H, 0x24046800, 0x40040800, 0x10101010, 0x10101010); + Test(6, A, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef); + Test(4, G, 64, 15, 116, 26); + Test(4, G, 64, 15, 127, 3); + Test(4, G, 64, 15, 123, 1); + Test(4, H, 64, 15, 123, 128); + Test(4, H, 64, 15, 123, 129); + Test(4, A, 10, 0, 0, 52); + Test(4, B, 10, 0, 0, 220); + Test(4, A, 10, 1, 0, 2); + Test(4, B, 10, 1, 0, 6); + Test(4, C, 10, 1, 0, 10); + Test(4, D, 10, 1, 0, 20); + + Insert(4, A, 1, 0, 0, 0, 32); + Insert(4, A, 64, 0, 0, 0, 32); + Insert(4, A, 128, 0, 0, 0, 32); + Insert(4, A, 192, 0, 0, 0, 32); + Insert(4, A, 255, 0, 0, 0, 32); + AllowedIpsRemoveByPeer(&t, A, &Mutex); + TestNegative(4, A, 1, 0, 0, 0); + TestNegative(4, A, 64, 0, 0, 0); + TestNegative(4, A, 128, 0, 0, 0); + TestNegative(4, A, 192, 0, 0, 0); + TestNegative(4, A, 255, 0, 0, 0); + + AllowedIpsFree(&t, &Mutex); + AllowedIpsInit(&t); + Insert(4, A, 192, 168, 0, 0, 16); + Insert(4, A, 192, 168, 0, 0, 24); + AllowedIpsRemoveByPeer(&t, A, &Mutex); + TestNegative(4, A, 192, 168, 0, 1); + + /* These will hit the NT_ASSERT(len < 128) in free_node if something + * goes wrong. + */ + for (i = 0; i < 128; ++i) + { + IN6_ADDR Ip; + + Part = CpuToBe64(~(1LLU << (i % 64))); + RtlFillMemory(&Ip, 16, 0xff); + RtlCopyMemory((UINT8 *)&Ip + (SIZE_T)(i < 64) * 8, &Part, 8); + AllowedIpsInsertV6(&t, &Ip, 128, A, &Mutex); + } + + AllowedIpsFree(&t, &Mutex); + + AllowedIpsInit(&t); + Insert(4, A, 192, 95, 5, 93, 27); + Insert(6, A, 0x26075300, 0x60006b00, 0, 0xc05f0543, 128); + Insert(4, A, 10, 1, 0, 20, 29); + Insert(6, A, 0x26075300, 0x6d8a6bf8, 0xdab1f1df, 0xc05f1523, 83); + Insert(6, A, 0x26075300, 0x6d8a6bf8, 0xdab1f1df, 0xc05f1523, 21); + LIST_FOR_EACH_ENTRY (IterNode, &A->AllowedIpsList, ALLOWEDIPS_NODE, PeerList) + { + UINT8 Cidr; + __declspec(align(8)) UINT8 Ip[16]; + ADDRESS_FAMILY Family = AllowedIpsReadNode(IterNode, Ip, &Cidr); + + ++Count; + + if (Cidr == 27 && Family == AF_INET && RtlEqualMemory(Ip, Ip4(192, 95, 5, 64), sizeof(IN_ADDR))) + FoundA = TRUE; + else if ( + Cidr == 128 && Family == AF_INET6 && + RtlEqualMemory(Ip, Ip6(0x26075300, 0x60006b00, 0, 0xc05f0543), sizeof(IN6_ADDR))) + FoundB = TRUE; + else if (Cidr == 29 && Family == AF_INET && RtlEqualMemory(Ip, Ip4(10, 1, 0, 16), sizeof(IN_ADDR))) + FoundC = TRUE; + else if ( + Cidr == 83 && Family == AF_INET6 && + RtlEqualMemory(Ip, Ip6(0x26075300, 0x6d8a6bf8, 0xdab1e000, 0), sizeof(IN6_ADDR))) + FoundD = TRUE; + else if (Cidr == 21 && Family == AF_INET6 && RtlEqualMemory(Ip, Ip6(0x26075000, 0, 0, 0), sizeof(IN6_ADDR))) + FoundE = TRUE; + else + FoundOther = TRUE; + } + TestBoolean(Count == 5); + TestBoolean(FoundA); + TestBoolean(FoundB); + TestBoolean(FoundC); + TestBoolean(FoundD); + TestBoolean(FoundE); + TestBoolean(!FoundOther); + + if (Success) + LogDebug("allowedips self-tests: pass"); + +free: + AllowedIpsFree(&t, &Mutex); + MemFree(A); + MemFree(B); + MemFree(C); + MemFree(D); + MemFree(E); + MemFree(F); + MemFree(G); + MemFree(H); + MuReleasePushLockExclusive(&Mutex); + + return Success; +} + +#undef TestNegative +#undef Test +#undef Remove +#undef Insert +#undef InitPeer diff --git a/driver/selftest/chacha20poly1305.c b/driver/selftest/chacha20poly1305.c new file mode 100644 index 0000000..8c2b43d --- /dev/null +++ b/driver/selftest/chacha20poly1305.c @@ -0,0 +1,5252 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include "../logging.h" +#include "../memory.h" + +struct ChaCha20Poly1305TestVector +{ + CONST UINT8 *Input, *Output, *Ad, *Nonce, *Key; + SIZE_T InLen, AdLen, NonceLen; + BOOLEAN Failure; +}; + +#pragma data_seg("INITDATA") +#pragma bss_seg("INITBSS") + +/* The first of these are the ChaCha20-Poly1305 AEAD test vectors from RFC7539 + * 2.8.2. After they are generated by reference implementations. And the final + * marked ones are taken from wycheproof, but we only do these for the encrypt + * side, because mostly we're stressing the primitives rather than the actual + * chapoly construction. This also requires adding a 96-bit nonce construction, + * just for the purpose of the tests. + */ + +static CONST UINT8 EncInput001[] = { + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, + 0x20, 0x64, 0x72, 0x61, 0x66, 0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, + 0x6c, 0x69, 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20, 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20, 0x6f, + 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d, 0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x6d, 0x61, + 0x79, 0x20, 0x62, 0x65, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, + 0x63, 0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f, 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x20, 0x62, + 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x20, 0x61, + 0x74, 0x20, 0x61, 0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69, + 0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72, 0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20, 0x75, 0x73, 0x65, + 0x20, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x73, + 0x20, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, 0x6c, + 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, 0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65, 0x6d, 0x20, 0x6f, 0x74, + 0x68, 0x65, 0x72, 0x20, 0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20, 0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, + 0x6b, 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d +}; +static CONST UINT8 EncOutput001[] = { + 0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4, 0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd, 0x5e, 0x80, 0x5c, + 0xfd, 0x34, 0x5c, 0xf3, 0x89, 0xf1, 0x08, 0x67, 0x0a, 0xc7, 0x6c, 0x8c, 0xb2, 0x4c, 0x6c, 0xfc, 0x18, 0x75, 0x5d, + 0x43, 0xee, 0xa0, 0x9e, 0xe9, 0x4e, 0x38, 0x2d, 0x26, 0xb0, 0xbd, 0xb7, 0xb7, 0x3c, 0x32, 0x1b, 0x01, 0x00, 0xd4, + 0xf0, 0x3b, 0x7f, 0x35, 0x58, 0x94, 0xcf, 0x33, 0x2f, 0x83, 0x0e, 0x71, 0x0b, 0x97, 0xce, 0x98, 0xc8, 0xa8, 0x4a, + 0xbd, 0x0b, 0x94, 0x81, 0x14, 0xad, 0x17, 0x6e, 0x00, 0x8d, 0x33, 0xbd, 0x60, 0xf9, 0x82, 0xb1, 0xff, 0x37, 0xc8, + 0x55, 0x97, 0x97, 0xa0, 0x6e, 0xf4, 0xf0, 0xef, 0x61, 0xc1, 0x86, 0x32, 0x4e, 0x2b, 0x35, 0x06, 0x38, 0x36, 0x06, + 0x90, 0x7b, 0x6a, 0x7c, 0x02, 0xb0, 0xf9, 0xf6, 0x15, 0x7b, 0x53, 0xc8, 0x67, 0xe4, 0xb9, 0x16, 0x6c, 0x76, 0x7b, + 0x80, 0x4d, 0x46, 0xa5, 0x9b, 0x52, 0x16, 0xcd, 0xe7, 0xa4, 0xe9, 0x90, 0x40, 0xc5, 0xa4, 0x04, 0x33, 0x22, 0x5e, + 0xe2, 0x82, 0xa1, 0xb0, 0xa0, 0x6c, 0x52, 0x3e, 0xaf, 0x45, 0x34, 0xd7, 0xf8, 0x3f, 0xa1, 0x15, 0x5b, 0x00, 0x47, + 0x71, 0x8c, 0xbc, 0x54, 0x6a, 0x0d, 0x07, 0x2b, 0x04, 0xb3, 0x56, 0x4e, 0xea, 0x1b, 0x42, 0x22, 0x73, 0xf5, 0x48, + 0x27, 0x1a, 0x0b, 0xb2, 0x31, 0x60, 0x53, 0xfa, 0x76, 0x99, 0x19, 0x55, 0xeb, 0xd6, 0x31, 0x59, 0x43, 0x4e, 0xce, + 0xbb, 0x4e, 0x46, 0x6d, 0xae, 0x5a, 0x10, 0x73, 0xa6, 0x72, 0x76, 0x27, 0x09, 0x7a, 0x10, 0x49, 0xe6, 0x17, 0xd9, + 0x1d, 0x36, 0x10, 0x94, 0xfa, 0x68, 0xf0, 0xff, 0x77, 0x98, 0x71, 0x30, 0x30, 0x5b, 0xea, 0xba, 0x2e, 0xda, 0x04, + 0xdf, 0x99, 0x7b, 0x71, 0x4d, 0x6c, 0x6f, 0x2c, 0x29, 0xa6, 0xad, 0x5c, 0xb4, 0x02, 0x2b, 0x02, 0x70, 0x9b, 0xee, + 0xad, 0x9d, 0x67, 0x89, 0x0c, 0xbb, 0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f, 0x38 +}; +static CONST UINT8 EncAssoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; +static CONST UINT8 EncNonce001[] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 }; +static CONST UINT8 EncKey001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, + 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, + 0x80, 0x09, 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 }; + +// static CONST UINT8 EncInput002[] = { }; +static CONST UINT8 EncOutput002[] = { 0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1, + 0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92 }; +// static CONST UINT8 EncAssoc002[] = { }; +static CONST UINT8 EncNonce002[] = { 0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e }; +static CONST UINT8 EncKey002[] = { 0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f, 0x2d, 0x29, 0x25, + 0x76, 0xd5, 0x75, 0x27, 0x86, 0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, + 0xc5, 0xef, 0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68 }; + +// static CONST UINT8 EncInput003[] = { }; +static CONST UINT8 EncOutput003[] = { 0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6, + 0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77 }; +static CONST UINT8 EncAssoc003[] = { 0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b }; +static CONST UINT8 EncNonce003[] = { 0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d }; +static CONST UINT8 EncKey003[] = { 0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88, 0x34, 0xd1, 0x13, + 0xaf, 0x57, 0xa1, 0xeb, 0x3a, 0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, + 0xbc, 0x08, 0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d }; + +static CONST UINT8 EncInput004[] = { 0xa4 }; +static CONST UINT8 EncOutput004[] = { 0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2, 0x6d, + 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac, 0x89 }; +static CONST UINT8 EncAssoc004[] = { 0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40 }; +static CONST UINT8 EncNonce004[] = { 0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4 }; +static CONST UINT8 EncKey004[] = { 0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8, 0x31, 0x80, 0x82, + 0xd7, 0xd8, 0xe8, 0xb5, 0xa1, 0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, + 0xa3, 0x3d, 0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e }; + +static CONST UINT8 EncInput005[] = { 0x2d }; +static CONST UINT8 EncOutput005[] = { 0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e, 0x6c, + 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c, 0xac }; +// static CONST UINT8 EncAssoc005[] = { }; +static CONST UINT8 EncNonce005[] = { 0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30 }; +static CONST UINT8 EncKey005[] = { 0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31, 0x0e, 0x92, 0x89, + 0x8b, 0xf4, 0x93, 0xc7, 0x87, 0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, + 0xa7, 0x01, 0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87 }; + +static CONST UINT8 EncInput006[] = { + 0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a, 0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92, 0x3c, 0xd9, 0x24, + 0x11, 0xa9, 0x71, 0xf9, 0x37, 0x14, 0x99, 0xfa, 0xbe, 0xe6, 0x80, 0xde, 0x50, 0xc9, 0x96, 0xd4, 0xb0, 0xec, 0x9e, + 0x17, 0xec, 0xd2, 0x5e, 0x72, 0x99, 0xfc, 0x0a, 0xe1, 0xcb, 0x48, 0xd2, 0x85, 0xdd, 0x2f, 0x90, 0xe0, 0x66, 0x3b, + 0xe6, 0x20, 0x74, 0xbe, 0x23, 0x8f, 0xcb, 0xb4, 0xe4, 0xda, 0x48, 0x40, 0xa6, 0xd1, 0x1b, 0xc7, 0x42, 0xce, 0x2f, + 0x0c, 0xa6, 0x85, 0x6e, 0x87, 0x37, 0x03, 0xb1, 0x7c, 0x25, 0x96, 0xa3, 0x05, 0xd8, 0xb0, 0xf4, 0xed, 0xea, 0xc2, + 0xf0, 0x31, 0x98, 0x6c, 0xd1, 0x14, 0x25, 0xc0, 0xcb, 0x01, 0x74, 0xd0, 0x82, 0xf4, 0x36, 0xf5, 0x41, 0xd5, 0xdc, + 0xca, 0xc5, 0xbb, 0x98, 0xfe, 0xfc, 0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde, 0x8f +}; +static CONST UINT8 EncOutput006[] = { + 0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1, 0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15, 0x81, 0x2c, 0xb5, + 0xf0, 0xc6, 0x2b, 0xc7, 0x8c, 0x56, 0xd1, 0xbf, 0x69, 0x6c, 0x07, 0xa0, 0xda, 0x65, 0x27, 0xc9, 0x90, 0x3d, 0xef, + 0x4b, 0x11, 0x0f, 0x19, 0x07, 0xfd, 0x29, 0x92, 0xd9, 0xc8, 0xf7, 0x99, 0x2e, 0x4a, 0xd0, 0xb8, 0x2c, 0xdc, 0x93, + 0xf5, 0x9e, 0x33, 0x78, 0xd1, 0x37, 0xc3, 0x66, 0xd7, 0x5e, 0xbc, 0x44, 0xbf, 0x53, 0xa5, 0xbc, 0xc4, 0xcb, 0x7b, + 0x3a, 0x8e, 0x7f, 0x02, 0xbd, 0xbb, 0xe7, 0xca, 0xa6, 0x6c, 0x6b, 0x93, 0x21, 0x93, 0x10, 0x61, 0xe7, 0x69, 0xd0, + 0x78, 0xf3, 0x07, 0x5a, 0x1a, 0x8f, 0x73, 0xaa, 0xb1, 0x4e, 0xd3, 0xda, 0x4f, 0xf3, 0x32, 0xe1, 0x66, 0x3e, 0x6c, + 0xc6, 0x13, 0xba, 0x06, 0x5b, 0xfc, 0x6a, 0xe5, 0x6f, 0x60, 0xfb, 0x07, 0x40, 0xb0, 0x8c, 0x9d, 0x84, 0x43, 0x6b, + 0xc1, 0xf7, 0x8d, 0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a, 0xeb +}; +static CONST UINT8 EncAssoc006[] = { 0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b }; +static CONST UINT8 EncNonce006[] = { 0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c }; +static CONST UINT8 EncKey006[] = { 0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae, 0xdf, 0x72, 0x7f, + 0x53, 0x72, 0x25, 0x1e, 0x78, 0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, + 0x93, 0xf9, 0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01 }; + +static CONST UINT8 EncInput007[] = { + 0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5, 0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a, 0x67, 0x30, 0x12, + 0xe2, 0x34, 0x77, 0x4b, 0xc1, 0xd9, 0xc6, 0x1f, 0xab, 0xc6, 0x18, 0x50, 0x17, 0xa7, 0x9d, 0x3c, 0xa6, 0xc5, 0x35, + 0x8c, 0x1c, 0xc0, 0xa1, 0x7c, 0x9f, 0x03, 0x89, 0xca, 0xe1, 0xe6, 0xe9, 0xd4, 0xd3, 0x88, 0xdb, 0xb4, 0x51, 0x9d, + 0xec, 0xb4, 0xfc, 0x52, 0xee, 0x6d, 0xf1, 0x75, 0x42, 0xc6, 0xfd, 0xbd, 0x7a, 0x8e, 0x86, 0xfc, 0x44, 0xb3, 0x4f, + 0xf3, 0xea, 0x67, 0x5a, 0x41, 0x13, 0xba, 0xb0, 0xdc, 0xe1, 0xd3, 0x2a, 0x7c, 0x22, 0xb3, 0xca, 0xac, 0x6a, 0x37, + 0x98, 0x3e, 0x1d, 0x40, 0x97, 0xf7, 0x9b, 0x1d, 0x36, 0x6b, 0xb3, 0x28, 0xbd, 0x60, 0x82, 0x47, 0x34, 0xaa, 0x2f, + 0x7d, 0xe9, 0xa8, 0x70, 0x81, 0x57, 0xd4, 0xb9, 0x77, 0x0a, 0x9d, 0x29, 0xa7, 0x84, 0x52, 0x4f, 0xc2, 0x4a, 0x40, + 0x3b, 0x3c, 0xd4, 0xc9, 0x2a, 0xdb, 0x4a, 0x53, 0xc4, 0xbe, 0x80, 0xe9, 0x51, 0x7f, 0x8f, 0xc7, 0xa2, 0xce, 0x82, + 0x5c, 0x91, 0x1e, 0x74, 0xd9, 0xd0, 0xbd, 0xd5, 0xf3, 0xfd, 0xda, 0x4d, 0x25, 0xb4, 0xbb, 0x2d, 0xac, 0x2f, 0x3d, + 0x71, 0x85, 0x7b, 0xcf, 0x3c, 0x7b, 0x3e, 0x0e, 0x22, 0x78, 0x0c, 0x29, 0xbf, 0xe4, 0xf4, 0x57, 0xb3, 0xcb, 0x49, + 0xa0, 0xfc, 0x1e, 0x05, 0x4e, 0x16, 0xbc, 0xd5, 0xa8, 0xa3, 0xee, 0x05, 0x35, 0xc6, 0x7c, 0xab, 0x60, 0x14, 0x55, + 0x1a, 0x8e, 0xc5, 0x88, 0x5d, 0xd5, 0x81, 0xc2, 0x81, 0xa5, 0xc4, 0x60, 0xdb, 0xaf, 0x77, 0x91, 0xe1, 0xce, 0xa2, + 0x7e, 0x7f, 0x42, 0xe3, 0xb0, 0x13, 0x1c, 0x1f, 0x25, 0x60, 0x21, 0xe2, 0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, + 0x2b, 0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3 +}; +static CONST UINT8 EncOutput007[] = { + 0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c, 0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8, 0xc9, 0x50, 0xc3, + 0xc6, 0xa5, 0xe3, 0xa4, 0x7c, 0xc3, 0x23, 0x49, 0x5e, 0xa9, 0xb9, 0x32, 0xeb, 0x8a, 0x7c, 0xca, 0xe5, 0xec, 0xfb, + 0x7c, 0xc0, 0xcb, 0x7d, 0xdc, 0x2c, 0x9d, 0x92, 0x55, 0x21, 0x0a, 0xc8, 0x43, 0x63, 0x59, 0x0a, 0x31, 0x70, 0x82, + 0x67, 0x41, 0x03, 0xf8, 0xdf, 0xf2, 0xac, 0xa7, 0x02, 0xd4, 0xd5, 0x8a, 0x2d, 0xc8, 0x99, 0x19, 0x66, 0xd0, 0xf6, + 0x88, 0x2c, 0x77, 0xd9, 0xd4, 0x0d, 0x6c, 0xbd, 0x98, 0xde, 0xe7, 0x7f, 0xad, 0x7e, 0x8a, 0xfb, 0xe9, 0x4b, 0xe5, + 0xf7, 0xe5, 0x50, 0xa0, 0x90, 0x3f, 0xd6, 0x22, 0x53, 0xe3, 0xfe, 0x1b, 0xcc, 0x79, 0x3b, 0xec, 0x12, 0x47, 0x52, + 0xa7, 0xd6, 0x04, 0xe3, 0x52, 0xe6, 0x93, 0x90, 0x91, 0x32, 0x73, 0x79, 0xb8, 0xd0, 0x31, 0xde, 0x1f, 0x9f, 0x2f, + 0x05, 0x38, 0x54, 0x2f, 0x35, 0x04, 0x39, 0xe0, 0xa7, 0xba, 0xc6, 0x52, 0xf6, 0x37, 0x65, 0x4c, 0x07, 0xa9, 0x7e, + 0xb3, 0x21, 0x6f, 0x74, 0x8c, 0xc9, 0xde, 0xdb, 0x65, 0x1b, 0x9b, 0xaa, 0x60, 0xb1, 0x03, 0x30, 0x6b, 0xb2, 0x03, + 0xc4, 0x1c, 0x04, 0xf8, 0x0f, 0x64, 0xaf, 0x46, 0xe4, 0x65, 0x99, 0x49, 0xe2, 0xea, 0xce, 0x78, 0x00, 0xd8, 0x8b, + 0xd5, 0x2e, 0xcf, 0xfc, 0x40, 0x49, 0xe8, 0x58, 0xdc, 0x34, 0x9c, 0x8c, 0x61, 0xbf, 0x0a, 0x8e, 0xec, 0x39, 0xa9, + 0x30, 0x05, 0x5a, 0xd2, 0x56, 0x01, 0xc7, 0xda, 0x8f, 0x4e, 0xbb, 0x43, 0xa3, 0x3a, 0xf9, 0x15, 0x2a, 0xd0, 0xa0, + 0x7a, 0x87, 0x34, 0x82, 0xfe, 0x8a, 0xd1, 0x2d, 0x5e, 0xc7, 0xbf, 0x04, 0x53, 0x5f, 0x3b, 0x36, 0xd4, 0x25, 0x5c, + 0x34, 0x7a, 0x8d, 0xd5, 0x05, 0xce, 0x72, 0xca, 0xef, 0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42, 0x3a, 0x00, + 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53 +}; +// static CONST UINT8 EncAssoc007[] = { }; +static CONST UINT8 EncNonce007[] = { 0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0 }; +static CONST UINT8 EncKey007[] = { 0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd, 0xf9, 0x3f, 0xcd, + 0xd9, 0xa0, 0x1e, 0x42, 0x4c, 0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, + 0x05, 0x80, 0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01 }; + +static CONST UINT8 EncInput008[] = { + 0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10, 0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2, 0x0f, 0xc2, 0x8b, + 0x28, 0xdc, 0xba, 0xb4, 0x3c, 0xe4, 0x21, 0x58, 0x61, 0xcd, 0x8b, 0xcd, 0xfb, 0xac, 0x94, 0xa1, 0x45, 0xf5, 0x1c, + 0xe1, 0x12, 0xe0, 0x3b, 0x67, 0x21, 0x54, 0x5e, 0x8c, 0xaa, 0xcf, 0xdb, 0xb4, 0x51, 0xd4, 0x13, 0xda, 0xe6, 0x83, + 0x89, 0xb6, 0x92, 0xe9, 0x21, 0x76, 0xa4, 0x93, 0x7d, 0x0e, 0xfd, 0x96, 0x36, 0x03, 0x91, 0x43, 0x5c, 0x92, 0x49, + 0x62, 0x61, 0x7b, 0xeb, 0x43, 0x89, 0xb8, 0x12, 0x20, 0x43, 0xd4, 0x47, 0x06, 0x84, 0xee, 0x47, 0xe9, 0x8a, 0x73, + 0x15, 0x0f, 0x72, 0xcf, 0xed, 0xce, 0x96, 0xb2, 0x7f, 0x21, 0x45, 0x76, 0xeb, 0x26, 0x28, 0x83, 0x6a, 0xad, 0xaa, + 0xa6, 0x81, 0xd8, 0x55, 0xb1, 0xa3, 0x85, 0xb3, 0x0c, 0xdf, 0xf1, 0x69, 0x2d, 0x97, 0x05, 0x2a, 0xbc, 0x7c, 0x7b, + 0x25, 0xf8, 0x80, 0x9d, 0x39, 0x25, 0xf3, 0x62, 0xf0, 0x66, 0x5e, 0xf4, 0xa0, 0xcf, 0xd8, 0xfd, 0x4f, 0xb1, 0x1f, + 0x60, 0x3a, 0x08, 0x47, 0xaf, 0xe1, 0xf6, 0x10, 0x77, 0x09, 0xa7, 0x27, 0x8f, 0x9a, 0x97, 0x5a, 0x26, 0xfa, 0xfe, + 0x41, 0x32, 0x83, 0x10, 0xe0, 0x1d, 0xbf, 0x64, 0x0d, 0xf4, 0x1c, 0x32, 0x35, 0xe5, 0x1b, 0x36, 0xef, 0xd4, 0x4a, + 0x93, 0x4d, 0x00, 0x7c, 0xec, 0x02, 0x07, 0x8b, 0x5d, 0x7d, 0x1b, 0x0e, 0xd1, 0xa6, 0xa5, 0x5d, 0x7d, 0x57, 0x88, + 0xa8, 0xcc, 0x81, 0xb4, 0x86, 0x4e, 0xb4, 0x40, 0xe9, 0x1d, 0xc3, 0xb1, 0x24, 0x3e, 0x7f, 0xcc, 0x8a, 0x24, 0x9b, + 0xdf, 0x6d, 0xf0, 0x39, 0x69, 0x3e, 0x4c, 0xc0, 0x96, 0xe4, 0x13, 0xda, 0x90, 0xda, 0xf4, 0x95, 0x66, 0x8b, 0x17, + 0x17, 0xfe, 0x39, 0x43, 0x25, 0xaa, 0xda, 0xa0, 0x43, 0x3c, 0xb1, 0x41, 0x02, 0xa3, 0xf0, 0xa7, 0x19, 0x59, 0xbc, + 0x1d, 0x7d, 0x6c, 0x6d, 0x91, 0x09, 0x5c, 0xb7, 0x5b, 0x01, 0xd1, 0x6f, 0x17, 0x21, 0x97, 0xbf, 0x89, 0x71, 0xa5, + 0xb0, 0x6e, 0x07, 0x45, 0xfd, 0x9d, 0xea, 0x07, 0xf6, 0x7a, 0x9f, 0x10, 0x18, 0x22, 0x30, 0x73, 0xac, 0xd4, 0x6b, + 0x72, 0x44, 0xed, 0xd9, 0x19, 0x9b, 0x2d, 0x4a, 0x41, 0xdd, 0xd1, 0x85, 0x5e, 0x37, 0x19, 0xed, 0xd2, 0x15, 0x8f, + 0x5e, 0x91, 0xdb, 0x33, 0xf2, 0xe4, 0xdb, 0xff, 0x98, 0xfb, 0xa3, 0xb5, 0xca, 0x21, 0x69, 0x08, 0xe7, 0x8a, 0xdf, + 0x90, 0xff, 0x3e, 0xe9, 0x20, 0x86, 0x3c, 0xe9, 0xfc, 0x0b, 0xfe, 0x5c, 0x61, 0xaa, 0x13, 0x92, 0x7f, 0x7b, 0xec, + 0xe0, 0x6d, 0xa8, 0x23, 0x22, 0xf6, 0x6b, 0x77, 0xc4, 0xfe, 0x40, 0x07, 0x3b, 0xb6, 0xf6, 0x8e, 0x5f, 0xd4, 0xb9, + 0xb7, 0x0f, 0x21, 0x04, 0xef, 0x83, 0x63, 0x91, 0x69, 0x40, 0xa3, 0x48, 0x5c, 0xd2, 0x60, 0xf9, 0x4f, 0x6c, 0x47, + 0x8b, 0x3b, 0xb1, 0x9f, 0x8e, 0xee, 0x16, 0x8a, 0x13, 0xfc, 0x46, 0x17, 0xc3, 0xc3, 0x32, 0x56, 0xf8, 0x3c, 0x85, + 0x3a, 0xb6, 0x3e, 0xaa, 0x89, 0x4f, 0xb3, 0xdf, 0x38, 0xfd, 0xf1, 0xe4, 0x3a, 0xc0, 0xe6, 0x58, 0xb5, 0x8f, 0xc5, + 0x29, 0xa2, 0x92, 0x4a, 0xb6, 0xa0, 0x34, 0x7f, 0xab, 0xb5, 0x8a, 0x90, 0xa1, 0xdb, 0x4d, 0xca, 0xb6, 0x2c, 0x41, + 0x3c, 0xf7, 0x2b, 0x21, 0xc3, 0xfd, 0xf4, 0x17, 0x5c, 0xb5, 0x33, 0x17, 0x68, 0x2b, 0x08, 0x30, 0xf3, 0xf7, 0x30, + 0x3c, 0x96, 0xe6, 0x6a, 0x20, 0x97, 0xe7, 0x4d, 0x10, 0x5f, 0x47, 0x5f, 0x49, 0x96, 0x09, 0xf0, 0x27, 0x91, 0xc8, + 0xf8, 0x5a, 0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b, 0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3 +}; +static CONST UINT8 EncOutput008[] = { + 0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd, 0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1, 0x1e, 0x6b, 0xd2, + 0xbc, 0x11, 0xf4, 0x28, 0x93, 0x63, 0x54, 0xef, 0xbb, 0x5e, 0x1d, 0x3a, 0x1d, 0x37, 0x3c, 0x0a, 0x6c, 0x1e, 0xc2, + 0xd1, 0x2c, 0xb5, 0xa3, 0xb5, 0x7b, 0xb8, 0x8f, 0x25, 0xa6, 0x1b, 0x61, 0x1c, 0xec, 0x28, 0x58, 0x26, 0xa4, 0xa8, + 0x33, 0x28, 0x25, 0x5c, 0x45, 0x05, 0xe5, 0x6c, 0x99, 0xe5, 0x45, 0xc4, 0xa2, 0x03, 0x84, 0x03, 0x73, 0x1e, 0x8c, + 0x49, 0xac, 0x20, 0xdd, 0x8d, 0xb3, 0xc4, 0xf5, 0xe7, 0x4f, 0xf1, 0xed, 0xa1, 0x98, 0xde, 0xa4, 0x96, 0xdd, 0x2f, + 0xab, 0xab, 0x97, 0xcf, 0x3e, 0xd2, 0x9e, 0xb8, 0x13, 0x07, 0x28, 0x29, 0x19, 0xaf, 0xfd, 0xf2, 0x49, 0x43, 0xea, + 0x49, 0x26, 0x91, 0xc1, 0x07, 0xd6, 0xbb, 0x81, 0x75, 0x35, 0x0d, 0x24, 0x7f, 0xc8, 0xda, 0xd4, 0xb7, 0xeb, 0xe8, + 0x5c, 0x09, 0xa2, 0x2f, 0xdc, 0x28, 0x7d, 0x3a, 0x03, 0xfa, 0x94, 0xb5, 0x1d, 0x17, 0x99, 0x36, 0xc3, 0x1c, 0x18, + 0x34, 0xe3, 0x9f, 0xf5, 0x55, 0x7c, 0xb0, 0x60, 0x9d, 0xff, 0xac, 0xd4, 0x61, 0xf2, 0xad, 0xf8, 0xce, 0xc7, 0xbe, + 0x5c, 0xd2, 0x95, 0xa8, 0x4b, 0x77, 0x13, 0x19, 0x59, 0x26, 0xc9, 0xb7, 0x8f, 0x6a, 0xcb, 0x2d, 0x37, 0x91, 0xea, + 0x92, 0x9c, 0x94, 0x5b, 0xda, 0x0b, 0xce, 0xfe, 0x30, 0x20, 0xf8, 0x51, 0xad, 0xf2, 0xbe, 0xe7, 0xc7, 0xff, 0xb3, + 0x33, 0x91, 0x6a, 0xc9, 0x1a, 0x41, 0xc9, 0x0f, 0xf3, 0x10, 0x0e, 0xfd, 0x53, 0xff, 0x6c, 0x16, 0x52, 0xd9, 0xf3, + 0xf7, 0x98, 0x2e, 0xc9, 0x07, 0x31, 0x2c, 0x0c, 0x72, 0xd7, 0xc5, 0xc6, 0x08, 0x2a, 0x7b, 0xda, 0xbd, 0x7e, 0x02, + 0xea, 0x1a, 0xbb, 0xf2, 0x04, 0x27, 0x61, 0x28, 0x8e, 0xf5, 0x04, 0x03, 0x1f, 0x4c, 0x07, 0x55, 0x82, 0xec, 0x1e, + 0xd7, 0x8b, 0x2f, 0x65, 0x56, 0xd1, 0xd9, 0x1e, 0x3c, 0xe9, 0x1f, 0x5e, 0x98, 0x70, 0x38, 0x4a, 0x8c, 0x49, 0xc5, + 0x43, 0xa0, 0xa1, 0x8b, 0x74, 0x9d, 0x4c, 0x62, 0x0d, 0x10, 0x0c, 0xf4, 0x6c, 0x8f, 0xe0, 0xaa, 0x9a, 0x8d, 0xb7, + 0xe0, 0xbe, 0x4c, 0x87, 0xf1, 0x98, 0x2f, 0xcc, 0xed, 0xc0, 0x52, 0x29, 0xdc, 0x83, 0xf8, 0xfc, 0x2c, 0x0e, 0xa8, + 0x51, 0x4d, 0x80, 0x0d, 0xa3, 0xfe, 0xd8, 0x37, 0xe7, 0x41, 0x24, 0xfc, 0xfb, 0x75, 0xe3, 0x71, 0x7b, 0x57, 0x45, + 0xf5, 0x97, 0x73, 0x65, 0x63, 0x14, 0x74, 0xb8, 0x82, 0x9f, 0xf8, 0x60, 0x2f, 0x8a, 0xf2, 0x4e, 0xf1, 0x39, 0xda, + 0x33, 0x91, 0xf8, 0x36, 0xe0, 0x8d, 0x3f, 0x1f, 0x3b, 0x56, 0xdc, 0xa0, 0x8f, 0x3c, 0x9d, 0x71, 0x52, 0xa7, 0xb8, + 0xc0, 0xa5, 0xc6, 0xa2, 0x73, 0xda, 0xf4, 0x4b, 0x74, 0x5b, 0x00, 0x3d, 0x99, 0xd7, 0x96, 0xba, 0xe6, 0xe1, 0xa6, + 0x96, 0x38, 0xad, 0xb3, 0xc0, 0xd2, 0xba, 0x91, 0x6b, 0xf9, 0x19, 0xdd, 0x3b, 0xbe, 0xbe, 0x9c, 0x20, 0x50, 0xba, + 0xa1, 0xd0, 0xce, 0x11, 0xbd, 0x95, 0xd8, 0xd1, 0xdd, 0x33, 0x85, 0x74, 0xdc, 0xdb, 0x66, 0x76, 0x44, 0xdc, 0x03, + 0x74, 0x48, 0x35, 0x98, 0xb1, 0x18, 0x47, 0x94, 0x7d, 0xff, 0x62, 0xe4, 0x58, 0x78, 0xab, 0xed, 0x95, 0x36, 0xd9, + 0x84, 0x91, 0x82, 0x64, 0x41, 0xbb, 0x58, 0xe6, 0x1c, 0x20, 0x6d, 0x15, 0x6b, 0x13, 0x96, 0xe8, 0x35, 0x7f, 0xdc, + 0x40, 0x2c, 0xe9, 0xbc, 0x8a, 0x4f, 0x92, 0xec, 0x06, 0x2d, 0x50, 0xdf, 0x93, 0x5d, 0x65, 0x5a, 0xa8, 0xfc, 0x20, + 0x50, 0x14, 0xa9, 0x8a, 0x7e, 0x1d, 0x08, 0x1f, 0xe2, 0x99, 0xd0, 0xbe, 0xfb, 0x3a, 0x21, 0x9d, 0xad, 0x86, 0x54, + 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f, 0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54 +}; +// static CONST UINT8 EncAssoc008[] = { }; +static CONST UINT8 EncNonce008[] = { 0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02 }; +static CONST UINT8 EncKey008[] = { 0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53, 0xc1, 0x44, 0xe9, + 0x81, 0x18, 0xdc, 0xf5, 0xf0, 0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, + 0x44, 0x86, 0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba }; + +static CONST UINT8 EncInput009[] = { + 0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b, 0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8, 0x3a, 0x6b, 0xd7, + 0x81, 0x96, 0x35, 0x97, 0xca, 0xd7, 0x68, 0xe8, 0xef, 0xce, 0xab, 0xda, 0x09, 0x6e, 0xd6, 0x8e, 0xcb, 0x55, 0xb5, + 0xe1, 0xe5, 0x57, 0xfd, 0xc4, 0xe3, 0xe0, 0x18, 0x4f, 0x85, 0xf5, 0x3f, 0x7e, 0x4b, 0x88, 0xc9, 0x52, 0x44, 0x0f, + 0xea, 0xaf, 0x1f, 0x71, 0x48, 0x9f, 0x97, 0x6d, 0xb9, 0x6f, 0x00, 0xa6, 0xde, 0x2b, 0x77, 0x8b, 0x15, 0xad, 0x10, + 0xa0, 0x2b, 0x7b, 0x41, 0x90, 0x03, 0x2d, 0x69, 0xae, 0xcc, 0x77, 0x7c, 0xa5, 0x9d, 0x29, 0x22, 0xc2, 0xea, 0xb4, + 0x00, 0x1a, 0xd2, 0x7a, 0x98, 0x8a, 0xf9, 0xf7, 0x82, 0xb0, 0xab, 0xd8, 0xa6, 0x94, 0x8d, 0x58, 0x2f, 0x01, 0x9e, + 0x00, 0x20, 0xfc, 0x49, 0xdc, 0x0e, 0x03, 0xe8, 0x45, 0x10, 0xd6, 0xa8, 0xda, 0x55, 0x10, 0x9a, 0xdf, 0x67, 0x22, + 0x8b, 0x43, 0xab, 0x00, 0xbb, 0x02, 0xc8, 0xdd, 0x7b, 0x97, 0x17, 0xd7, 0x1d, 0x9e, 0x02, 0x5e, 0x48, 0xde, 0x8e, + 0xcf, 0x99, 0x07, 0x95, 0x92, 0x3c, 0x5f, 0x9f, 0xc5, 0x8a, 0xc0, 0x23, 0xaa, 0xd5, 0x8c, 0x82, 0x6e, 0x16, 0x92, + 0xb1, 0x12, 0x17, 0x07, 0xc3, 0xfb, 0x36, 0xf5, 0x6c, 0x35, 0xd6, 0x06, 0x1f, 0x9f, 0xa7, 0x94, 0xa2, 0x38, 0x63, + 0x9c, 0xb0, 0x71, 0xb3, 0xa5, 0xd2, 0xd8, 0xba, 0x9f, 0x08, 0x01, 0xb3, 0xff, 0x04, 0x97, 0x73, 0x45, 0x1b, 0xd5, + 0xa9, 0x9c, 0x80, 0xaf, 0x04, 0x9a, 0x85, 0xdb, 0x32, 0x5b, 0x5d, 0x1a, 0xc1, 0x36, 0x28, 0x10, 0x79, 0xf1, 0x3c, + 0xbf, 0x1a, 0x41, 0x5c, 0x4e, 0xdf, 0xb2, 0x7c, 0x79, 0x3b, 0x7a, 0x62, 0x3d, 0x4b, 0xc9, 0x9b, 0x2a, 0x2e, 0x7c, + 0xa2, 0xb1, 0x11, 0x98, 0xa7, 0x34, 0x1a, 0x00, 0xf3, 0xd1, 0xbc, 0x18, 0x22, 0xba, 0x02, 0x56, 0x62, 0x31, 0x10, + 0x11, 0x6d, 0xe0, 0x54, 0x9d, 0x40, 0x1f, 0x26, 0x80, 0x41, 0xca, 0x3f, 0x68, 0x0f, 0x32, 0x1d, 0x0a, 0x8e, 0x79, + 0xd8, 0xa4, 0x1b, 0x29, 0x1c, 0x90, 0x8e, 0xc5, 0xe3, 0xb4, 0x91, 0x37, 0x9a, 0x97, 0x86, 0x99, 0xd5, 0x09, 0xc5, + 0xbb, 0xa3, 0x3f, 0x21, 0x29, 0x82, 0x14, 0x5c, 0xab, 0x25, 0xfb, 0xf2, 0x4f, 0x58, 0x26, 0xd4, 0x83, 0xaa, 0x66, + 0x89, 0x67, 0x7e, 0xc0, 0x49, 0xe1, 0x11, 0x10, 0x7f, 0x7a, 0xda, 0x29, 0x04, 0xff, 0xf0, 0xcb, 0x09, 0x7c, 0x9d, + 0xfa, 0x03, 0x6f, 0x81, 0x09, 0x31, 0x60, 0xfb, 0x08, 0xfa, 0x74, 0xd3, 0x64, 0x44, 0x7c, 0x55, 0x85, 0xec, 0x9c, + 0x6e, 0x25, 0xb7, 0x6c, 0xc5, 0x37, 0xb6, 0x83, 0x87, 0x72, 0x95, 0x8b, 0x9d, 0xe1, 0x69, 0x5c, 0x31, 0x95, 0x42, + 0xa6, 0x2c, 0xd1, 0x36, 0x47, 0x1f, 0xec, 0x54, 0xab, 0xa2, 0x1c, 0xd8, 0x00, 0xcc, 0xbc, 0x0d, 0x65, 0xe2, 0x67, + 0xbf, 0xbc, 0xea, 0xee, 0x9e, 0xe4, 0x36, 0x95, 0xbe, 0x73, 0xd9, 0xa6, 0xd9, 0x0f, 0xa0, 0xcc, 0x82, 0x76, 0x26, + 0xad, 0x5b, 0x58, 0x6c, 0x4e, 0xab, 0x29, 0x64, 0xd3, 0xd9, 0xa9, 0x08, 0x8c, 0x1d, 0xa1, 0x4f, 0x80, 0xd8, 0x3f, + 0x94, 0xfb, 0xd3, 0x7b, 0xfc, 0xd1, 0x2b, 0xc3, 0x21, 0xeb, 0xe5, 0x1c, 0x84, 0x23, 0x7f, 0x4b, 0xfa, 0xdb, 0x34, + 0x18, 0xa2, 0xc2, 0xe5, 0x13, 0xfe, 0x6c, 0x49, 0x81, 0xd2, 0x73, 0xe7, 0xe2, 0xd7, 0xe4, 0x4f, 0x4b, 0x08, 0x6e, + 0xb1, 0x12, 0x22, 0x10, 0x9d, 0xac, 0x51, 0x1e, 0x17, 0xd9, 0x8a, 0x0b, 0x42, 0x88, 0x16, 0x81, 0x37, 0x7c, 0x6a, + 0xf7, 0xef, 0x2d, 0xe3, 0xd9, 0xf8, 0x5f, 0xe0, 0x53, 0x27, 0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52, 0x65 +}; +static CONST UINT8 EncOutput009[] = { + 0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf, 0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66, 0x4b, 0x2e, 0x0c, + 0x27, 0x9c, 0x96, 0x4c, 0x72, 0x02, 0xa3, 0x65, 0xc3, 0xb3, 0x6f, 0x2e, 0xbd, 0x63, 0x8a, 0x4a, 0x5d, 0x29, 0xa2, + 0xd0, 0x28, 0x48, 0xc5, 0x3d, 0x98, 0xa3, 0xbc, 0xe0, 0xbe, 0x3b, 0x3f, 0xe6, 0x8a, 0xa4, 0x7f, 0x53, 0x06, 0xfa, + 0x7f, 0x27, 0x76, 0x72, 0x31, 0xa1, 0xf5, 0xd6, 0x0c, 0x52, 0x47, 0xba, 0xcd, 0x4f, 0xd7, 0xeb, 0x05, 0x48, 0x0d, + 0x7c, 0x35, 0x4a, 0x09, 0xc9, 0x76, 0x71, 0x02, 0xa3, 0xfb, 0xb7, 0x1a, 0x65, 0xb7, 0xed, 0x98, 0xc6, 0x30, 0x8a, + 0x00, 0xae, 0xa1, 0x31, 0xe5, 0xb5, 0x9e, 0x6d, 0x62, 0xda, 0xda, 0x07, 0x0f, 0x38, 0x38, 0xd3, 0xcb, 0xc1, 0xb0, + 0xad, 0xec, 0x72, 0xec, 0xb1, 0xa2, 0x7b, 0x59, 0xf3, 0x3d, 0x2b, 0xef, 0xcd, 0x28, 0x5b, 0x83, 0xcc, 0x18, 0x91, + 0x88, 0xb0, 0x2e, 0xf9, 0x29, 0x31, 0x18, 0xf9, 0x4e, 0xe9, 0x0a, 0x91, 0x92, 0x9f, 0xae, 0x2d, 0xad, 0xf4, 0xe6, + 0x1a, 0xe2, 0xa4, 0xee, 0x47, 0x15, 0xbf, 0x83, 0x6e, 0xd7, 0x72, 0x12, 0x3b, 0x2d, 0x24, 0xe9, 0xb2, 0x55, 0xcb, + 0x3c, 0x10, 0xf0, 0x24, 0x8a, 0x4a, 0x02, 0xea, 0x90, 0x25, 0xf0, 0xb4, 0x79, 0x3a, 0xef, 0x6e, 0xf5, 0x52, 0xdf, + 0xb0, 0x0a, 0xcd, 0x24, 0x1c, 0xd3, 0x2e, 0x22, 0x74, 0xea, 0x21, 0x6f, 0xe9, 0xbd, 0xc8, 0x3e, 0x36, 0x5b, 0x19, + 0xf1, 0xca, 0x99, 0x0a, 0xb4, 0xa7, 0x52, 0x1a, 0x4e, 0xf2, 0xad, 0x8d, 0x56, 0x85, 0xbb, 0x64, 0x89, 0xba, 0x26, + 0xf9, 0xc7, 0xe1, 0x89, 0x19, 0x22, 0x77, 0xc3, 0xa8, 0xfc, 0xff, 0xad, 0xfe, 0xb9, 0x48, 0xae, 0x12, 0x30, 0x9f, + 0x19, 0xfb, 0x1b, 0xef, 0x14, 0x87, 0x8a, 0x78, 0x71, 0xf3, 0xf4, 0xb7, 0x00, 0x9c, 0x1d, 0xb5, 0x3d, 0x49, 0x00, + 0x0c, 0x06, 0xd4, 0x50, 0xf9, 0x54, 0x45, 0xb2, 0x5b, 0x43, 0xdb, 0x6d, 0xcf, 0x1a, 0xe9, 0x7a, 0x7a, 0xcf, 0xfc, + 0x8a, 0x4e, 0x4d, 0x0b, 0x07, 0x63, 0x28, 0xd8, 0xe7, 0x08, 0x95, 0xdf, 0xa6, 0x72, 0x93, 0x2e, 0xbb, 0xa0, 0x42, + 0x89, 0x16, 0xf1, 0xd9, 0x0c, 0xf9, 0xa1, 0x16, 0xfd, 0xd9, 0x03, 0xb4, 0x3b, 0x8a, 0xf5, 0xf6, 0xe7, 0x6b, 0x2e, + 0x8e, 0x4c, 0x3d, 0xe2, 0xaf, 0x08, 0x45, 0x03, 0xff, 0x09, 0xb6, 0xeb, 0x2d, 0xc6, 0x1b, 0x88, 0x94, 0xac, 0x3e, + 0xf1, 0x9f, 0x0e, 0x0e, 0x2b, 0xd5, 0x00, 0x4d, 0x3f, 0x3b, 0x53, 0xae, 0xaf, 0x1c, 0x33, 0x5f, 0x55, 0x6e, 0x8d, + 0xaf, 0x05, 0x7a, 0x10, 0x34, 0xc9, 0xf4, 0x66, 0xcb, 0x62, 0x12, 0xa6, 0xee, 0xe8, 0x1c, 0x5d, 0x12, 0x86, 0xdb, + 0x6f, 0x1c, 0x33, 0xc4, 0x1c, 0xda, 0x82, 0x2d, 0x3b, 0x59, 0xfe, 0xb1, 0xa4, 0x59, 0x41, 0x86, 0xd0, 0xef, 0xae, + 0xfb, 0xda, 0x6d, 0x11, 0xb8, 0xca, 0xe9, 0x6e, 0xff, 0xf7, 0xa9, 0xd9, 0x70, 0x30, 0xfc, 0x53, 0xe2, 0xd7, 0xa2, + 0x4e, 0xc7, 0x91, 0xd9, 0x07, 0x06, 0xaa, 0xdd, 0xb0, 0x59, 0x28, 0x1d, 0x00, 0x66, 0xc5, 0x54, 0xc2, 0xfc, 0x06, + 0xda, 0x05, 0x90, 0x52, 0x1d, 0x37, 0x66, 0xee, 0xf0, 0xb2, 0x55, 0x8a, 0x5d, 0xd2, 0x38, 0x86, 0x94, 0x9b, 0xfc, + 0x10, 0x4c, 0xa1, 0xb9, 0x64, 0x3e, 0x44, 0xb8, 0x5f, 0xb0, 0x0c, 0xec, 0xe0, 0xc9, 0xe5, 0x62, 0x75, 0x3f, 0x09, + 0xd5, 0xf5, 0xd9, 0x26, 0xba, 0x9e, 0xd2, 0xf4, 0xb9, 0x48, 0x0a, 0xbc, 0xa2, 0xd6, 0x7c, 0x36, 0x11, 0x7d, 0x26, + 0x81, 0x89, 0xcf, 0xa4, 0xad, 0x73, 0x0e, 0xee, 0xcc, 0x06, 0xa9, 0xdb, 0xb1, 0xfd, 0xfb, 0x09, 0x7f, 0x90, 0x42, + 0x37, 0x2f, 0xe1, 0x9c, 0x0f, 0x6f, 0xcf, 0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8, 0xae +}; +static CONST UINT8 EncAssoc009[] = { 0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e, 0xef }; +static CONST UINT8 EncNonce009[] = { 0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78 }; +static CONST UINT8 EncKey009[] = { 0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5, 0xcc, 0x1a, 0xd7, + 0xc1, 0x57, 0x72, 0xea, 0x86, 0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, + 0x9b, 0xb2, 0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b }; + +static CONST UINT8 EncInput010[] = { + 0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf, 0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c, 0xef, 0x0a, 0xa9, + 0x48, 0x5f, 0x5f, 0x37, 0x22, 0x3a, 0xde, 0xe3, 0xae, 0xbe, 0xad, 0x07, 0xcc, 0xb1, 0xf6, 0xf5, 0xf9, 0x56, 0xdd, + 0xe7, 0x16, 0x1e, 0x7f, 0xdf, 0x7a, 0x9e, 0x75, 0xb7, 0xc7, 0xbe, 0xbe, 0x8a, 0x36, 0x04, 0xc0, 0x10, 0xf4, 0x95, + 0x20, 0x03, 0xec, 0xdc, 0x05, 0xa1, 0x7d, 0xc4, 0xa9, 0x2c, 0x82, 0xd0, 0xbc, 0x8b, 0xc5, 0xc7, 0x45, 0x50, 0xf6, + 0xa2, 0x1a, 0xb5, 0x46, 0x3b, 0x73, 0x02, 0xa6, 0x83, 0x4b, 0x73, 0x82, 0x58, 0x5e, 0x3b, 0x65, 0x2f, 0x0e, 0xfd, + 0x2b, 0x59, 0x16, 0xce, 0xa1, 0x60, 0x9c, 0xe8, 0x3a, 0x99, 0xed, 0x8d, 0x5a, 0xcf, 0xf6, 0x83, 0xaf, 0xba, 0xd7, + 0x73, 0x73, 0x40, 0x97, 0x3d, 0xca, 0xef, 0x07, 0x57, 0xe6, 0xd9, 0x70, 0x0e, 0x95, 0xae, 0xa6, 0x8d, 0x04, 0xcc, + 0xee, 0xf7, 0x09, 0x31, 0x77, 0x12, 0xa3, 0x23, 0x97, 0x62, 0xb3, 0x7b, 0x32, 0xfb, 0x80, 0x14, 0x48, 0x81, 0xc3, + 0xe5, 0xea, 0x91, 0x39, 0x52, 0x81, 0xa2, 0x4f, 0xe4, 0xb3, 0x09, 0xff, 0xde, 0x5e, 0xe9, 0x58, 0x84, 0x6e, 0xf9, + 0x3d, 0xdf, 0x25, 0xea, 0xad, 0xae, 0xe6, 0x9a, 0xd1, 0x89, 0x55, 0xd3, 0xde, 0x6c, 0x52, 0xdb, 0x70, 0xfe, 0x37, + 0xce, 0x44, 0x0a, 0xa8, 0x25, 0x5f, 0x92, 0xc1, 0x33, 0x4a, 0x4f, 0x9b, 0x62, 0x35, 0xff, 0xce, 0xc0, 0xa9, 0x60, + 0xce, 0x52, 0x00, 0x97, 0x51, 0x35, 0x26, 0x2e, 0xb9, 0x36, 0xa9, 0x87, 0x6e, 0x1e, 0xcc, 0x91, 0x78, 0x53, 0x98, + 0x86, 0x5b, 0x9c, 0x74, 0x7d, 0x88, 0x33, 0xe1, 0xdf, 0x37, 0x69, 0x2b, 0xbb, 0xf1, 0x4d, 0xf4, 0xd1, 0xf1, 0x39, + 0x93, 0x17, 0x51, 0x19, 0xe3, 0x19, 0x1e, 0x76, 0x37, 0x25, 0xfb, 0x09, 0x27, 0x6a, 0xab, 0x67, 0x6f, 0x14, 0x12, + 0x64, 0xe7, 0xc4, 0x07, 0xdf, 0x4d, 0x17, 0xbb, 0x6d, 0xe0, 0xe9, 0xb9, 0xab, 0xca, 0x10, 0x68, 0xaf, 0x7e, 0xb7, + 0x33, 0x54, 0x73, 0x07, 0x6e, 0xf7, 0x81, 0x97, 0x9c, 0x05, 0x6f, 0x84, 0x5f, 0xd2, 0x42, 0xfb, 0x38, 0xcf, 0xd1, + 0x2f, 0x14, 0x30, 0x88, 0x98, 0x4d, 0x5a, 0xa9, 0x76, 0xd5, 0x4f, 0x3e, 0x70, 0x6c, 0x85, 0x76, 0xd7, 0x01, 0xa0, + 0x1a, 0xc8, 0x4e, 0xaa, 0xac, 0x78, 0xfe, 0x46, 0xde, 0x6a, 0x05, 0x46, 0xa7, 0x43, 0x0c, 0xb9, 0xde, 0xb9, 0x68, + 0xfb, 0xce, 0x42, 0x99, 0x07, 0x4d, 0x0b, 0x3b, 0x5a, 0x30, 0x35, 0xa8, 0xf9, 0x3a, 0x73, 0xef, 0x0f, 0xdb, 0x1e, + 0x16, 0x42, 0xc4, 0xba, 0xae, 0x58, 0xaa, 0xf8, 0xe5, 0x75, 0x2f, 0x1b, 0x15, 0x5c, 0xfd, 0x0a, 0x97, 0xd0, 0xe4, + 0x37, 0x83, 0x61, 0x5f, 0x43, 0xa6, 0xc7, 0x3f, 0x38, 0x59, 0xe6, 0xeb, 0xa3, 0x90, 0xc3, 0xaa, 0xaa, 0x5a, 0xd3, + 0x34, 0xd4, 0x17, 0xc8, 0x65, 0x3e, 0x57, 0xbc, 0x5e, 0xdd, 0x9e, 0xb7, 0xf0, 0x2e, 0x5b, 0xb2, 0x1f, 0x8a, 0x08, + 0x0d, 0x45, 0x91, 0x0b, 0x29, 0x53, 0x4f, 0x4c, 0x5a, 0x73, 0x56, 0xfe, 0xaf, 0x41, 0x01, 0x39, 0x0a, 0x24, 0x3c, + 0x7e, 0xbe, 0x4e, 0x53, 0xf3, 0xeb, 0x06, 0x66, 0x51, 0x28, 0x1d, 0xbd, 0x41, 0x0a, 0x01, 0xab, 0x16, 0x47, 0x27, + 0x47, 0x47, 0xf7, 0xcb, 0x46, 0x0a, 0x70, 0x9e, 0x01, 0x9c, 0x09, 0xe1, 0x2a, 0x00, 0x1a, 0xd8, 0xd4, 0x79, 0x9d, + 0x80, 0x15, 0x8e, 0x53, 0x2a, 0x65, 0x83, 0x78, 0x3e, 0x03, 0x00, 0x07, 0x12, 0x1f, 0x33, 0x3e, 0x7b, 0x13, 0x37, + 0xf1, 0xc3, 0xef, 0xb7, 0xc1, 0x20, 0x3c, 0x3e, 0x67, 0x66, 0x5d, 0x88, 0xa7, 0x7d, 0x33, 0x50, 0x77, 0xb0, 0x28, + 0x8e, 0xe7, 0x2c, 0x2e, 0x7a, 0xf4, 0x3c, 0x8d, 0x74, 0x83, 0xaf, 0x8e, 0x87, 0x0f, 0xe4, 0x50, 0xff, 0x84, 0x5c, + 0x47, 0x0c, 0x6a, 0x49, 0xbf, 0x42, 0x86, 0x77, 0x15, 0x48, 0xa5, 0x90, 0x5d, 0x93, 0xd6, 0x2a, 0x11, 0xd5, 0xd5, + 0x11, 0xaa, 0xce, 0xe7, 0x6f, 0xa5, 0xb0, 0x09, 0x2c, 0x8d, 0xd3, 0x92, 0xf0, 0x5a, 0x2a, 0xda, 0x5b, 0x1e, 0xd5, + 0x9a, 0xc4, 0xc4, 0xf3, 0x49, 0x74, 0x41, 0xca, 0xe8, 0xc1, 0xf8, 0x44, 0xd6, 0x3c, 0xae, 0x6c, 0x1d, 0x9a, 0x30, + 0x04, 0x4d, 0x27, 0x0e, 0xb1, 0x5f, 0x59, 0xa2, 0x24, 0xe8, 0xe1, 0x98, 0xc5, 0x6a, 0x4c, 0xfe, 0x41, 0xd2, 0x27, + 0x42, 0x52, 0xe1, 0xe9, 0x7d, 0x62, 0xe4, 0x88, 0x0f, 0xad, 0xb2, 0x70, 0xcb, 0x9d, 0x4c, 0x27, 0x2e, 0x76, 0x1e, + 0x1a, 0x63, 0x65, 0xf5, 0x3b, 0xf8, 0x57, 0x69, 0xeb, 0x5b, 0x38, 0x26, 0x39, 0x33, 0x25, 0x45, 0x3e, 0x91, 0xb8, + 0xd8, 0xc7, 0xd5, 0x42, 0xc0, 0x22, 0x31, 0x74, 0xf4, 0xbc, 0x0c, 0x23, 0xf1, 0xca, 0xc1, 0x8d, 0xd7, 0xbe, 0xc9, + 0x62, 0xe4, 0x08, 0x1a, 0xcf, 0x36, 0xd5, 0xfe, 0x55, 0x21, 0x59, 0x91, 0x87, 0x87, 0xdf, 0x06, 0xdb, 0xdf, 0x96, + 0x45, 0x58, 0xda, 0x05, 0xcd, 0x50, 0x4d, 0xd2, 0x7d, 0x05, 0x18, 0x73, 0x6a, 0x8d, 0x11, 0x85, 0xa6, 0x88, 0xe8, + 0xda, 0xe6, 0x30, 0x33, 0xa4, 0x89, 0x31, 0x75, 0xbe, 0x69, 0x43, 0x84, 0x43, 0x50, 0x87, 0xdd, 0x71, 0x36, 0x83, + 0xc3, 0x78, 0x74, 0x24, 0x0a, 0xed, 0x7b, 0xdb, 0xa4, 0x24, 0x0b, 0xb9, 0x7e, 0x5d, 0xff, 0xde, 0xb1, 0xef, 0x61, + 0x5a, 0x45, 0x33, 0xf6, 0x17, 0x07, 0x08, 0x98, 0x83, 0x92, 0x0f, 0x23, 0x6d, 0xe6, 0xaa, 0x17, 0x54, 0xad, 0x6a, + 0xc8, 0xdb, 0x26, 0xbe, 0xb8, 0xb6, 0x08, 0xfa, 0x68, 0xf1, 0xd7, 0x79, 0x6f, 0x18, 0xb4, 0x9e, 0x2d, 0x3f, 0x1b, + 0x64, 0xaf, 0x8d, 0x06, 0x0e, 0x49, 0x28, 0xe0, 0x5d, 0x45, 0x68, 0x13, 0x87, 0xfa, 0xde, 0x40, 0x7b, 0xd2, 0xc3, + 0x94, 0xd5, 0xe1, 0xd9, 0xc2, 0xaf, 0x55, 0x89, 0xeb, 0xb4, 0x12, 0x59, 0xa8, 0xd4, 0xc5, 0x29, 0x66, 0x38, 0xe6, + 0xac, 0x22, 0x22, 0xd9, 0x64, 0x9b, 0x34, 0x0a, 0x32, 0x9f, 0xc2, 0xbf, 0x17, 0x6c, 0x3f, 0x71, 0x7a, 0x38, 0x6b, + 0x98, 0xfb, 0x49, 0x36, 0x89, 0xc9, 0xe2, 0xd6, 0xc7, 0x5d, 0xd0, 0x69, 0x5f, 0x23, 0x35, 0xc9, 0x30, 0xe2, 0xfd, + 0x44, 0x58, 0x39, 0xd7, 0x97, 0xfb, 0x5c, 0x00, 0xd5, 0x4f, 0x7a, 0x1a, 0x95, 0x8b, 0x62, 0x4b, 0xce, 0xe5, 0x91, + 0x21, 0x7b, 0x30, 0x00, 0xd6, 0xdd, 0x6d, 0x02, 0x86, 0x49, 0x0f, 0x3c, 0x1a, 0x27, 0x3c, 0xd3, 0x0e, 0x71, 0xf2, + 0xff, 0xf5, 0x2f, 0x87, 0xac, 0x67, 0x59, 0x81, 0xa3, 0xf7, 0xf8, 0xd6, 0x11, 0x0c, 0x84, 0xa9, 0x03, 0xee, 0x2a, + 0xc4, 0xf3, 0x22, 0xab, 0x7c, 0xe2, 0x25, 0xf5, 0x67, 0xa3, 0xe4, 0x11, 0xe0, 0x59, 0xb3, 0xca, 0x87, 0xa0, 0xae, + 0xc9, 0xa6, 0x62, 0x1b, 0x6e, 0x4d, 0x02, 0x6b, 0x07, 0x9d, 0xfd, 0xd0, 0x92, 0x06, 0xe1, 0xb2, 0x9a, 0x4a, 0x1f, + 0x1f, 0x13, 0x49, 0x99, 0x97, 0x08, 0xde, 0x7f, 0x98, 0xaf, 0x51, 0x98, 0xee, 0x2c, 0xcb, 0xf0, 0x0b, 0xc6, 0xb6, + 0xb7, 0x2d, 0x9a, 0xb1, 0xac, 0xa6, 0xe3, 0x15, 0x77, 0x9d, 0x6b, 0x1a, 0xe4, 0xfc, 0x8b, 0xf2, 0x17, 0x59, 0x08, + 0x04, 0x58, 0x81, 0x9d, 0x1b, 0x1b, 0x69, 0x55, 0xc2, 0xb4, 0x3c, 0x1f, 0x50, 0xf1, 0x7f, 0x77, 0x90, 0x4c, 0x66, + 0x40, 0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c, 0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f +}; +static CONST UINT8 EncOutput010[] = { + 0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b, 0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74, 0xa6, 0xdd, 0xbd, + 0x95, 0xeb, 0xf9, 0xa4, 0xf1, 0x59, 0x93, 0x91, 0x70, 0xd9, 0xfe, 0x9a, 0xcd, 0x53, 0x1f, 0x3a, 0xab, 0xa6, 0x7c, + 0x9f, 0xa6, 0x9e, 0xbd, 0x99, 0xd9, 0xb5, 0x97, 0x44, 0xd5, 0x14, 0x48, 0x4d, 0x9d, 0xc0, 0xd0, 0x05, 0x96, 0xeb, + 0x4c, 0x78, 0x55, 0x09, 0x08, 0x01, 0x02, 0x30, 0x90, 0x7b, 0x96, 0x7a, 0x7b, 0x5f, 0x30, 0x41, 0x24, 0xce, 0x68, + 0x61, 0x49, 0x86, 0x57, 0x82, 0xdd, 0x53, 0x1c, 0x51, 0x28, 0x2b, 0x53, 0x6e, 0x2d, 0xc2, 0x20, 0x4c, 0xdd, 0x8f, + 0x65, 0x10, 0x20, 0x50, 0xdd, 0x9d, 0x50, 0xe5, 0x71, 0x40, 0x53, 0x69, 0xfc, 0x77, 0x48, 0x11, 0xb9, 0xde, 0xa4, + 0x8d, 0x58, 0xe4, 0xa6, 0x1a, 0x18, 0x47, 0x81, 0x7e, 0xfc, 0xdd, 0xf6, 0xef, 0xce, 0x2f, 0x43, 0x68, 0xd6, 0x06, + 0xe2, 0x74, 0x6a, 0xad, 0x90, 0xf5, 0x37, 0xf3, 0x3d, 0x82, 0x69, 0x40, 0xe9, 0x6b, 0xa7, 0x3d, 0xa8, 0x1e, 0xd2, + 0x02, 0x7c, 0xb7, 0x9b, 0xe4, 0xda, 0x8f, 0x95, 0x06, 0xc5, 0xdf, 0x73, 0xa3, 0x20, 0x9a, 0x49, 0xde, 0x9c, 0xbc, + 0xee, 0x14, 0x3f, 0x81, 0x5e, 0xf8, 0x3b, 0x59, 0x3c, 0xe1, 0x68, 0x12, 0x5a, 0x3a, 0x76, 0x3a, 0x3f, 0xf7, 0x87, + 0x33, 0x0a, 0x01, 0xb8, 0xd4, 0xed, 0xb6, 0xbe, 0x94, 0x5e, 0x70, 0x40, 0x56, 0x67, 0x1f, 0x50, 0x44, 0x19, 0xce, + 0x82, 0x70, 0x10, 0x87, 0x13, 0x20, 0x0b, 0x4c, 0x5a, 0xb6, 0xf6, 0xa7, 0xae, 0x81, 0x75, 0x01, 0x81, 0xe6, 0x4b, + 0x57, 0x7c, 0xdd, 0x6d, 0xf8, 0x1c, 0x29, 0x32, 0xf7, 0xda, 0x3c, 0x2d, 0xf8, 0x9b, 0x25, 0x6e, 0x00, 0xb4, 0xf7, + 0x2f, 0xf7, 0x04, 0xf7, 0xa1, 0x56, 0xac, 0x4f, 0x1a, 0x64, 0xb8, 0x47, 0x55, 0x18, 0x7b, 0x07, 0x4d, 0xbd, 0x47, + 0x24, 0x80, 0x5d, 0xa2, 0x70, 0xc5, 0xdd, 0x8e, 0x82, 0xd4, 0xeb, 0xec, 0xb2, 0x0c, 0x39, 0xd2, 0x97, 0xc1, 0xcb, + 0xeb, 0xf4, 0x77, 0x59, 0xb4, 0x87, 0xef, 0xcb, 0x43, 0x2d, 0x46, 0x54, 0xd1, 0xa7, 0xd7, 0x15, 0x99, 0x0a, 0x43, + 0xa1, 0xe0, 0x99, 0x33, 0x71, 0xc1, 0xed, 0xfe, 0x72, 0x46, 0x33, 0x8e, 0x91, 0x08, 0x9f, 0xc8, 0x2e, 0xca, 0xfa, + 0xdc, 0x59, 0xd5, 0xc3, 0x76, 0x84, 0x9f, 0xa3, 0x37, 0x68, 0xc3, 0xf0, 0x47, 0x2c, 0x68, 0xdb, 0x5e, 0xc3, 0x49, + 0x4c, 0xe8, 0x92, 0x85, 0xe2, 0x23, 0xd3, 0x3f, 0xad, 0x32, 0xe5, 0x2b, 0x82, 0xd7, 0x8f, 0x99, 0x0a, 0x59, 0x5c, + 0x45, 0xd9, 0xb4, 0x51, 0x52, 0xc2, 0xae, 0xbf, 0x80, 0xcf, 0xc9, 0xc9, 0x51, 0x24, 0x2a, 0x3b, 0x3a, 0x4d, 0xae, + 0xeb, 0xbd, 0x22, 0xc3, 0x0e, 0x0f, 0x59, 0x25, 0x92, 0x17, 0xe9, 0x74, 0xc7, 0x8b, 0x70, 0x70, 0x36, 0x55, 0x95, + 0x75, 0x4b, 0xad, 0x61, 0x2b, 0x09, 0xbc, 0x82, 0xf2, 0x6e, 0x94, 0x43, 0xae, 0xc3, 0xd5, 0xcd, 0x8e, 0xfe, 0x5b, + 0x9a, 0x88, 0x43, 0x01, 0x75, 0xb2, 0x23, 0x09, 0xf7, 0x89, 0x83, 0xe7, 0xfa, 0xf9, 0xb4, 0x9b, 0xf8, 0xef, 0xbd, + 0x1c, 0x92, 0xc1, 0xda, 0x7e, 0xfe, 0x05, 0xba, 0x5a, 0xcd, 0x07, 0x6a, 0x78, 0x9e, 0x5d, 0xfb, 0x11, 0x2f, 0x79, + 0x38, 0xb6, 0xc2, 0x5b, 0x6b, 0x51, 0xb4, 0x71, 0xdd, 0xf7, 0x2a, 0xe4, 0xf4, 0x72, 0x76, 0xad, 0xc2, 0xdd, 0x64, + 0x5d, 0x79, 0xb6, 0xf5, 0x7a, 0x77, 0x20, 0x05, 0x3d, 0x30, 0x06, 0xd4, 0x4c, 0x0a, 0x2c, 0x98, 0x5a, 0xb9, 0xd4, + 0x98, 0xa9, 0x3f, 0xc6, 0x12, 0xea, 0x3b, 0x4b, 0xc5, 0x79, 0x64, 0x63, 0x6b, 0x09, 0x54, 0x3b, 0x14, 0x27, 0xba, + 0x99, 0x80, 0xc8, 0x72, 0xa8, 0x12, 0x90, 0x29, 0xba, 0x40, 0x54, 0x97, 0x2b, 0x7b, 0xfe, 0xeb, 0xcd, 0x01, 0x05, + 0x44, 0x72, 0xdb, 0x99, 0xe4, 0x61, 0xc9, 0x69, 0xd6, 0xb9, 0x28, 0xd1, 0x05, 0x3e, 0xf9, 0x0b, 0x49, 0x0a, 0x49, + 0xe9, 0x8d, 0x0e, 0xa7, 0x4a, 0x0f, 0xaf, 0x32, 0xd0, 0xe0, 0xb2, 0x3a, 0x55, 0x58, 0xfe, 0x5c, 0x28, 0x70, 0x51, + 0x23, 0xb0, 0x7b, 0x6a, 0x5f, 0x1e, 0xb8, 0x17, 0xd7, 0x94, 0x15, 0x8f, 0xee, 0x20, 0xc7, 0x42, 0x25, 0x3e, 0x9a, + 0x14, 0xd7, 0x60, 0x72, 0x39, 0x47, 0x48, 0xa9, 0xfe, 0xdd, 0x47, 0x0a, 0xb1, 0xe6, 0x60, 0x28, 0x8c, 0x11, 0x68, + 0xe1, 0xff, 0xd7, 0xce, 0xc8, 0xbe, 0xb3, 0xfe, 0x27, 0x30, 0x09, 0x70, 0xd7, 0xfa, 0x02, 0x33, 0x3a, 0x61, 0x2e, + 0xc7, 0xff, 0xa4, 0x2a, 0xa8, 0x6e, 0xb4, 0x79, 0x35, 0x6d, 0x4c, 0x1e, 0x38, 0xf8, 0xee, 0xd4, 0x84, 0x4e, 0x6e, + 0x28, 0xa7, 0xce, 0xc8, 0xc1, 0xcf, 0x80, 0x05, 0xf3, 0x04, 0xef, 0xc8, 0x18, 0x28, 0x2e, 0x8d, 0x5e, 0x0c, 0xdf, + 0xb8, 0x5f, 0x96, 0xe8, 0xc6, 0x9c, 0x2f, 0xe5, 0xa6, 0x44, 0xd7, 0xe7, 0x99, 0x44, 0x0c, 0xec, 0xd7, 0x05, 0x60, + 0x97, 0xbb, 0x74, 0x77, 0x58, 0xd5, 0xbb, 0x48, 0xde, 0x5a, 0xb2, 0x54, 0x7f, 0x0e, 0x46, 0x70, 0x6a, 0x6f, 0x78, + 0xa5, 0x08, 0x89, 0x05, 0x4e, 0x7e, 0xa0, 0x69, 0xb4, 0x40, 0x60, 0x55, 0x77, 0x75, 0x9b, 0x19, 0xf2, 0xd5, 0x13, + 0x80, 0x77, 0xf9, 0x4b, 0x3f, 0x1e, 0xee, 0xe6, 0x76, 0x84, 0x7b, 0x8c, 0xe5, 0x27, 0xa8, 0x0a, 0x91, 0x01, 0x68, + 0x71, 0x8a, 0x3f, 0x06, 0xab, 0xf6, 0xa9, 0xa5, 0xe6, 0x72, 0x92, 0xe4, 0x67, 0xe2, 0xa2, 0x46, 0x35, 0x84, 0x55, + 0x7d, 0xca, 0xa8, 0x85, 0xd0, 0xf1, 0x3f, 0xbe, 0xd7, 0x34, 0x64, 0xfc, 0xae, 0xe3, 0xe4, 0x04, 0x9f, 0x66, 0x02, + 0xb9, 0x88, 0x10, 0xd9, 0xc4, 0x4c, 0x31, 0x43, 0x7a, 0x93, 0xe2, 0x9b, 0x56, 0x43, 0x84, 0xdc, 0xdc, 0xde, 0x1d, + 0xa4, 0x02, 0x0e, 0xc2, 0xef, 0xc3, 0xf8, 0x78, 0xd1, 0xb2, 0x6b, 0x63, 0x18, 0xc9, 0xa9, 0xe5, 0x72, 0xd8, 0xf3, + 0xb9, 0xd1, 0x8a, 0xc7, 0x1a, 0x02, 0x27, 0x20, 0x77, 0x10, 0xe5, 0xc8, 0xd4, 0x4a, 0x47, 0xe5, 0xdf, 0x5f, 0x01, + 0xaa, 0xb0, 0xd4, 0x10, 0xbb, 0x69, 0xe3, 0x36, 0xc8, 0xe1, 0x3d, 0x43, 0xfb, 0x86, 0xcd, 0xcc, 0xbf, 0xf4, 0x88, + 0xe0, 0x20, 0xca, 0xb7, 0x1b, 0xf1, 0x2f, 0x5c, 0xee, 0xd4, 0xd3, 0xa3, 0xcc, 0xa4, 0x1e, 0x1c, 0x47, 0xfb, 0xbf, + 0xfc, 0xa2, 0x41, 0x55, 0x9d, 0xf6, 0x5a, 0x5e, 0x65, 0x32, 0x34, 0x7b, 0x52, 0x8d, 0xd5, 0xd0, 0x20, 0x60, 0x03, + 0xab, 0x3f, 0x8c, 0xd4, 0x21, 0xea, 0x2a, 0xd9, 0xc4, 0xd0, 0xd3, 0x65, 0xd8, 0x7a, 0x13, 0x28, 0x62, 0x32, 0x4b, + 0x2c, 0x87, 0x93, 0xa8, 0xb4, 0x52, 0x45, 0x09, 0x44, 0xec, 0xec, 0xc3, 0x17, 0xdb, 0x9a, 0x4d, 0x5c, 0xa9, 0x11, + 0xd4, 0x7d, 0xaf, 0x9e, 0xf1, 0x2d, 0xb2, 0x66, 0xc5, 0x1d, 0xed, 0xb7, 0xcd, 0x0b, 0x25, 0x5e, 0x30, 0x47, 0x3f, + 0x40, 0xf4, 0xa1, 0xa0, 0x00, 0x94, 0x10, 0xc5, 0x6a, 0x63, 0x1a, 0xd5, 0x88, 0x92, 0x8e, 0x82, 0x39, 0x87, 0x3c, + 0x78, 0x65, 0x58, 0x42, 0x75, 0x5b, 0xdd, 0x77, 0x3e, 0x09, 0x4e, 0x76, 0x5b, 0xe6, 0x0e, 0x4d, 0x38, 0xb2, 0xc0, + 0xb8, 0x95, 0x01, 0x7a, 0x10, 0xe0, 0xfb, 0x07, 0xf2, 0xab, 0x2d, 0x8c, 0x32, 0xed, 0x2b, 0xc0, 0x46, 0xc2, 0xf5, + 0x38, 0x83, 0xf0, 0x17, 0xec, 0xc1, 0x20, 0x6a, 0x9a, 0x0b, 0x00, 0xa0, 0x98, 0x22, 0x50, 0x23, 0xd5, 0x80, 0x6b, + 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9, 0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0 +}; +static CONST UINT8 EncAssoc010[] = { 0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27, + 0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2 }; +static CONST UINT8 EncNonce010[] = { 0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30 }; +static CONST UINT8 EncKey010[] = { 0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44, 0x34, 0xda, 0x7f, + 0x57, 0x03, 0x39, 0x0c, 0xaf, 0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, + 0x8e, 0x74, 0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7 }; + +static CONST UINT8 EncInput011[] = { + 0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b, 0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b, 0x45, 0x60, 0xbe, + 0x9a, 0x31, 0x9f, 0xff, 0x5d, 0x66, 0x34, 0xb4, 0xdc, 0xfb, 0x9d, 0x8e, 0xee, 0x6a, 0x33, 0xa4, 0x07, 0x3c, 0xf9, + 0x4c, 0x30, 0xa1, 0x24, 0x52, 0xf9, 0x50, 0x46, 0x88, 0x20, 0x02, 0x32, 0x3a, 0x0e, 0x99, 0x63, 0xaf, 0x1f, 0x15, + 0x28, 0x2a, 0x05, 0xff, 0x57, 0x59, 0x5e, 0x18, 0xa1, 0x1f, 0xd0, 0x92, 0x5c, 0x88, 0x66, 0x1b, 0x00, 0x64, 0xa5, + 0x93, 0x8d, 0x06, 0x46, 0xb0, 0x64, 0x8b, 0x8b, 0xef, 0x99, 0x05, 0x35, 0x85, 0xb3, 0xf3, 0x33, 0xbb, 0xec, 0x66, + 0xb6, 0x3d, 0x57, 0x42, 0xe3, 0xb4, 0xc6, 0xaa, 0xb0, 0x41, 0x2a, 0xb9, 0x59, 0xa9, 0xf6, 0x3e, 0x15, 0x26, 0x12, + 0x03, 0x21, 0x4c, 0x74, 0x43, 0x13, 0x2a, 0x03, 0x27, 0x09, 0xb4, 0xfb, 0xe7, 0xb7, 0x40, 0xff, 0x5e, 0xce, 0x48, + 0x9a, 0x60, 0xe3, 0x8b, 0x80, 0x8c, 0x38, 0x2d, 0xcb, 0x93, 0x37, 0x74, 0x05, 0x52, 0x6f, 0x73, 0x3e, 0xc3, 0xbc, + 0xca, 0x72, 0x0a, 0xeb, 0xf1, 0x3b, 0xa0, 0x95, 0xdc, 0x8a, 0xc4, 0xa9, 0xdc, 0xca, 0x44, 0xd8, 0x08, 0x63, 0x6a, + 0x36, 0xd3, 0x3c, 0xb8, 0xac, 0x46, 0x7d, 0xfd, 0xaa, 0xeb, 0x3e, 0x0f, 0x45, 0x8f, 0x49, 0xda, 0x2b, 0xf2, 0x12, + 0xbd, 0xaf, 0x67, 0x8a, 0x63, 0x48, 0x4b, 0x55, 0x5f, 0x6d, 0x8c, 0xb9, 0x76, 0x34, 0x84, 0xae, 0xc2, 0xfc, 0x52, + 0x64, 0x82, 0xf7, 0xb0, 0x06, 0xf0, 0x45, 0x73, 0x12, 0x50, 0x30, 0x72, 0xea, 0x78, 0x9a, 0xa8, 0xaf, 0xb5, 0xe3, + 0xbb, 0x77, 0x52, 0xec, 0x59, 0x84, 0xbf, 0x6b, 0x8f, 0xce, 0x86, 0x5e, 0x1f, 0x23, 0xe9, 0xfb, 0x08, 0x86, 0xf7, + 0x10, 0xb9, 0xf2, 0x44, 0x96, 0x44, 0x63, 0xa9, 0xa8, 0x78, 0x00, 0x23, 0xd6, 0xc7, 0xe7, 0x6e, 0x66, 0x4f, 0xcc, + 0xee, 0x15, 0xb3, 0xbd, 0x1d, 0xa0, 0xe5, 0x9c, 0x1b, 0x24, 0x2c, 0x4d, 0x3c, 0x62, 0x35, 0x9c, 0x88, 0x59, 0x09, + 0xdd, 0x82, 0x1b, 0xcf, 0x0a, 0x83, 0x6b, 0x3f, 0xae, 0x03, 0xc4, 0xb4, 0xdd, 0x7e, 0x5b, 0x28, 0x76, 0x25, 0x96, + 0xd9, 0xc9, 0x9d, 0x5f, 0x86, 0xfa, 0xf6, 0xd7, 0xd2, 0xe6, 0x76, 0x1d, 0x0f, 0xa1, 0xdc, 0x74, 0x05, 0x1b, 0x1d, + 0xe0, 0xcd, 0x16, 0xb0, 0xa8, 0x8a, 0x34, 0x7b, 0x15, 0x11, 0x77, 0xe5, 0x7b, 0x7e, 0x20, 0xf7, 0xda, 0x38, 0xda, + 0xce, 0x70, 0xe9, 0xf5, 0x6c, 0xd9, 0xbe, 0x0c, 0x4c, 0x95, 0x4c, 0xc2, 0x9b, 0x34, 0x55, 0x55, 0xe1, 0xf3, 0x46, + 0x8e, 0x48, 0x74, 0x14, 0x4f, 0x9d, 0xc9, 0xf5, 0xe8, 0x1a, 0xf0, 0x11, 0x4a, 0xc1, 0x8d, 0xe0, 0x93, 0xa0, 0xbe, + 0x09, 0x1c, 0x2b, 0x4e, 0x0f, 0xb2, 0x87, 0x8b, 0x84, 0xfe, 0x92, 0x32, 0x14, 0xd7, 0x93, 0xdf, 0xe7, 0x44, 0xbc, + 0xc5, 0xae, 0x53, 0x69, 0xd8, 0xb3, 0x79, 0x37, 0x80, 0xe3, 0x17, 0x5c, 0xec, 0x53, 0x00, 0x9a, 0xe3, 0x8e, 0xdc, + 0x38, 0xb8, 0x66, 0xf0, 0xd3, 0xad, 0x1d, 0x02, 0x96, 0x86, 0x3e, 0x9d, 0x3b, 0x5d, 0xa5, 0x7f, 0x21, 0x10, 0xf1, + 0x1f, 0x13, 0x20, 0xf9, 0x57, 0x87, 0x20, 0xf5, 0x5f, 0xf1, 0x17, 0x48, 0x0a, 0x51, 0x5a, 0xcd, 0x19, 0x03, 0xa6, + 0x5a, 0xd1, 0x12, 0x97, 0xe9, 0x48, 0xe2, 0x1d, 0x83, 0x75, 0x50, 0xd9, 0x75, 0x7d, 0x6a, 0x82, 0xa1, 0xf9, 0x4e, + 0x54, 0x87, 0x89, 0xc9, 0x0c, 0xb7, 0x5b, 0x6a, 0x91, 0xc1, 0x9c, 0xb2, 0xa9, 0xdc, 0x9a, 0xa4, 0x49, 0x0a, 0x6d, + 0x0d, 0xbb, 0xde, 0x86, 0x44, 0xdd, 0x5d, 0x89, 0x2b, 0x96, 0x0f, 0x23, 0x95, 0xad, 0xcc, 0xa2, 0xb3, 0xb9, 0x7e, + 0x74, 0x38, 0xba, 0x9f, 0x73, 0xae, 0x5f, 0xf8, 0x68, 0xa2, 0xe0, 0xa9, 0xce, 0xbd, 0x40, 0xd4, 0x4c, 0x6b, 0xd2, + 0x56, 0x62, 0xb0, 0xcc, 0x63, 0x7e, 0x5b, 0xd3, 0xae, 0xd1, 0x75, 0xce, 0xbb, 0xb4, 0x5b, 0xa8, 0xf8, 0xb4, 0xac, + 0x71, 0x75, 0xaa, 0xc9, 0x9f, 0xbb, 0x6c, 0xad, 0x0f, 0x55, 0x5d, 0xe8, 0x85, 0x7d, 0xf9, 0x21, 0x35, 0xea, 0x92, + 0x85, 0x2b, 0x00, 0xec, 0x84, 0x90, 0x0a, 0x63, 0x96, 0xe4, 0x6b, 0xa9, 0x77, 0xb8, 0x91, 0xf8, 0x46, 0x15, 0x72, + 0x63, 0x70, 0x01, 0x40, 0xa3, 0xa5, 0x76, 0x62, 0x2b, 0xbf, 0xf1, 0xe5, 0x8d, 0x9f, 0xa3, 0xfa, 0x9b, 0x03, 0xbe, + 0xfe, 0x65, 0x6f, 0xa2, 0x29, 0x0d, 0x54, 0xb4, 0x71, 0xce, 0xa9, 0xd6, 0x3d, 0x88, 0xf9, 0xaf, 0x6b, 0xa8, 0x9e, + 0xf4, 0x16, 0x96, 0x36, 0xb9, 0x00, 0xdc, 0x10, 0xab, 0xb5, 0x08, 0x31, 0x1f, 0x00, 0xb1, 0x3c, 0xd9, 0x38, 0x3e, + 0xc6, 0x04, 0xa7, 0x4e, 0xe8, 0xae, 0xed, 0x98, 0xc2, 0xf7, 0xb9, 0x00, 0x5f, 0x8c, 0x60, 0xd1, 0xe5, 0x15, 0xf7, + 0xae, 0x1e, 0x84, 0x88, 0xd1, 0xf6, 0xbc, 0x3a, 0x89, 0x35, 0x22, 0x83, 0x7c, 0xca, 0xf0, 0x33, 0x82, 0x4c, 0x79, + 0x3c, 0xfd, 0xb1, 0xae, 0x52, 0x62, 0x55, 0xd2, 0x41, 0x60, 0xc6, 0xbb, 0xfa, 0x0e, 0x59, 0xd6, 0xa8, 0xfe, 0x5d, + 0xed, 0x47, 0x3d, 0xe0, 0xea, 0x1f, 0x6e, 0x43, 0x51, 0xec, 0x10, 0x52, 0x56, 0x77, 0x42, 0x6b, 0x52, 0x87, 0xd8, + 0xec, 0xe0, 0xaa, 0x76, 0xa5, 0x84, 0x2a, 0x22, 0x24, 0xfd, 0x92, 0x40, 0x88, 0xd5, 0x85, 0x1c, 0x1f, 0x6b, 0x47, + 0xa0, 0xc4, 0xe4, 0xef, 0xf4, 0xea, 0xd7, 0x59, 0xac, 0x2a, 0x9e, 0x8c, 0xfa, 0x1f, 0x42, 0x08, 0xfe, 0x4f, 0x74, + 0xa0, 0x26, 0xf5, 0xb3, 0x84, 0xf6, 0x58, 0x5f, 0x26, 0x66, 0x3e, 0xd7, 0xe4, 0x22, 0x91, 0x13, 0xc8, 0xac, 0x25, + 0x96, 0x23, 0xd8, 0x09, 0xea, 0x45, 0x75, 0x23, 0xb8, 0x5f, 0xc2, 0x90, 0x8b, 0x09, 0xc4, 0xfc, 0x47, 0x6c, 0x6d, + 0x0a, 0xef, 0x69, 0xa4, 0x38, 0x19, 0xcf, 0x7d, 0xf9, 0x09, 0x73, 0x9b, 0x60, 0x5a, 0xf7, 0x37, 0xb5, 0xfe, 0x9f, + 0xe3, 0x2b, 0x4c, 0x0d, 0x6e, 0x19, 0xf1, 0xd6, 0xc0, 0x70, 0xf3, 0x9d, 0x22, 0x3c, 0xf9, 0x49, 0xce, 0x30, 0x8e, + 0x44, 0xb5, 0x76, 0x15, 0x8f, 0x52, 0xfd, 0xa5, 0x04, 0xb8, 0x55, 0x6a, 0x36, 0x59, 0x7c, 0xc4, 0x48, 0xb8, 0xd7, + 0xab, 0x05, 0x66, 0xe9, 0x5e, 0x21, 0x6f, 0x6b, 0x36, 0x29, 0xbb, 0xe9, 0xe3, 0xa2, 0x9a, 0xa8, 0xcd, 0x55, 0x25, + 0x11, 0xba, 0x5a, 0x58, 0xa0, 0xde, 0xae, 0x19, 0x2a, 0x48, 0x5a, 0xff, 0x36, 0xcd, 0x6d, 0x16, 0x7a, 0x73, 0x38, + 0x46, 0xe5, 0x47, 0x59, 0xc8, 0xa2, 0xf6, 0xe2, 0x6c, 0x83, 0xc5, 0x36, 0x2c, 0x83, 0x7d, 0xb4, 0x01, 0x05, 0x69, + 0xe7, 0xaf, 0x5c, 0xc4, 0x64, 0x82, 0x12, 0x21, 0xef, 0xf7, 0xd1, 0x7d, 0xb8, 0x8d, 0x8c, 0x98, 0x7c, 0x5f, 0x7d, + 0x92, 0x88, 0xb9, 0x94, 0x07, 0x9c, 0xd8, 0xe9, 0x9c, 0x17, 0x38, 0xe3, 0x57, 0x6c, 0xe0, 0xdc, 0xa5, 0x92, 0x42, + 0xb3, 0xbd, 0x50, 0xa2, 0x7e, 0xb5, 0xb1, 0x52, 0x72, 0x03, 0x97, 0xd8, 0xaa, 0x9a, 0x1e, 0x75, 0x41, 0x11, 0xa3, + 0x4f, 0xcc, 0xd4, 0xe3, 0x73, 0xad, 0x96, 0xdc, 0x47, 0x41, 0x9f, 0xb0, 0xbe, 0x79, 0x91, 0xf5, 0xb6, 0x18, 0xfe, + 0xc2, 0x83, 0x18, 0x7d, 0x73, 0xd9, 0x4f, 0x83, 0x84, 0x03, 0xb3, 0xf0, 0x77, 0x66, 0x3d, 0x83, 0x63, 0x2e, 0x2c, + 0xf9, 0xdd, 0xa6, 0x1f, 0x89, 0x82, 0xb8, 0x23, 0x42, 0xeb, 0xe2, 0xca, 0x70, 0x82, 0x61, 0x41, 0x0a, 0x6d, 0x5f, + 0x75, 0xc5, 0xe2, 0xc4, 0x91, 0x18, 0x44, 0x22, 0xfa, 0x34, 0x10, 0xf5, 0x20, 0xdc, 0xb7, 0xdd, 0x2a, 0x20, 0x77, + 0xf5, 0xf9, 0xce, 0xdb, 0xa0, 0x0a, 0x52, 0x2a, 0x4e, 0xdd, 0xcc, 0x97, 0xdf, 0x05, 0xe4, 0x5e, 0xb7, 0xaa, 0xf0, + 0xe2, 0x80, 0xff, 0xba, 0x1a, 0x0f, 0xac, 0xdf, 0x02, 0x32, 0xe6, 0xf7, 0xc7, 0x17, 0x13, 0xb7, 0xfc, 0x98, 0x48, + 0x8c, 0x0d, 0x82, 0xc9, 0x80, 0x7a, 0xe2, 0x0a, 0xc5, 0xb4, 0xde, 0x7c, 0x3c, 0x79, 0x81, 0x0e, 0x28, 0x65, 0x79, + 0x67, 0x82, 0x69, 0x44, 0x66, 0x09, 0xf7, 0x16, 0x1a, 0xf9, 0x7d, 0x80, 0xa1, 0x79, 0x14, 0xa9, 0xc8, 0x20, 0xfb, + 0xa2, 0x46, 0xbe, 0x08, 0x35, 0x17, 0x58, 0xc1, 0x1a, 0xda, 0x2a, 0x6b, 0x2e, 0x1e, 0xe6, 0x27, 0x55, 0x7b, 0x19, + 0xe2, 0xfb, 0x64, 0xfc, 0x5e, 0x15, 0x54, 0x3c, 0xe7, 0xc2, 0x11, 0x50, 0x30, 0xb8, 0x72, 0x03, 0x0b, 0x1a, 0x9f, + 0x86, 0x27, 0x11, 0x5c, 0x06, 0x2b, 0xbd, 0x75, 0x1a, 0x0a, 0xda, 0x01, 0xfa, 0x5c, 0x4a, 0xc1, 0x80, 0x3a, 0x6e, + 0x30, 0xc8, 0x2c, 0xeb, 0x56, 0xec, 0x89, 0xfa, 0x35, 0x7b, 0xb2, 0xf0, 0x97, 0x08, 0x86, 0x53, 0xbe, 0xbd, 0x40, + 0x41, 0x38, 0x1c, 0xb4, 0x8b, 0x79, 0x2e, 0x18, 0x96, 0x94, 0xde, 0xe8, 0xca, 0xe5, 0x9f, 0x92, 0x9f, 0x15, 0x5d, + 0x56, 0x60, 0x5c, 0x09, 0xf9, 0x16, 0xf4, 0x17, 0x0f, 0xf6, 0x4c, 0xda, 0xe6, 0x67, 0x89, 0x9f, 0xca, 0x6c, 0xe7, + 0x9b, 0x04, 0x62, 0x0e, 0x26, 0xa6, 0x52, 0xbd, 0x29, 0xff, 0xc7, 0xa4, 0x96, 0xe6, 0x6a, 0x02, 0xa5, 0x2e, 0x7b, + 0xfe, 0x97, 0x68, 0x3e, 0x2e, 0x5f, 0x3b, 0x0f, 0x36, 0xd6, 0x98, 0x19, 0x59, 0x48, 0xd2, 0xc6, 0xe1, 0x55, 0x1a, + 0x6e, 0xd6, 0xed, 0x2c, 0xba, 0xc3, 0x9e, 0x64, 0xc9, 0x95, 0x86, 0x35, 0x5e, 0x3e, 0x88, 0x69, 0x99, 0x4b, 0xee, + 0xbe, 0x9a, 0x99, 0xb5, 0x6e, 0x58, 0xae, 0xdd, 0x22, 0xdb, 0xdd, 0x6b, 0xfc, 0xaf, 0x90, 0xa3, 0x3d, 0xa4, 0xc1, + 0x15, 0x92, 0x18, 0x8d, 0xd2, 0x4b, 0x7b, 0x06, 0xd1, 0x37, 0xb5, 0xe2, 0x7c, 0x2c, 0xf0, 0x25, 0xe4, 0x94, 0x2a, + 0xbd, 0xe3, 0x82, 0x70, 0x78, 0xa3, 0x82, 0x10, 0x5a, 0x90, 0xd7, 0xa4, 0xfa, 0xaf, 0x1a, 0x88, 0x59, 0xdc, 0x74, + 0x12, 0xb4, 0x8e, 0xd7, 0x19, 0x46, 0xf4, 0x84, 0x69, 0x9f, 0xbb, 0x70, 0xa8, 0x4c, 0x52, 0x81, 0xa9, 0xff, 0x76, + 0x1c, 0xae, 0xd8, 0x11, 0x3d, 0x7f, 0x7d, 0xc5, 0x12, 0x59, 0x28, 0x18, 0xc2, 0xa2, 0xb7, 0x1c, 0x88, 0xf8, 0xd6, + 0x1b, 0xa6, 0x7d, 0x9e, 0xde, 0x29, 0xf8, 0xed, 0xff, 0xeb, 0x92, 0x24, 0x4f, 0x05, 0xaa, 0xd9, 0x49, 0xba, 0x87, + 0x59, 0x51, 0xc9, 0x20, 0x5c, 0x9b, 0x74, 0xcf, 0x03, 0xd9, 0x2d, 0x34, 0xc7, 0x5b, 0xa5, 0x40, 0xb2, 0x99, 0xf5, + 0xcb, 0xb4, 0xf6, 0xb7, 0x72, 0x4a, 0xd6, 0xbd, 0xb0, 0xf3, 0x93, 0xe0, 0x1b, 0xa8, 0x04, 0x1e, 0x35, 0xd4, 0x80, + 0x20, 0xf4, 0x9c, 0x31, 0x6b, 0x45, 0xb9, 0x15, 0xb0, 0x5e, 0xdd, 0x0a, 0x33, 0x9c, 0x83, 0xcd, 0x58, 0x89, 0x50, + 0x56, 0xbb, 0x81, 0x00, 0x91, 0x32, 0xf3, 0x1b, 0x3e, 0xcf, 0x45, 0xe1, 0xf9, 0xe1, 0x2c, 0x26, 0x78, 0x93, 0x9a, + 0x60, 0x46, 0xc9, 0xb5, 0x5e, 0x6a, 0x28, 0x92, 0x87, 0x3f, 0x63, 0x7b, 0xdb, 0xf7, 0xd0, 0x13, 0x9d, 0x32, 0x40, + 0x5e, 0xcf, 0xfb, 0x79, 0x68, 0x47, 0x4c, 0xfd, 0x01, 0x17, 0xe6, 0x97, 0x93, 0x78, 0xbb, 0xa6, 0x27, 0xa3, 0xe8, + 0x1a, 0xe8, 0x94, 0x55, 0x7d, 0x08, 0xe5, 0xdc, 0x66, 0xa3, 0x69, 0xc8, 0xca, 0xc5, 0xa1, 0x84, 0x55, 0xde, 0x08, + 0x91, 0x16, 0x3a, 0x0c, 0x86, 0xab, 0x27, 0x2b, 0x64, 0x34, 0x02, 0x6c, 0x76, 0x8b, 0xc6, 0xaf, 0xcc, 0xe1, 0xd6, + 0x8c, 0x2a, 0x18, 0x3d, 0xa6, 0x1b, 0x37, 0x75, 0x45, 0x73, 0xc2, 0x75, 0xd7, 0x53, 0x78, 0x3a, 0xd6, 0xe8, 0x29, + 0xd2, 0x4a, 0xa8, 0x1e, 0x82, 0xf6, 0xb6, 0x81, 0xde, 0x21, 0xed, 0x2b, 0x56, 0xbb, 0xf2, 0xd0, 0x57, 0xc1, 0x7c, + 0xd2, 0x6a, 0xd2, 0x56, 0xf5, 0x13, 0x5f, 0x1c, 0x6a, 0x0b, 0x74, 0xfb, 0xe9, 0xfe, 0x9e, 0xea, 0x95, 0xb2, 0x46, + 0xab, 0x0a, 0xfc, 0xfd, 0xf3, 0xbb, 0x04, 0x2b, 0x76, 0x1b, 0xa4, 0x74, 0xb0, 0xc1, 0x78, 0xc3, 0x69, 0xe2, 0xb0, + 0x01, 0xe1, 0xde, 0x32, 0x4c, 0x8d, 0x1a, 0xb3, 0x38, 0x08, 0xd5, 0xfc, 0x1f, 0xdc, 0x0e, 0x2c, 0x9c, 0xb1, 0xa1, + 0x63, 0x17, 0x22, 0xf5, 0x6c, 0x93, 0x70, 0x74, 0x00, 0xf8, 0x39, 0x01, 0x94, 0xd1, 0x32, 0x23, 0x56, 0x5d, 0xa6, + 0x02, 0x76, 0x76, 0x93, 0xce, 0x2f, 0x19, 0xe9, 0x17, 0x52, 0xae, 0x6e, 0x2c, 0x6d, 0x61, 0x7f, 0x3b, 0xaa, 0xe0, + 0x52, 0x85, 0xc5, 0x65, 0xc1, 0xbb, 0x8e, 0x5b, 0x21, 0xd5, 0xc9, 0x78, 0x83, 0x07, 0x97, 0x4c, 0x62, 0x61, 0x41, + 0xd4, 0xfc, 0xc9, 0x39, 0xe3, 0x9b, 0xd0, 0xcc, 0x75, 0xc4, 0x97, 0xe6, 0xdd, 0x2a, 0x5f, 0xa6, 0xe8, 0x59, 0x6c, + 0x98, 0xb9, 0x02, 0xe2, 0xa2, 0xd6, 0x68, 0xee, 0x3b, 0x1d, 0xe3, 0x4d, 0x5b, 0x30, 0xef, 0x03, 0xf2, 0xeb, 0x18, + 0x57, 0x36, 0xe8, 0xa1, 0xf4, 0x47, 0xfb, 0xcb, 0x8f, 0xcb, 0xc8, 0xf3, 0x4f, 0x74, 0x9d, 0x9d, 0xb1, 0x8d, 0x14, + 0x44, 0xd9, 0x19, 0xb4, 0x54, 0x4f, 0x75, 0x19, 0x09, 0xa0, 0x75, 0xbc, 0x3b, 0x82, 0xc6, 0x3f, 0xb8, 0x83, 0x19, + 0x6e, 0xd6, 0x37, 0xfe, 0x6e, 0x8a, 0x4e, 0xe0, 0x4a, 0xab, 0x7b, 0xc8, 0xb4, 0x1d, 0xf4, 0xed, 0x27, 0x03, 0x65, + 0xa2, 0xa1, 0xae, 0x11, 0xe7, 0x98, 0x78, 0x48, 0x91, 0xd2, 0xd2, 0xd4, 0x23, 0x78, 0x50, 0xb1, 0x5b, 0x85, 0x10, + 0x8d, 0xca, 0x5f, 0x0f, 0x71, 0xae, 0x72, 0x9a, 0xf6, 0x25, 0x19, 0x60, 0x06, 0xf7, 0x10, 0x34, 0x18, 0x0d, 0xc9, + 0x9f, 0x7b, 0x0c, 0x9b, 0x8f, 0x91, 0x1b, 0x9f, 0xcd, 0x10, 0xee, 0x75, 0xf9, 0x97, 0x66, 0xfc, 0x4d, 0x33, 0x6e, + 0x28, 0x2b, 0x92, 0x85, 0x4f, 0xab, 0x43, 0x8d, 0x8f, 0x7d, 0x86, 0xa7, 0xc7, 0xd8, 0xd3, 0x0b, 0x8b, 0x57, 0xb6, + 0x1d, 0x95, 0x0d, 0xe9, 0xbc, 0xd9, 0x03, 0xd9, 0x10, 0x19, 0xc3, 0x46, 0x63, 0x55, 0x87, 0x61, 0x79, 0x6c, 0x95, + 0x0e, 0x9c, 0xdd, 0xca, 0xc3, 0xf3, 0x64, 0xf0, 0x7d, 0x76, 0xb7, 0x53, 0x67, 0x2b, 0x1e, 0x44, 0x56, 0x81, 0xea, + 0x8f, 0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61, 0x10, 0x1e, 0xbf, 0xec, 0xa8 +}; +static CONST UINT8 EncOutput011[] = { + 0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8, 0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc, 0x3c, 0xba, 0x6a, + 0x77, 0x47, 0xdb, 0xe3, 0x74, 0x7a, 0x4d, 0x5f, 0x8d, 0x37, 0x55, 0x80, 0x73, 0x90, 0x66, 0x5d, 0x3a, 0x7d, 0x5d, + 0x86, 0x5e, 0x8d, 0xfd, 0x83, 0xff, 0x4e, 0x74, 0x6f, 0xf9, 0xe6, 0x70, 0x17, 0x70, 0x3e, 0x96, 0xa7, 0x7e, 0xcb, + 0xab, 0x8f, 0x58, 0x24, 0x9b, 0x01, 0xfd, 0xcb, 0xe6, 0x4d, 0x9b, 0xf0, 0x88, 0x94, 0x57, 0x66, 0xef, 0x72, 0x4c, + 0x42, 0x6e, 0x16, 0x19, 0x15, 0xea, 0x70, 0x5b, 0xac, 0x13, 0xdb, 0x9f, 0x18, 0xe2, 0x3c, 0x26, 0x97, 0xbc, 0xdc, + 0x45, 0x8c, 0x6c, 0x24, 0x69, 0x9c, 0xf7, 0x65, 0x1e, 0x18, 0x59, 0x31, 0x7c, 0xe4, 0x73, 0xbc, 0x39, 0x62, 0xc6, + 0x5c, 0x9f, 0xbf, 0xfa, 0x90, 0x03, 0xc9, 0x72, 0x26, 0xb6, 0x1b, 0xc2, 0xb7, 0x3f, 0xf2, 0x13, 0x77, 0xf2, 0x8d, + 0xb9, 0x47, 0xd0, 0x53, 0xdd, 0xc8, 0x91, 0x83, 0x8b, 0xb1, 0xce, 0xa3, 0xfe, 0xcd, 0xd9, 0xdd, 0x92, 0x7b, 0xdb, + 0xb8, 0xfb, 0xc9, 0x2d, 0x01, 0x59, 0x39, 0x52, 0xad, 0x1b, 0xec, 0xcf, 0xd7, 0x70, 0x13, 0x21, 0xf5, 0x47, 0xaa, + 0x18, 0x21, 0x5c, 0xc9, 0x9a, 0xd2, 0x6b, 0x05, 0x9c, 0x01, 0xa1, 0xda, 0x35, 0x5d, 0xb3, 0x70, 0xe6, 0xa9, 0x80, + 0x8b, 0x91, 0xb7, 0xb3, 0x5f, 0x24, 0x9a, 0xb7, 0xd1, 0x6b, 0xa1, 0x1c, 0x50, 0xba, 0x49, 0xe0, 0xee, 0x2e, 0x75, + 0xac, 0x69, 0xc0, 0xeb, 0x03, 0xdd, 0x19, 0xe5, 0xf6, 0x06, 0xdd, 0xc3, 0xd7, 0x2b, 0x07, 0x07, 0x30, 0xa7, 0x19, + 0x0c, 0xbf, 0xe6, 0x18, 0xcc, 0xb1, 0x01, 0x11, 0x85, 0x77, 0x1d, 0x96, 0xa7, 0xa3, 0x00, 0x84, 0x02, 0xa2, 0x83, + 0x68, 0xda, 0x17, 0x27, 0xc8, 0x7f, 0x23, 0xb7, 0xf4, 0x13, 0x85, 0xcf, 0xdd, 0x7a, 0x7d, 0x24, 0x57, 0xfe, 0x05, + 0x93, 0xf5, 0x74, 0xce, 0xed, 0x0c, 0x20, 0x98, 0x8d, 0x92, 0x30, 0xa1, 0x29, 0x23, 0x1a, 0xa0, 0x4f, 0x69, 0x56, + 0x4c, 0xe1, 0xc8, 0xce, 0xf6, 0x9a, 0x0c, 0xa4, 0xfa, 0x04, 0xf6, 0x62, 0x95, 0xf2, 0xfa, 0xc7, 0x40, 0x68, 0x40, + 0x8f, 0x41, 0xda, 0xb4, 0x26, 0x6f, 0x70, 0xab, 0x40, 0x61, 0xa4, 0x0e, 0x75, 0xfb, 0x86, 0xeb, 0x9d, 0x9a, 0x1f, + 0xec, 0x76, 0x99, 0xe7, 0xea, 0xaa, 0x1e, 0x2d, 0xb5, 0xd4, 0xa6, 0x1a, 0xb8, 0x61, 0x0a, 0x1d, 0x16, 0x5b, 0x98, + 0xc2, 0x31, 0x40, 0xe7, 0x23, 0x1d, 0x66, 0x99, 0xc8, 0xc0, 0xd7, 0xce, 0xf3, 0x57, 0x40, 0x04, 0x3f, 0xfc, 0xea, + 0xb3, 0xfc, 0xd2, 0xd3, 0x99, 0xa4, 0x94, 0x69, 0xa0, 0xef, 0xd1, 0x85, 0xb3, 0xa6, 0xb1, 0x28, 0xbf, 0x94, 0x67, + 0x22, 0xc3, 0x36, 0x46, 0xf8, 0xd2, 0x0f, 0x5f, 0xf4, 0x59, 0x80, 0xe6, 0x2d, 0x43, 0x08, 0x7d, 0x19, 0x09, 0x97, + 0xa7, 0x4c, 0x3d, 0x8d, 0xba, 0x65, 0x62, 0xa3, 0x71, 0x33, 0x29, 0x62, 0xdb, 0xc1, 0x33, 0x34, 0x1a, 0x63, 0x33, + 0x16, 0xb6, 0x64, 0x7e, 0xab, 0x33, 0xf0, 0xe6, 0x26, 0x68, 0xba, 0x1d, 0x2e, 0x38, 0x08, 0xe6, 0x02, 0xd3, 0x25, + 0x2c, 0x47, 0x23, 0x58, 0x34, 0x0f, 0x9d, 0x63, 0x4f, 0x63, 0xbb, 0x7f, 0x3b, 0x34, 0x38, 0xa7, 0xb5, 0x8d, 0x65, + 0xd9, 0x9f, 0x79, 0x55, 0x3e, 0x4d, 0xe7, 0x73, 0xd8, 0xf6, 0x98, 0x97, 0x84, 0x60, 0x9c, 0xc8, 0xa9, 0x3c, 0xf6, + 0xdc, 0x12, 0x5c, 0xe1, 0xbb, 0x0b, 0x8b, 0x98, 0x9c, 0x9d, 0x26, 0x7c, 0x4a, 0xe6, 0x46, 0x36, 0x58, 0x21, 0x4a, + 0xee, 0xca, 0xd7, 0x3b, 0xc2, 0x6c, 0x49, 0x2f, 0xe5, 0xd5, 0x03, 0x59, 0x84, 0x53, 0xcb, 0xfe, 0x92, 0x71, 0x2e, + 0x7c, 0x21, 0xcc, 0x99, 0x85, 0x7f, 0xb8, 0x74, 0x90, 0x13, 0x42, 0x3f, 0xe0, 0x6b, 0x1d, 0xf2, 0x4d, 0x54, 0xd4, + 0xfc, 0x3a, 0x05, 0xe6, 0x74, 0xaf, 0xa6, 0xa0, 0x2a, 0x20, 0x23, 0x5d, 0x34, 0x5c, 0xd9, 0x3e, 0x4e, 0xfa, 0x93, + 0xe7, 0xaa, 0xe9, 0x6f, 0x08, 0x43, 0x67, 0x41, 0xc5, 0xad, 0xfb, 0x31, 0x95, 0x82, 0x73, 0x32, 0xd8, 0xa6, 0xa3, + 0xed, 0x0e, 0x2d, 0xf6, 0x5f, 0xfd, 0x80, 0xa6, 0x7a, 0xe0, 0xdf, 0x78, 0x15, 0x29, 0x74, 0x33, 0xd0, 0x9e, 0x83, + 0x86, 0x72, 0x22, 0x57, 0x29, 0xb9, 0x9e, 0x5d, 0xd3, 0x1a, 0xb5, 0x96, 0x72, 0x41, 0x3d, 0xf1, 0x64, 0x43, 0x67, + 0xee, 0xaa, 0x5c, 0xd3, 0x9a, 0x96, 0x13, 0x11, 0x5d, 0xf3, 0x0c, 0x87, 0x82, 0x1e, 0x41, 0x9e, 0xd0, 0x27, 0xd7, + 0x54, 0x3b, 0x67, 0x73, 0x09, 0x91, 0xe9, 0xd5, 0x36, 0xa7, 0xb5, 0x55, 0xe4, 0xf3, 0x21, 0x51, 0x49, 0x22, 0x07, + 0x55, 0x4f, 0x44, 0x4b, 0xd2, 0x15, 0x93, 0x17, 0x2a, 0xfa, 0x4d, 0x4a, 0x57, 0xdb, 0x4c, 0xa6, 0xeb, 0xec, 0x53, + 0x25, 0x6c, 0x21, 0xed, 0x00, 0x4c, 0x3b, 0xca, 0x14, 0x57, 0xa9, 0xd6, 0x6a, 0xcd, 0x8d, 0x5e, 0x74, 0xac, 0x72, + 0xc1, 0x97, 0xe5, 0x1b, 0x45, 0x4e, 0xda, 0xfc, 0xcc, 0x40, 0xe8, 0x48, 0x88, 0x0b, 0xa3, 0xe3, 0x8d, 0x83, 0x42, + 0xc3, 0x23, 0xfd, 0x68, 0xb5, 0x8e, 0xf1, 0x9d, 0x63, 0x77, 0xe9, 0xa3, 0x8e, 0x8c, 0x26, 0x6b, 0xbd, 0x72, 0x73, + 0x35, 0x0c, 0x03, 0xf8, 0x43, 0x78, 0x52, 0x71, 0x15, 0x1f, 0x71, 0x5d, 0x6e, 0xed, 0xb9, 0xcc, 0x86, 0x30, 0xdb, + 0x2b, 0xd3, 0x82, 0x88, 0x23, 0x71, 0x90, 0x53, 0x5c, 0xa9, 0x2f, 0x76, 0x01, 0xb7, 0x9a, 0xfe, 0x43, 0x55, 0xa3, + 0x04, 0x9b, 0x0e, 0xe4, 0x59, 0xdf, 0xc9, 0xe9, 0xb1, 0xea, 0x29, 0x28, 0x3c, 0x5c, 0xae, 0x72, 0x84, 0xb6, 0xc6, + 0xeb, 0x0c, 0x27, 0x07, 0x74, 0x90, 0x0d, 0x31, 0xb0, 0x00, 0x77, 0xe9, 0x40, 0x70, 0x6f, 0x68, 0xa7, 0xfd, 0x06, + 0xec, 0x4b, 0xc0, 0xb7, 0xac, 0xbc, 0x33, 0xb7, 0x6d, 0x0a, 0xbd, 0x12, 0x1b, 0x59, 0xcb, 0xdd, 0x32, 0xf5, 0x1d, + 0x94, 0x57, 0x76, 0x9e, 0x0c, 0x18, 0x98, 0x71, 0xd7, 0x2a, 0xdb, 0x0b, 0x7b, 0xa7, 0x71, 0xb7, 0x67, 0x81, 0x23, + 0x96, 0xae, 0xb9, 0x7e, 0x32, 0x43, 0x92, 0x8a, 0x19, 0xa0, 0xc4, 0xd4, 0x3b, 0x57, 0xf9, 0x4a, 0x2c, 0xfb, 0x51, + 0x46, 0xbb, 0xcb, 0x5d, 0xb3, 0xef, 0x13, 0x93, 0x6e, 0x68, 0x42, 0x54, 0x57, 0xd3, 0x6a, 0x3a, 0x8f, 0x9d, 0x66, + 0xbf, 0xbd, 0x36, 0x23, 0xf5, 0x93, 0x83, 0x7b, 0x9c, 0xc0, 0xdd, 0xc5, 0x49, 0xc0, 0x64, 0xed, 0x07, 0x12, 0xb3, + 0xe6, 0xe4, 0xe5, 0x38, 0x95, 0x23, 0xb1, 0xa0, 0x3b, 0x1a, 0x61, 0xda, 0x17, 0xac, 0xc3, 0x58, 0xdd, 0x74, 0x64, + 0x22, 0x11, 0xe8, 0x32, 0x1d, 0x16, 0x93, 0x85, 0x99, 0xa5, 0x9c, 0x34, 0x55, 0xb1, 0xe9, 0x20, 0x72, 0xc9, 0x28, + 0x7b, 0x79, 0x00, 0xa1, 0xa6, 0xa3, 0x27, 0x40, 0x18, 0x8a, 0x54, 0xe0, 0xcc, 0xe8, 0x4e, 0x8e, 0x43, 0x96, 0xe7, + 0x3f, 0xc8, 0xe9, 0xb2, 0xf9, 0xc9, 0xda, 0x04, 0x71, 0x50, 0x47, 0xe4, 0xaa, 0xce, 0xa2, 0x30, 0xc8, 0xe4, 0xac, + 0xc7, 0x0d, 0x06, 0x2e, 0xe6, 0xe8, 0x80, 0x36, 0x29, 0x9e, 0x01, 0xb8, 0xc3, 0xf0, 0xa0, 0x5d, 0x7a, 0xca, 0x4d, + 0xa0, 0x57, 0xbd, 0x2a, 0x45, 0xa7, 0x7f, 0x9c, 0x93, 0x07, 0x8f, 0x35, 0x67, 0x92, 0xe3, 0xe9, 0x7f, 0xa8, 0x61, + 0x43, 0x9e, 0x25, 0x4f, 0x33, 0x76, 0x13, 0x6e, 0x12, 0xb9, 0xdd, 0xa4, 0x7c, 0x08, 0x9f, 0x7c, 0xe7, 0x0a, 0x8d, + 0x84, 0x06, 0xa4, 0x33, 0x17, 0x34, 0x5e, 0x10, 0x7c, 0xc0, 0xa8, 0x3d, 0x1f, 0x42, 0x20, 0x51, 0x65, 0x5d, 0x09, + 0xc3, 0xaa, 0xc0, 0xc8, 0x0d, 0xf0, 0x79, 0xbc, 0x20, 0x1b, 0x95, 0xe7, 0x06, 0x7d, 0x47, 0x20, 0x03, 0x1a, 0x74, + 0xdd, 0xe2, 0xd4, 0xae, 0x38, 0x71, 0x9b, 0xf5, 0x80, 0xec, 0x08, 0x4e, 0x56, 0xba, 0x76, 0x12, 0x1a, 0xdf, 0x48, + 0xf3, 0xae, 0xb3, 0xe6, 0xe6, 0xbe, 0xc0, 0x91, 0x2e, 0x01, 0xb3, 0x01, 0x86, 0xa2, 0xb9, 0x52, 0xd1, 0x21, 0xae, + 0xd4, 0x97, 0x1d, 0xef, 0x41, 0x12, 0x95, 0x3d, 0x48, 0x45, 0x1c, 0x56, 0x32, 0x8f, 0xb8, 0x43, 0xbb, 0x19, 0xf3, + 0xca, 0xe9, 0xeb, 0x6d, 0x84, 0xbe, 0x86, 0x06, 0xe2, 0x36, 0xb2, 0x62, 0x9d, 0xd3, 0x4c, 0x48, 0x18, 0x54, 0x13, + 0x4e, 0xcf, 0xfd, 0xba, 0x84, 0xb9, 0x30, 0x53, 0xcf, 0xfb, 0xb9, 0x29, 0x8f, 0xdc, 0x9f, 0xef, 0x60, 0x0b, 0x64, + 0xf6, 0x8b, 0xee, 0xa6, 0x91, 0xc2, 0x41, 0x6c, 0xf6, 0xfa, 0x79, 0x67, 0x4b, 0xc1, 0x3f, 0xaf, 0x09, 0x81, 0xd4, + 0x5d, 0xcb, 0x09, 0xdf, 0x36, 0x31, 0xc0, 0x14, 0x3c, 0x7c, 0x0e, 0x65, 0x95, 0x99, 0x6d, 0xa3, 0xf4, 0xd7, 0x38, + 0xee, 0x1a, 0x2b, 0x37, 0xe2, 0xa4, 0x3b, 0x4b, 0xd0, 0x65, 0xca, 0xf8, 0xc3, 0xe8, 0x15, 0x20, 0xef, 0xf2, 0x00, + 0xfd, 0x01, 0x09, 0xc5, 0xc8, 0x17, 0x04, 0x93, 0xd0, 0x93, 0x03, 0x55, 0xc5, 0xfe, 0x32, 0xa3, 0x3e, 0x28, 0x2d, + 0x3b, 0x93, 0x8a, 0xcc, 0x07, 0x72, 0x80, 0x8b, 0x74, 0x16, 0x24, 0xbb, 0xda, 0x94, 0x39, 0x30, 0x8f, 0xb1, 0xcd, + 0x4a, 0x90, 0x92, 0x7c, 0x14, 0x8f, 0x95, 0x4e, 0xac, 0x9b, 0xd8, 0x8f, 0x1a, 0x87, 0xa4, 0x32, 0x27, 0x8a, 0xba, + 0xf7, 0x41, 0xcf, 0x84, 0x37, 0x19, 0xe6, 0x06, 0xf5, 0x0e, 0xcf, 0x36, 0xf5, 0x9e, 0x6c, 0xde, 0xbc, 0xff, 0x64, + 0x7e, 0x4e, 0x59, 0x57, 0x48, 0xfe, 0x14, 0xf7, 0x9c, 0x93, 0x5d, 0x15, 0xad, 0xcc, 0x11, 0xb1, 0x17, 0x18, 0xb2, + 0x7e, 0xcc, 0xab, 0xe9, 0xce, 0x7d, 0x77, 0x5b, 0x51, 0x1b, 0x1e, 0x20, 0xa8, 0x32, 0x06, 0x0e, 0x75, 0x93, 0xac, + 0xdb, 0x35, 0x37, 0x1f, 0xe9, 0x19, 0x1d, 0xb4, 0x71, 0x97, 0xd6, 0x4e, 0x2c, 0x08, 0xa5, 0x13, 0xf9, 0x0e, 0x7e, + 0x78, 0x6e, 0x14, 0xe0, 0xa9, 0xb9, 0x96, 0x4c, 0x80, 0x82, 0xba, 0x17, 0xb3, 0x9d, 0x69, 0xb0, 0x84, 0x46, 0xff, + 0xf9, 0x52, 0x79, 0x94, 0x58, 0x3a, 0x62, 0x90, 0x15, 0x35, 0x71, 0x10, 0x37, 0xed, 0xa1, 0x8e, 0x53, 0x6e, 0xf4, + 0x26, 0x57, 0x93, 0x15, 0x93, 0xf6, 0x81, 0x2c, 0x5a, 0x10, 0xda, 0x92, 0xad, 0x2f, 0xdb, 0x28, 0x31, 0x2d, 0x55, + 0x04, 0xd2, 0x06, 0x28, 0x8c, 0x1e, 0xdc, 0xea, 0x54, 0xac, 0xff, 0xb7, 0x6c, 0x30, 0x15, 0xd4, 0xb4, 0x0d, 0x00, + 0x93, 0x57, 0xdd, 0xd2, 0x07, 0x07, 0x06, 0xd9, 0x43, 0x9b, 0xcd, 0x3a, 0xf4, 0x7d, 0x4c, 0x36, 0x5d, 0x23, 0xa2, + 0xcc, 0x57, 0x40, 0x91, 0xe9, 0x2c, 0x2f, 0x2c, 0xd5, 0x30, 0x9b, 0x17, 0xb0, 0xc9, 0xf7, 0xa7, 0x2f, 0xd1, 0x93, + 0x20, 0x6b, 0xc6, 0xc1, 0xe4, 0x6f, 0xcb, 0xd1, 0xe7, 0x09, 0x0f, 0x9e, 0xdc, 0xaa, 0x9f, 0x2f, 0xdf, 0x56, 0x9f, + 0xd4, 0x33, 0x04, 0xaf, 0xd3, 0x6c, 0x58, 0x61, 0xf0, 0x30, 0xec, 0xf2, 0x7f, 0xf2, 0x9c, 0xdf, 0x39, 0xbb, 0x6f, + 0xa2, 0x8c, 0x7e, 0xc4, 0x22, 0x51, 0x71, 0xc0, 0x4d, 0x14, 0x1a, 0xc4, 0xcd, 0x04, 0xd9, 0x87, 0x08, 0x50, 0x05, + 0xcc, 0xaf, 0xf6, 0xf0, 0x8f, 0x92, 0x54, 0x58, 0xc2, 0xc7, 0x09, 0x7a, 0x59, 0x02, 0x05, 0xe8, 0xb0, 0x86, 0xd9, + 0xbf, 0x7b, 0x35, 0x51, 0x4d, 0xaf, 0x08, 0x97, 0x2c, 0x65, 0xda, 0x2a, 0x71, 0x3a, 0xa8, 0x51, 0xcc, 0xf2, 0x73, + 0x27, 0xc3, 0xfd, 0x62, 0xcf, 0xe3, 0xb2, 0xca, 0xcb, 0xbe, 0x1a, 0x0a, 0xa1, 0x34, 0x7b, 0x77, 0xc4, 0x62, 0x68, + 0x78, 0x5f, 0x94, 0x07, 0x04, 0x65, 0x16, 0x4b, 0x61, 0xcb, 0xff, 0x75, 0x26, 0x50, 0x66, 0x1f, 0x6e, 0x93, 0xf8, + 0xc5, 0x51, 0xeb, 0xa4, 0x4a, 0x48, 0x68, 0x6b, 0xe2, 0x5e, 0x44, 0xb2, 0x50, 0x2c, 0x6c, 0xae, 0x79, 0x4e, 0x66, + 0x35, 0x81, 0x50, 0xac, 0xbc, 0x3f, 0xb1, 0x0c, 0xf3, 0x05, 0x3c, 0x4a, 0xa3, 0x6c, 0x2a, 0x79, 0xb4, 0xb7, 0xab, + 0xca, 0xc7, 0x9b, 0x8e, 0xcd, 0x5f, 0x11, 0x03, 0xcb, 0x30, 0xa3, 0xab, 0xda, 0xfe, 0x64, 0xb9, 0xbb, 0xd8, 0x5e, + 0x3a, 0x1a, 0x56, 0xe5, 0x05, 0x48, 0x90, 0x1e, 0x61, 0x69, 0x1b, 0x22, 0xe6, 0x1a, 0x3c, 0x75, 0xad, 0x1f, 0x37, + 0x28, 0xdc, 0xe4, 0x6d, 0xbd, 0x42, 0xdc, 0xd3, 0xc8, 0xb6, 0x1c, 0x48, 0xfe, 0x94, 0x77, 0x7f, 0xbd, 0x62, 0xac, + 0xa3, 0x47, 0x27, 0xcf, 0x5f, 0xd9, 0xdb, 0xaf, 0xec, 0xf7, 0x5e, 0xc1, 0xb0, 0x9d, 0x01, 0x26, 0x99, 0x7e, 0x8f, + 0x03, 0x70, 0xb5, 0x42, 0xbe, 0x67, 0x28, 0x1b, 0x7c, 0xbd, 0x61, 0x21, 0x97, 0xcc, 0x5c, 0xe1, 0x97, 0x8f, 0x8d, + 0xde, 0x2b, 0xaa, 0xa7, 0x71, 0x1d, 0x1e, 0x02, 0x73, 0x70, 0x58, 0x32, 0x5b, 0x1d, 0x67, 0x3d, 0xe0, 0x74, 0x4f, + 0x03, 0xf2, 0x70, 0x51, 0x79, 0xf1, 0x61, 0x70, 0x15, 0x74, 0x9d, 0x23, 0x89, 0xde, 0xac, 0xfd, 0xde, 0xd0, 0x1f, + 0xc3, 0x87, 0x44, 0x35, 0x4b, 0xe5, 0xb0, 0x60, 0xc5, 0x22, 0xe4, 0x9e, 0xca, 0xeb, 0xd5, 0x3a, 0x09, 0x45, 0xa4, + 0xdb, 0xfa, 0x3f, 0xeb, 0x1b, 0xc7, 0xc8, 0x14, 0x99, 0x51, 0x92, 0x10, 0xed, 0xed, 0x28, 0xe0, 0xa1, 0xf8, 0x26, + 0xcf, 0xcd, 0xcb, 0x63, 0xa1, 0x3b, 0xe3, 0xdf, 0x7e, 0xfe, 0xa6, 0xf0, 0x81, 0x9a, 0xbf, 0x55, 0xde, 0x54, 0xd5, + 0x56, 0x60, 0x98, 0x10, 0x68, 0xf4, 0x38, 0x96, 0x8e, 0x6f, 0x1d, 0x44, 0x7f, 0xd6, 0x2f, 0xfe, 0x55, 0xfb, 0x0c, + 0x7e, 0x67, 0xe2, 0x61, 0x44, 0xed, 0xf2, 0x35, 0x30, 0x5d, 0xe9, 0xc7, 0xd6, 0x6d, 0xe0, 0xa0, 0xed, 0xf3, 0xfc, + 0xd8, 0x3e, 0x0a, 0x7b, 0xcd, 0xaf, 0x65, 0x68, 0x18, 0xc0, 0xec, 0x04, 0x1c, 0x74, 0x6d, 0xe2, 0x6e, 0x79, 0xd4, + 0x11, 0x2b, 0x62, 0xd5, 0x27, 0xad, 0x4f, 0x01, 0x59, 0x73, 0xcc, 0x6a, 0x53, 0xfb, 0x2d, 0xd5, 0x4e, 0x99, 0x21, + 0x65, 0x4d, 0xf5, 0x82, 0xf7, 0xd8, 0x42, 0xce, 0x6f, 0x3d, 0x36, 0x47, 0xf1, 0x05, 0x16, 0xe8, 0x1b, 0x6a, 0x8f, + 0x93, 0xf2, 0x8f, 0x37, 0x40, 0x12, 0x28, 0xa3, 0xe6, 0xb9, 0x17, 0x4a, 0x1f, 0xb1, 0xd1, 0x66, 0x69, 0x86, 0xc4, + 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e, 0x2b, 0xdf, 0xcd, 0xf9, 0x3c +}; +static CONST UINT8 EncAssoc011[] = { 0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7 }; +static CONST UINT8 EncNonce011[] = { 0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa }; +static CONST UINT8 EncKey011[] = { 0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85, 0xf2, 0xfb, 0xed, + 0x7b, 0xd0, 0x9e, 0x97, 0xca, 0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, + 0xcc, 0x52, 0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38 }; + +static CONST UINT8 EncInput012[] = { + 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, 0x42, 0x34, 0x8a, + 0xa1, 0x03, 0xb7, 0xe9, 0x57, 0x46, 0xbe, 0x20, 0xe4, 0x6e, 0xb0, 0xeb, 0xff, 0xea, 0x07, 0x7e, 0xef, 0xe2, 0x55, + 0x9f, 0xe5, 0x78, 0x3a, 0xb7, 0x83, 0xc2, 0x18, 0x40, 0x7b, 0xeb, 0xcd, 0x81, 0xfb, 0x90, 0x12, 0x9e, 0x46, 0xa9, + 0xd6, 0x4a, 0xba, 0xb0, 0x62, 0xdb, 0x6b, 0x99, 0xc4, 0xdb, 0x54, 0x4b, 0xb8, 0xa5, 0x71, 0xcb, 0xcd, 0x63, 0x32, + 0x55, 0xfb, 0x31, 0xf0, 0x38, 0xf5, 0xbe, 0x78, 0xe4, 0x45, 0xce, 0x1b, 0x6a, 0x5b, 0x0e, 0xf4, 0x16, 0xe4, 0xb1, + 0x3d, 0xf6, 0x63, 0x7b, 0xa7, 0x0c, 0xde, 0x6f, 0x8f, 0x74, 0xdf, 0xe0, 0x1e, 0x9d, 0xce, 0x8f, 0x24, 0xef, 0x23, + 0x35, 0x33, 0x7b, 0x83, 0x34, 0x23, 0x58, 0x74, 0x14, 0x77, 0x1f, 0xc2, 0x4f, 0x4e, 0xc6, 0x89, 0xf9, 0x52, 0x09, + 0x37, 0x64, 0x14, 0xc4, 0x01, 0x6b, 0x9d, 0x77, 0xe8, 0x90, 0x5d, 0xa8, 0x4a, 0x2a, 0xef, 0x5c, 0x7f, 0xeb, 0xbb, + 0xb2, 0xc6, 0x93, 0x99, 0x66, 0xdc, 0x7f, 0xd4, 0x9e, 0x2a, 0xca, 0x8d, 0xdb, 0xe7, 0x20, 0xcf, 0xe4, 0x73, 0xae, + 0x49, 0x7d, 0x64, 0x0f, 0x0e, 0x28, 0x46, 0xa9, 0xa8, 0x32, 0xe4, 0x0e, 0xf6, 0x51, 0x53, 0xb8, 0x3c, 0xb1, 0xff, + 0xa3, 0x33, 0x41, 0x75, 0xff, 0xf1, 0x6f, 0xf1, 0xfb, 0xbb, 0x83, 0x7f, 0x06, 0x9b, 0xe7, 0x1b, 0x0a, 0xe0, 0x5c, + 0x33, 0x60, 0x5b, 0xdb, 0x5b, 0xed, 0xfe, 0xa5, 0x16, 0x19, 0x72, 0xa3, 0x64, 0x23, 0x00, 0x02, 0xc7, 0xf3, 0x6a, + 0x81, 0x3e, 0x44, 0x1d, 0x79, 0x15, 0x5f, 0x9a, 0xde, 0xe2, 0xfd, 0x1b, 0x73, 0xc1, 0xbc, 0x23, 0xba, 0x31, 0xd2, + 0x50, 0xd5, 0xad, 0x7f, 0x74, 0xa7, 0xc9, 0xf8, 0x3e, 0x2b, 0x26, 0x10, 0xf6, 0x03, 0x36, 0x74, 0xe4, 0x0e, 0x6a, + 0x72, 0xb7, 0x73, 0x0a, 0x42, 0x28, 0xc2, 0xad, 0x5e, 0x03, 0xbe, 0xb8, 0x0b, 0xa8, 0x5b, 0xd4, 0xb8, 0xba, 0x52, + 0x89, 0xb1, 0x9b, 0xc1, 0xc3, 0x65, 0x87, 0xed, 0xa5, 0xf4, 0x86, 0xfd, 0x41, 0x80, 0x91, 0x27, 0x59, 0x53, 0x67, + 0x15, 0x78, 0x54, 0x8b, 0x2d, 0x3d, 0xc7, 0xff, 0x02, 0x92, 0x07, 0x5f, 0x7a, 0x4b, 0x60, 0x59, 0x3c, 0x6f, 0x5c, + 0xd8, 0xec, 0x95, 0xd2, 0xfe, 0xa0, 0x3b, 0xd8, 0x3f, 0xd1, 0x69, 0xa6, 0xd6, 0x41, 0xb2, 0xf4, 0x4d, 0x12, 0xf4, + 0x58, 0x3e, 0x66, 0x64, 0x80, 0x31, 0x9b, 0xa8, 0x4c, 0x8b, 0x07, 0xb2, 0xec, 0x66, 0x94, 0x66, 0x47, 0x50, 0x50, + 0x5f, 0x18, 0x0b, 0x0e, 0xd6, 0xc0, 0x39, 0x21, 0x13, 0x9e, 0x33, 0xbc, 0x79, 0x36, 0x02, 0x96, 0x70, 0xf0, 0x48, + 0x67, 0x2f, 0x26, 0xe9, 0x6d, 0x10, 0xbb, 0xd6, 0x3f, 0xd1, 0x64, 0x7a, 0x2e, 0xbe, 0x0c, 0x61, 0xf0, 0x75, 0x42, + 0x38, 0x23, 0xb1, 0x9e, 0x9f, 0x7c, 0x67, 0x66, 0xd9, 0x58, 0x9a, 0xf1, 0xbb, 0x41, 0x2a, 0x8d, 0x65, 0x84, 0x94, + 0xfc, 0xdc, 0x6a, 0x50, 0x64, 0xdb, 0x56, 0x33, 0x76, 0x00, 0x10, 0xed, 0xbe, 0xd2, 0x12, 0xf6, 0xf6, 0x1b, 0xa2, + 0x16, 0xde, 0xae, 0x31, 0x95, 0xdd, 0xb1, 0x08, 0x7e, 0x4e, 0xee, 0xe7, 0xf9, 0xa5, 0xfb, 0x5b, 0x61, 0x43, 0x00, + 0x40, 0xf6, 0x7e, 0x02, 0x04, 0x32, 0x4e, 0x0c, 0xe2, 0x66, 0x0d, 0xd7, 0x07, 0x98, 0x0e, 0xf8, 0x72, 0x34, 0x6d, + 0x95, 0x86, 0xd7, 0xcb, 0x31, 0x54, 0x47, 0xd0, 0x38, 0x29, 0x9c, 0x5a, 0x68, 0xd4, 0x87, 0x76, 0xc9, 0xe7, 0x7e, + 0xe3, 0xf4, 0x81, 0x6d, 0x18, 0xcb, 0xc9, 0x05, 0xaf, 0xa0, 0xfb, 0x66, 0xf7, 0xf1, 0x1c, 0xc6, 0x14, 0x11, 0x4f, + 0x2b, 0x79, 0x42, 0x8b, 0xbc, 0xac, 0xe7, 0x6c, 0xfe, 0x0f, 0x58, 0xe7, 0x7c, 0x78, 0x39, 0x30, 0xb0, 0x66, 0x2c, + 0x9b, 0x6d, 0x3a, 0xe1, 0xcf, 0xc9, 0xa4, 0x0e, 0x6d, 0x6d, 0x8a, 0xa1, 0x3a, 0xe7, 0x28, 0xd4, 0x78, 0x4c, 0xa6, + 0xa2, 0x2a, 0xa6, 0x03, 0x30, 0xd7, 0xa8, 0x25, 0x66, 0x87, 0x2f, 0x69, 0x5c, 0x4e, 0xdd, 0xa5, 0x49, 0x5d, 0x37, + 0x4a, 0x59, 0xc4, 0xaf, 0x1f, 0xa2, 0xe4, 0xf8, 0xa6, 0x12, 0x97, 0xd5, 0x79, 0xf5, 0xe2, 0x4a, 0x2b, 0x5f, 0x61, + 0xe4, 0x9e, 0xe3, 0xee, 0xb8, 0xa7, 0x5b, 0x2f, 0xf4, 0x9e, 0x6c, 0xfb, 0xd1, 0xc6, 0x56, 0x77, 0xba, 0x75, 0xaa, + 0x3d, 0x1a, 0xa8, 0x0b, 0xb3, 0x68, 0x24, 0x00, 0x10, 0x7f, 0xfd, 0xd7, 0xa1, 0x8d, 0x83, 0x54, 0x4f, 0x1f, 0xd8, + 0x2a, 0xbe, 0x8a, 0x0c, 0x87, 0xab, 0xa2, 0xde, 0xc3, 0x39, 0xbf, 0x09, 0x03, 0xa5, 0xf3, 0x05, 0x28, 0xe1, 0xe1, + 0xee, 0x39, 0x70, 0x9c, 0xd8, 0x81, 0x12, 0x1e, 0x02, 0x40, 0xd2, 0x6e, 0xf0, 0xeb, 0x1b, 0x3d, 0x22, 0xc6, 0xe5, + 0xe3, 0xb4, 0x5a, 0x98, 0xbb, 0xf0, 0x22, 0x28, 0x8d, 0xe5, 0xd3, 0x16, 0x48, 0x24, 0xa5, 0xe6, 0x66, 0x0c, 0xf9, + 0x08, 0xf9, 0x7e, 0x1e, 0xe1, 0x28, 0x26, 0x22, 0xc7, 0xc7, 0x0a, 0x32, 0x47, 0xfa, 0xa3, 0xbe, 0x3c, 0xc4, 0xc5, + 0x53, 0x0a, 0xd5, 0x94, 0x4a, 0xd7, 0x93, 0xd8, 0x42, 0x99, 0xb9, 0x0a, 0xdb, 0x56, 0xf7, 0xb9, 0x1c, 0x53, 0x4f, + 0xfa, 0xd3, 0x74, 0xad, 0xd9, 0x68, 0xf1, 0x1b, 0xdf, 0x61, 0xc6, 0x5e, 0xa8, 0x48, 0xfc, 0xd4, 0x4a, 0x4c, 0x3c, + 0x32, 0xf7, 0x1c, 0x96, 0x21, 0x9b, 0xf9, 0xa3, 0xcc, 0x5a, 0xce, 0xd5, 0xd7, 0x08, 0x24, 0xf6, 0x1c, 0xfd, 0xdd, + 0x38, 0xc2, 0x32, 0xe9, 0xb8, 0xe7, 0xb6, 0xfa, 0x9d, 0x45, 0x13, 0x2c, 0x83, 0xfd, 0x4a, 0x69, 0x82, 0xcd, 0xdc, + 0xb3, 0x76, 0x0c, 0x9e, 0xd8, 0xf4, 0x1b, 0x45, 0x15, 0xb4, 0x97, 0xe7, 0x58, 0x34, 0xe2, 0x03, 0x29, 0x5a, 0xbf, + 0xb6, 0xe0, 0x5d, 0x13, 0xd9, 0x2b, 0xb4, 0x80, 0xb2, 0x45, 0x81, 0x6a, 0x2e, 0x6c, 0x89, 0x7d, 0xee, 0xbb, 0x52, + 0xdd, 0x1f, 0x18, 0xe7, 0x13, 0x6b, 0x33, 0x0e, 0xea, 0x36, 0x92, 0x77, 0x7b, 0x6d, 0x9c, 0x5a, 0x5f, 0x45, 0x7b, + 0x7b, 0x35, 0x62, 0x23, 0xd1, 0xbf, 0x0f, 0xd0, 0x08, 0x1b, 0x2b, 0x80, 0x6b, 0x7e, 0xf1, 0x21, 0x47, 0xb0, 0x57, + 0xd1, 0x98, 0x72, 0x90, 0x34, 0x1c, 0x20, 0x04, 0xff, 0x3d, 0x5c, 0xee, 0x0e, 0x57, 0x5f, 0x6f, 0x24, 0x4e, 0x3c, + 0xea, 0xfc, 0xa5, 0xa9, 0x83, 0xc9, 0x61, 0xb4, 0x51, 0x24, 0xf8, 0x27, 0x5e, 0x46, 0x8c, 0xb1, 0x53, 0x02, 0x96, + 0x35, 0xba, 0xb8, 0x4c, 0x71, 0xd3, 0x15, 0x59, 0x35, 0x22, 0x20, 0xad, 0x03, 0x9f, 0x66, 0x44, 0x3b, 0x9c, 0x35, + 0x37, 0x1f, 0x9b, 0xbb, 0xf3, 0xdb, 0x35, 0x63, 0x30, 0x64, 0xaa, 0xa2, 0x06, 0xa8, 0x5d, 0xbb, 0xe1, 0x9f, 0x70, + 0xec, 0x82, 0x11, 0x06, 0x36, 0xec, 0x8b, 0x69, 0x66, 0x24, 0x44, 0xc9, 0x4a, 0x57, 0xbb, 0x9b, 0x78, 0x13, 0xce, + 0x9c, 0x0c, 0xba, 0x92, 0x93, 0x63, 0xb8, 0xe2, 0x95, 0x0f, 0x0f, 0x16, 0x39, 0x52, 0xfd, 0x3a, 0x6d, 0x02, 0x4b, + 0xdf, 0x13, 0xd3, 0x2a, 0x22, 0xb4, 0x03, 0x7c, 0x54, 0x49, 0x96, 0x68, 0x54, 0x10, 0xfa, 0xef, 0xaa, 0x6c, 0xe8, + 0x22, 0xdc, 0x71, 0x16, 0x13, 0x1a, 0xf6, 0x28, 0xe5, 0x6d, 0x77, 0x3d, 0xcd, 0x30, 0x63, 0xb1, 0x70, 0x52, 0xa1, + 0xc5, 0x94, 0x5f, 0xcf, 0xe8, 0xb8, 0x26, 0x98, 0xf7, 0x06, 0xa0, 0x0a, 0x70, 0xfa, 0x03, 0x80, 0xac, 0xc1, 0xec, + 0xd6, 0x4c, 0x54, 0xd7, 0xfe, 0x47, 0xb6, 0x88, 0x4a, 0xf7, 0x71, 0x24, 0xee, 0xf3, 0xd2, 0xc2, 0x4a, 0x7f, 0xfe, + 0x61, 0xc7, 0x35, 0xc9, 0x37, 0x67, 0xcb, 0x24, 0x35, 0xda, 0x7e, 0xca, 0x5f, 0xf3, 0x8d, 0xd4, 0x13, 0x8e, 0xd6, + 0xcb, 0x4d, 0x53, 0x8f, 0x53, 0x1f, 0xc0, 0x74, 0xf7, 0x53, 0xb9, 0x5e, 0x23, 0x37, 0xba, 0x6e, 0xe3, 0x9d, 0x07, + 0x55, 0x25, 0x7b, 0xe6, 0x2a, 0x64, 0xd1, 0x32, 0xdd, 0x54, 0x1b, 0x4b, 0xc0, 0xe1, 0xd7, 0x69, 0x58, 0xf8, 0x93, + 0x29, 0xc4, 0xdd, 0x23, 0x2f, 0xa5, 0xfc, 0x9d, 0x7e, 0xf8, 0xd4, 0x90, 0xcd, 0x82, 0x55, 0xdc, 0x16, 0x16, 0x9f, + 0x07, 0x52, 0x9b, 0x9d, 0x25, 0xed, 0x32, 0xc5, 0x7b, 0xdf, 0xf6, 0x83, 0x46, 0x3d, 0x65, 0xb7, 0xef, 0x87, 0x7a, + 0x12, 0x69, 0x8f, 0x06, 0x7c, 0x51, 0x15, 0x4a, 0x08, 0xe8, 0xac, 0x9a, 0x0c, 0x24, 0xa7, 0x27, 0xd8, 0x46, 0x2f, + 0xe7, 0x01, 0x0e, 0x1c, 0xc6, 0x91, 0xb0, 0x6e, 0x85, 0x65, 0xf0, 0x29, 0x0d, 0x2e, 0x6b, 0x3b, 0xfb, 0x4b, 0xdf, + 0xe4, 0x80, 0x93, 0x03, 0x66, 0x46, 0x3e, 0x8a, 0x6e, 0xf3, 0x5e, 0x4d, 0x62, 0x0e, 0x49, 0x05, 0xaf, 0xd4, 0xf8, + 0x21, 0x20, 0x61, 0x1d, 0x39, 0x17, 0xf4, 0x61, 0x47, 0x95, 0xfb, 0x15, 0x2e, 0xb3, 0x4f, 0xd0, 0x5d, 0xf5, 0x7d, + 0x40, 0xda, 0x90, 0x3c, 0x6b, 0xcb, 0x17, 0x00, 0x13, 0x3b, 0x64, 0x34, 0x1b, 0xf0, 0xf2, 0xe5, 0x3b, 0xb2, 0xc7, + 0xd3, 0x5f, 0x3a, 0x44, 0xa6, 0x9b, 0xb7, 0x78, 0x0e, 0x42, 0x5d, 0x4c, 0xc1, 0xe9, 0xd2, 0xcb, 0xb7, 0x78, 0xd1, + 0xfe, 0x9a, 0xb5, 0x07, 0xe9, 0xe0, 0xbe, 0xe2, 0x8a, 0xa7, 0x01, 0x83, 0x00, 0x8c, 0x5c, 0x08, 0xe6, 0x63, 0x12, + 0x92, 0xb7, 0xb7, 0xa6, 0x19, 0x7d, 0x38, 0x13, 0x38, 0x92, 0x87, 0x24, 0xf9, 0x48, 0xb3, 0x5e, 0x87, 0x6a, 0x40, + 0x39, 0x5c, 0x3f, 0xed, 0x8f, 0xee, 0xdb, 0x15, 0x82, 0x06, 0xda, 0x49, 0x21, 0x2b, 0xb5, 0xbf, 0x32, 0x7c, 0x9f, + 0x42, 0x28, 0x63, 0xcf, 0xaf, 0x1e, 0xf8, 0xc6, 0xa0, 0xd1, 0x02, 0x43, 0x57, 0x62, 0xec, 0x9b, 0x0f, 0x01, 0x9e, + 0x71, 0xd8, 0x87, 0x9d, 0x01, 0xc1, 0x58, 0x77, 0xd9, 0xaf, 0xb1, 0x10, 0x7e, 0xdd, 0xa6, 0x50, 0x96, 0xe5, 0xf0, + 0x72, 0x00, 0x6d, 0x4b, 0xf8, 0x2a, 0x8f, 0x19, 0xf3, 0x22, 0x88, 0x11, 0x4a, 0x8b, 0x7c, 0xfd, 0xb7, 0xed, 0xe1, + 0xf6, 0x40, 0x39, 0xe0, 0xe9, 0xf6, 0x3d, 0x25, 0xe6, 0x74, 0x3c, 0x58, 0x57, 0x7f, 0xe1, 0x22, 0x96, 0x47, 0x31, + 0x91, 0xba, 0x70, 0x85, 0x28, 0x6b, 0x9f, 0x6e, 0x25, 0xac, 0x23, 0x66, 0x2f, 0x29, 0x88, 0x28, 0xce, 0x8c, 0x5c, + 0x88, 0x53, 0xd1, 0x3b, 0xcc, 0x6a, 0x51, 0xb2, 0xe1, 0x28, 0x3f, 0x91, 0xb4, 0x0d, 0x00, 0x3a, 0xe3, 0xf8, 0xc3, + 0x8f, 0xd7, 0x96, 0x62, 0x0e, 0x2e, 0xfc, 0xc8, 0x6c, 0x77, 0xa6, 0x1d, 0x22, 0xc1, 0xb8, 0xe6, 0x61, 0xd7, 0x67, + 0x36, 0x13, 0x7b, 0xbb, 0x9b, 0x59, 0x09, 0xa6, 0xdf, 0xf7, 0x6b, 0xa3, 0x40, 0x1a, 0xf5, 0x4f, 0xb4, 0xda, 0xd3, + 0xf3, 0x81, 0x93, 0xc6, 0x18, 0xd9, 0x26, 0xee, 0xac, 0xf0, 0xaa, 0xdf, 0xc5, 0x9c, 0xca, 0xc2, 0xa2, 0xcc, 0x7b, + 0x5c, 0x24, 0xb0, 0xbc, 0xd0, 0x6a, 0x4d, 0x89, 0x09, 0xb8, 0x07, 0xfe, 0x87, 0xad, 0x0a, 0xea, 0xb8, 0x42, 0xf9, + 0x5e, 0xb3, 0x3e, 0x36, 0x4c, 0xaf, 0x75, 0x9e, 0x1c, 0xeb, 0xbd, 0xbc, 0xbb, 0x80, 0x40, 0xa7, 0x3a, 0x30, 0xbf, + 0xa8, 0x44, 0xf4, 0xeb, 0x38, 0xad, 0x29, 0xba, 0x23, 0xed, 0x41, 0x0c, 0xea, 0xd2, 0xbb, 0x41, 0x18, 0xd6, 0xb9, + 0xba, 0x65, 0x2b, 0xa3, 0x91, 0x6d, 0x1f, 0xa9, 0xf4, 0xd1, 0x25, 0x8d, 0x4d, 0x38, 0xff, 0x64, 0xa0, 0xec, 0xde, + 0xa6, 0xb6, 0x79, 0xab, 0x8e, 0x33, 0x6c, 0x47, 0xde, 0xaf, 0x94, 0xa4, 0xa5, 0x86, 0x77, 0x55, 0x09, 0x92, 0x81, + 0x31, 0x76, 0xc7, 0x34, 0x22, 0x89, 0x8e, 0x3d, 0x26, 0x26, 0xd7, 0xfc, 0x1e, 0x16, 0x72, 0x13, 0x33, 0x63, 0xd5, + 0x22, 0xbe, 0xb8, 0x04, 0x34, 0x84, 0x41, 0xbb, 0x80, 0xd0, 0x9f, 0x46, 0x48, 0x07, 0xa7, 0xfc, 0x2b, 0x3a, 0x75, + 0x55, 0x8c, 0xc7, 0x6a, 0xbd, 0x7e, 0x46, 0x08, 0x84, 0x0f, 0xd5, 0x74, 0xc0, 0x82, 0x8e, 0xaa, 0x61, 0x05, 0x01, + 0xb2, 0x47, 0x6e, 0x20, 0x6a, 0x2d, 0x58, 0x70, 0x48, 0x32, 0xa7, 0x37, 0xd2, 0xb8, 0x82, 0x1a, 0x51, 0xb9, 0x61, + 0xdd, 0xfd, 0x9d, 0x6b, 0x0e, 0x18, 0x97, 0xf8, 0x45, 0x5f, 0x87, 0x10, 0xcf, 0x34, 0x72, 0x45, 0x26, 0x49, 0x70, + 0xe7, 0xa3, 0x78, 0xe0, 0x52, 0x89, 0x84, 0x94, 0x83, 0x82, 0xc2, 0x69, 0x8f, 0xe3, 0xe1, 0x3f, 0x60, 0x74, 0x88, + 0xc4, 0xf7, 0x75, 0x2c, 0xfb, 0xbd, 0xb6, 0xc4, 0x7e, 0x10, 0x0a, 0x6c, 0x90, 0x04, 0x9e, 0xc3, 0x3f, 0x59, 0x7c, + 0xce, 0x31, 0x18, 0x60, 0x57, 0x73, 0x46, 0x94, 0x7d, 0x06, 0xa0, 0x6d, 0x44, 0xec, 0xa2, 0x0a, 0x9e, 0x05, 0x15, + 0xef, 0xca, 0x5c, 0xbf, 0x00, 0xeb, 0xf7, 0x3d, 0x32, 0xd4, 0xa5, 0xef, 0x49, 0x89, 0x5e, 0x46, 0xb0, 0xa6, 0x63, + 0x5b, 0x8a, 0x73, 0xae, 0x6f, 0xd5, 0x9d, 0xf8, 0x4f, 0x40, 0xb5, 0xb2, 0x6e, 0xd3, 0xb6, 0x01, 0xa9, 0x26, 0xa2, + 0x21, 0xcf, 0x33, 0x7a, 0x3a, 0xa4, 0x23, 0x13, 0xb0, 0x69, 0x6a, 0xee, 0xce, 0xd8, 0x9d, 0x01, 0x1d, 0x50, 0xc1, + 0x30, 0x6c, 0xb1, 0xcd, 0xa0, 0xf0, 0xf0, 0xa2, 0x64, 0x6f, 0xbb, 0xbf, 0x5e, 0xe6, 0xab, 0x87, 0xb4, 0x0f, 0x4f, + 0x15, 0xaf, 0xb5, 0x25, 0xa1, 0xb2, 0xd0, 0x80, 0x2c, 0xfb, 0xf9, 0xfe, 0xd2, 0x33, 0xbb, 0x76, 0xfe, 0x7c, 0xa8, + 0x66, 0xf7, 0xe7, 0x85, 0x9f, 0x1f, 0x85, 0x57, 0x88, 0xe1, 0xe9, 0x63, 0xe4, 0xd8, 0x1c, 0xa1, 0xfb, 0xda, 0x44, + 0x05, 0x2e, 0x1d, 0x3a, 0x1c, 0xff, 0xc8, 0x3b, 0xc0, 0xfe, 0xda, 0x22, 0x0b, 0x43, 0xd6, 0x88, 0x39, 0x4c, 0x4a, + 0xa6, 0x69, 0x18, 0x93, 0x42, 0x4e, 0xb5, 0xcc, 0x66, 0x0d, 0x09, 0xf8, 0x1e, 0x7c, 0xd3, 0x3c, 0x99, 0x0d, 0x50, + 0x1d, 0x62, 0xe9, 0x57, 0x06, 0xbf, 0x19, 0x88, 0xdd, 0xad, 0x7b, 0x4f, 0xf9, 0xc7, 0x82, 0x6d, 0x8d, 0xc8, 0xc4, + 0xc5, 0x78, 0x17, 0x20, 0x15, 0xc5, 0x52, 0x41, 0xcf, 0x5b, 0xd6, 0x7f, 0x94, 0x02, 0x41, 0xe0, 0x40, 0x22, 0x03, + 0x5e, 0xd1, 0x53, 0xd4, 0x86, 0xd3, 0x2c, 0x9f, 0x0f, 0x96, 0xe3, 0x6b, 0x9a, 0x76, 0x32, 0x06, 0x47, 0x4b, 0x11, + 0xb3, 0xdd, 0x03, 0x65, 0xbd, 0x9b, 0x01, 0xda, 0x9c, 0xb9, 0x7e, 0x3f, 0x6a, 0xc4, 0x7b, 0xea, 0xd4, 0x3c, 0xb9, + 0xfb, 0x5c, 0x6b, 0x64, 0x33, 0x52, 0xba, 0x64, 0x78, 0x8f, 0xa4, 0xaf, 0x7a, 0x61, 0x8d, 0xbc, 0xc5, 0x73, 0xe9, + 0x6b, 0x58, 0x97, 0x4b, 0xbf, 0x63, 0x22, 0xd3, 0x37, 0x02, 0x54, 0xc5, 0xb9, 0x16, 0x4a, 0xf0, 0x19, 0xd8, 0x94, + 0x57, 0xb8, 0x8a, 0xb3, 0x16, 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, 0x78, 0xec, 0x00 +}; +static CONST UINT8 EncOutput012[] = { + 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, 0xdf, 0x34, 0x56, + 0x82, 0xe2, 0xbe, 0xe5, 0xe1, 0x28, 0xd1, 0x4e, 0x5f, 0x4f, 0x01, 0x7d, 0x3f, 0x99, 0x6b, 0x30, 0x6e, 0x1a, 0x7c, + 0x4c, 0x8e, 0x62, 0x81, 0xae, 0x86, 0x3f, 0x6b, 0xd0, 0xb5, 0xa9, 0xcf, 0x50, 0xf1, 0x02, 0x12, 0xa0, 0x0b, 0x24, + 0xe9, 0xe6, 0x72, 0x89, 0x2c, 0x52, 0x1b, 0x34, 0x38, 0xf8, 0x75, 0x5f, 0xa0, 0x74, 0xe2, 0x99, 0xdd, 0xa6, 0x4b, + 0x14, 0x50, 0x4e, 0xf1, 0xbe, 0xd6, 0x9e, 0xdb, 0xb2, 0x24, 0x27, 0x74, 0x12, 0x4a, 0x78, 0x78, 0x17, 0xa5, 0x58, + 0x8e, 0x2f, 0xf9, 0xf4, 0x8d, 0xee, 0x03, 0x88, 0xae, 0xb8, 0x29, 0xa1, 0x2f, 0x4b, 0xee, 0x92, 0xbd, 0x87, 0xb3, + 0xce, 0x34, 0x21, 0x57, 0x46, 0x04, 0x49, 0x0c, 0x80, 0xf2, 0x01, 0x13, 0xa1, 0x55, 0xb3, 0xff, 0x44, 0x30, 0x3c, + 0x1c, 0xd0, 0xef, 0xbc, 0x18, 0x74, 0x26, 0xad, 0x41, 0x5b, 0x5b, 0x3e, 0x9a, 0x7a, 0x46, 0x4f, 0x16, 0xd6, 0x74, + 0x5a, 0xb7, 0x3a, 0x28, 0x31, 0xd8, 0xae, 0x26, 0xac, 0x50, 0x53, 0x86, 0xf2, 0x56, 0xd7, 0x3f, 0x29, 0xbc, 0x45, + 0x68, 0x8e, 0xcb, 0x98, 0x64, 0xdd, 0xc9, 0xba, 0xb8, 0x4b, 0x7b, 0x82, 0xdd, 0x14, 0xa7, 0xcb, 0x71, 0x72, 0x00, + 0x5c, 0xad, 0x7b, 0x6a, 0x89, 0xa4, 0x3d, 0xbf, 0xb5, 0x4b, 0x3e, 0x7c, 0x5a, 0xcf, 0xb8, 0xa1, 0xc5, 0x6e, 0xc8, + 0xb6, 0x31, 0x57, 0x7b, 0xdf, 0xa5, 0x7e, 0xb1, 0xd6, 0x42, 0x2a, 0x31, 0x36, 0xd1, 0xd0, 0x3f, 0x7a, 0xe5, 0x94, + 0xd6, 0x36, 0xa0, 0x6f, 0xb7, 0x40, 0x7d, 0x37, 0xc6, 0x55, 0x7c, 0x50, 0x40, 0x6d, 0x29, 0x89, 0xe3, 0x5a, 0xae, + 0x97, 0xe7, 0x44, 0x49, 0x6e, 0xbd, 0x81, 0x3d, 0x03, 0x93, 0x06, 0x12, 0x06, 0xe2, 0x41, 0x12, 0x4a, 0xf1, 0x6a, + 0xa4, 0x58, 0xa2, 0xfb, 0xd2, 0x15, 0xba, 0xc9, 0x79, 0xc9, 0xce, 0x5e, 0x13, 0xbb, 0xf1, 0x09, 0x04, 0xcc, 0xfd, + 0xe8, 0x51, 0x34, 0x6a, 0xe8, 0x61, 0x88, 0xda, 0xed, 0x01, 0x47, 0x84, 0xf5, 0x73, 0x25, 0xf9, 0x1c, 0x42, 0x86, + 0x07, 0xf3, 0x5b, 0x1a, 0x01, 0xb3, 0xeb, 0x24, 0x32, 0x8d, 0xf6, 0xed, 0x7c, 0x4b, 0xeb, 0x3c, 0x36, 0x42, 0x28, + 0xdf, 0xdf, 0xb6, 0xbe, 0xd9, 0x8c, 0x52, 0xd3, 0x2b, 0x08, 0x90, 0x8c, 0xe7, 0x98, 0x31, 0xe2, 0x32, 0x8e, 0xfc, + 0x11, 0x48, 0x00, 0xa8, 0x6a, 0x42, 0x4a, 0x02, 0xc6, 0x4b, 0x09, 0xf1, 0xe3, 0x49, 0xf3, 0x45, 0x1f, 0x0e, 0xbc, + 0x56, 0xe2, 0xe4, 0xdf, 0xfb, 0xeb, 0x61, 0xfa, 0x24, 0xc1, 0x63, 0x75, 0xbb, 0x47, 0x75, 0xaf, 0xe1, 0x53, 0x16, + 0x96, 0x21, 0x85, 0x26, 0x11, 0xb3, 0x76, 0xe3, 0x23, 0xa1, 0x6b, 0x74, 0x37, 0xd0, 0xde, 0x06, 0x90, 0x71, 0x5d, + 0x43, 0x88, 0x9b, 0x00, 0x54, 0xa6, 0x75, 0x2f, 0xa1, 0xc2, 0x0b, 0x73, 0x20, 0x1d, 0xb6, 0x21, 0x79, 0x57, 0x3f, + 0xfa, 0x09, 0xbe, 0x8a, 0x33, 0xc3, 0x52, 0xf0, 0x1d, 0x82, 0x31, 0xd1, 0x55, 0xb5, 0x6c, 0x99, 0x25, 0xcf, 0x5c, + 0x32, 0xce, 0xe9, 0x0d, 0xfa, 0x69, 0x2c, 0xd5, 0x0d, 0xc5, 0x6d, 0x86, 0xd0, 0x0c, 0x3b, 0x06, 0x50, 0x79, 0xe8, + 0xc3, 0xae, 0x04, 0xe6, 0xcd, 0x51, 0xe4, 0x26, 0x9b, 0x4f, 0x7e, 0xa6, 0x0f, 0xab, 0xd8, 0xe5, 0xde, 0xa9, 0x00, + 0x95, 0xbe, 0xa3, 0x9d, 0x5d, 0xb2, 0x09, 0x70, 0x18, 0x1c, 0xf0, 0xac, 0x29, 0x23, 0x02, 0x29, 0x28, 0xd2, 0x74, + 0x35, 0x57, 0x62, 0x0f, 0x24, 0xea, 0x5e, 0x33, 0xc2, 0x92, 0xf3, 0x78, 0x4d, 0x30, 0x1e, 0xa1, 0x99, 0xa9, 0x82, + 0xb0, 0x42, 0x31, 0x8d, 0xad, 0x8a, 0xbc, 0xfc, 0xd4, 0x57, 0x47, 0x3e, 0xb4, 0x50, 0xdd, 0x6e, 0x2c, 0x80, 0x4d, + 0x22, 0xf1, 0xfb, 0x57, 0xc4, 0xdd, 0x17, 0xe1, 0x8a, 0x36, 0x4a, 0xb3, 0x37, 0xca, 0xc9, 0x4e, 0xab, 0xd5, 0x69, + 0xc4, 0xf4, 0xbc, 0x0b, 0x3b, 0x44, 0x4b, 0x29, 0x9c, 0xee, 0xd4, 0x35, 0x22, 0x21, 0xb0, 0x1f, 0x27, 0x64, 0xa8, + 0x51, 0x1b, 0xf0, 0x9f, 0x19, 0x5c, 0xfb, 0x5a, 0x64, 0x74, 0x70, 0x45, 0x09, 0xf5, 0x64, 0xfe, 0x1a, 0x2d, 0xc9, + 0x14, 0x04, 0x14, 0xcf, 0xd5, 0x7d, 0x60, 0xaf, 0x94, 0x39, 0x94, 0xe2, 0x7d, 0x79, 0x82, 0xd0, 0x65, 0x3b, 0x6b, + 0x9c, 0x19, 0x84, 0xb4, 0x6d, 0xb3, 0x0c, 0x99, 0xc0, 0x56, 0xa8, 0xbd, 0x73, 0xce, 0x05, 0x84, 0x3e, 0x30, 0xaa, + 0xc4, 0x9b, 0x1b, 0x04, 0x2a, 0x9f, 0xd7, 0x43, 0x2b, 0x23, 0xdf, 0xbf, 0xaa, 0xd5, 0xc2, 0x43, 0x2d, 0x70, 0xab, + 0xdc, 0x75, 0xad, 0xac, 0xf7, 0xc0, 0xbe, 0x67, 0xb2, 0x74, 0xed, 0x67, 0x10, 0x4a, 0x92, 0x60, 0xc1, 0x40, 0x50, + 0x19, 0x8a, 0x8a, 0x8c, 0x09, 0x0e, 0x72, 0xe1, 0x73, 0x5e, 0xe8, 0x41, 0x85, 0x63, 0x9f, 0x3f, 0xd7, 0x7d, 0xc4, + 0xfb, 0x22, 0x5d, 0x92, 0x6c, 0xb3, 0x1e, 0xe2, 0x50, 0x2f, 0x82, 0xa8, 0x28, 0xc0, 0xb5, 0xd7, 0x5f, 0x68, 0x0d, + 0x2c, 0x2d, 0xaf, 0x7e, 0xfa, 0x2e, 0x08, 0x0f, 0x1f, 0x70, 0x9f, 0xe9, 0x19, 0x72, 0x55, 0xf8, 0xfb, 0x51, 0xd2, + 0x33, 0x5d, 0xa0, 0xd3, 0x2b, 0x0a, 0x6c, 0xbc, 0x4e, 0xcf, 0x36, 0x4d, 0xdc, 0x3b, 0xe9, 0x3e, 0x81, 0x7c, 0x61, + 0xdb, 0x20, 0x2d, 0x3a, 0xc3, 0xb3, 0x0c, 0x1e, 0x00, 0xb9, 0x7c, 0xf5, 0xca, 0x10, 0x5f, 0x3a, 0x71, 0xb3, 0xe4, + 0x20, 0xdb, 0x0c, 0x2a, 0x98, 0x63, 0x45, 0x00, 0x58, 0xf6, 0x68, 0xe4, 0x0b, 0xda, 0x13, 0x3b, 0x60, 0x5c, 0x76, + 0xdb, 0xb9, 0x97, 0x71, 0xe4, 0xd9, 0xb7, 0xdb, 0xbd, 0x68, 0xc7, 0x84, 0x84, 0xaa, 0x7c, 0x68, 0x62, 0x5e, 0x16, + 0xfc, 0xba, 0x72, 0xaa, 0x9a, 0xa9, 0xeb, 0x7c, 0x75, 0x47, 0x97, 0x7e, 0xad, 0xe2, 0xd9, 0x91, 0xe8, 0xe4, 0xa5, + 0x31, 0xd7, 0x01, 0x8e, 0xa2, 0x11, 0x88, 0x95, 0xb9, 0xf2, 0x9b, 0xd3, 0x7f, 0x1b, 0x81, 0x22, 0xf7, 0x98, 0x60, + 0x0a, 0x64, 0xa6, 0xc1, 0xf6, 0x49, 0xc7, 0xe3, 0x07, 0x4d, 0x94, 0x7a, 0xcf, 0x6e, 0x68, 0x0c, 0x1b, 0x3f, 0x6e, + 0x2e, 0xee, 0x92, 0xfa, 0x52, 0xb3, 0x59, 0xf8, 0xf1, 0x8f, 0x6a, 0x66, 0xa3, 0x82, 0x76, 0x4a, 0x07, 0x1a, 0xc7, + 0xdd, 0xf5, 0xda, 0x9c, 0x3c, 0x24, 0xbf, 0xfd, 0x42, 0xa1, 0x10, 0x64, 0x6a, 0x0f, 0x89, 0xee, 0x36, 0xa5, 0xce, + 0x99, 0x48, 0x6a, 0xf0, 0x9f, 0x9e, 0x69, 0xa4, 0x40, 0x20, 0xe9, 0x16, 0x15, 0xf7, 0xdb, 0x75, 0x02, 0xcb, 0xe9, + 0x73, 0x8b, 0x3b, 0x49, 0x2f, 0xf0, 0xaf, 0x51, 0x06, 0x5c, 0xdf, 0x27, 0x27, 0x49, 0x6a, 0xd1, 0xcc, 0xc7, 0xb5, + 0x63, 0xb5, 0xfc, 0xb8, 0x5c, 0x87, 0x7f, 0x84, 0xb4, 0xcc, 0x14, 0xa9, 0x53, 0xda, 0xa4, 0x56, 0xf8, 0xb6, 0x1b, + 0xcc, 0x40, 0x27, 0x52, 0x06, 0x5a, 0x13, 0x81, 0xd7, 0x3a, 0xd4, 0x3b, 0xfb, 0x49, 0x65, 0x31, 0x33, 0xb2, 0xfa, + 0xcd, 0xad, 0x58, 0x4e, 0x2b, 0xae, 0xd2, 0x20, 0xfb, 0x1a, 0x48, 0xb4, 0x3f, 0x9a, 0xd8, 0x7a, 0x35, 0x4a, 0xc8, + 0xee, 0x88, 0x5e, 0x07, 0x66, 0x54, 0xb9, 0xec, 0x9f, 0xa3, 0xe3, 0xb9, 0x37, 0xaa, 0x49, 0x76, 0x31, 0xda, 0x74, + 0x2d, 0x3c, 0xa4, 0x65, 0x10, 0x32, 0x38, 0xf0, 0xde, 0xd3, 0x99, 0x17, 0xaa, 0x71, 0xaa, 0x8f, 0x0f, 0x8c, 0xaf, + 0xa2, 0xf8, 0x5d, 0x64, 0xba, 0x1d, 0xa3, 0xef, 0x96, 0x73, 0xe8, 0xa1, 0x02, 0x8d, 0x0c, 0x6d, 0xb8, 0x06, 0x90, + 0xb8, 0x08, 0x56, 0x2c, 0xa7, 0x06, 0xc9, 0xc2, 0x38, 0xdb, 0x7c, 0x63, 0xb1, 0x57, 0x8e, 0xea, 0x7c, 0x79, 0xf3, + 0x49, 0x1d, 0xfe, 0x9f, 0xf3, 0x6e, 0xb1, 0x1d, 0xba, 0x19, 0x80, 0x1a, 0x0a, 0xd3, 0xb0, 0x26, 0x21, 0x40, 0xb1, + 0x7c, 0xf9, 0x4d, 0x8d, 0x10, 0xc1, 0x7e, 0xf4, 0xf6, 0x3c, 0xa8, 0xfd, 0x7c, 0xa3, 0x92, 0xb2, 0x0f, 0xaa, 0xcc, + 0xa6, 0x11, 0xfe, 0x04, 0xe3, 0xd1, 0x7a, 0x32, 0x89, 0xdf, 0x0d, 0xc4, 0x8f, 0x79, 0x6b, 0xca, 0x16, 0x7c, 0x6e, + 0xf9, 0xad, 0x0f, 0xf6, 0xfe, 0x27, 0xdb, 0xc4, 0x13, 0x70, 0xf1, 0x62, 0x1a, 0x4f, 0x79, 0x40, 0xc9, 0x9b, 0x8b, + 0x21, 0xea, 0x84, 0xfa, 0xf5, 0xf1, 0x89, 0xce, 0xb7, 0x55, 0x0a, 0x80, 0x39, 0x2f, 0x55, 0x36, 0x16, 0x9c, 0x7b, + 0x08, 0xbd, 0x87, 0x0d, 0xa5, 0x32, 0xf1, 0x52, 0x7c, 0xe8, 0x55, 0x60, 0x5b, 0xd7, 0x69, 0xe4, 0xfc, 0xfa, 0x12, + 0x85, 0x96, 0xea, 0x50, 0x28, 0xab, 0x8a, 0xf7, 0xbb, 0x0e, 0x53, 0x74, 0xca, 0xa6, 0x27, 0x09, 0xc2, 0xb5, 0xde, + 0x18, 0x14, 0xd9, 0xea, 0xe5, 0x29, 0x1c, 0x40, 0x56, 0xcf, 0xd7, 0xae, 0x05, 0x3f, 0x65, 0xaf, 0x05, 0x73, 0xe2, + 0x35, 0x96, 0x27, 0x07, 0x14, 0xc0, 0xad, 0x33, 0xf1, 0xdc, 0x44, 0x7a, 0x89, 0x17, 0x77, 0xd2, 0x9c, 0x58, 0x60, + 0xf0, 0x3f, 0x7b, 0x2d, 0x2e, 0x57, 0x95, 0x54, 0x87, 0xed, 0xf2, 0xc7, 0x4c, 0xf0, 0xae, 0x56, 0x29, 0x19, 0x7d, + 0x66, 0x4b, 0x9b, 0x83, 0x84, 0x42, 0x3b, 0x01, 0x25, 0x66, 0x8e, 0x02, 0xde, 0xb9, 0x83, 0x54, 0x19, 0xf6, 0x9f, + 0x79, 0x0d, 0x67, 0xc5, 0x1d, 0x7a, 0x44, 0x02, 0x98, 0xa7, 0x16, 0x1c, 0x29, 0x0d, 0x74, 0xff, 0x85, 0x40, 0x06, + 0xef, 0x2c, 0xa9, 0xc6, 0xf5, 0x53, 0x07, 0x06, 0xae, 0xe4, 0xfa, 0x5f, 0xd8, 0x39, 0x4d, 0xf1, 0x9b, 0x6b, 0xd9, + 0x24, 0x84, 0xfe, 0x03, 0x4c, 0xb2, 0x3f, 0xdf, 0xa1, 0x05, 0x9e, 0x50, 0x14, 0x5a, 0xd9, 0x1a, 0xa2, 0xa7, 0xfa, + 0xfa, 0x17, 0xf7, 0x78, 0xd6, 0xb5, 0x92, 0x61, 0x91, 0xac, 0x36, 0xfa, 0x56, 0x0d, 0x38, 0x32, 0x18, 0x85, 0x08, + 0x58, 0x37, 0xf0, 0x4b, 0xdb, 0x59, 0xe7, 0xa4, 0x34, 0xc0, 0x1b, 0x01, 0xaf, 0x2d, 0xde, 0xa1, 0xaa, 0x5d, 0xd3, + 0xec, 0xe1, 0xd4, 0xf7, 0xe6, 0x54, 0x68, 0xf0, 0x51, 0x97, 0xa7, 0x89, 0xea, 0x24, 0xad, 0xd3, 0x6e, 0x47, 0x93, + 0x8b, 0x4b, 0xb4, 0xf7, 0x1c, 0x42, 0x06, 0x67, 0xe8, 0x99, 0xf6, 0xf5, 0x7b, 0x85, 0xb5, 0x65, 0xb5, 0xb5, 0xd2, + 0x37, 0xf5, 0xf3, 0x02, 0xa6, 0x4d, 0x11, 0xa7, 0xdc, 0x51, 0x09, 0x7f, 0xa0, 0xd8, 0x88, 0x1c, 0x13, 0x71, 0xae, + 0x9c, 0xb7, 0x7b, 0x34, 0xd6, 0x4e, 0x68, 0x26, 0x83, 0x51, 0xaf, 0x1d, 0xee, 0x8b, 0xbb, 0x69, 0x43, 0x2b, 0x9e, + 0x8a, 0xbc, 0x02, 0x0e, 0xa0, 0x1b, 0xe0, 0xa8, 0x5f, 0x6f, 0xaf, 0x1b, 0x8f, 0xe7, 0x64, 0x71, 0x74, 0x11, 0x7e, + 0xa8, 0xd8, 0xf9, 0x97, 0x06, 0xc3, 0xb6, 0xfb, 0xfb, 0xb7, 0x3d, 0x35, 0x9d, 0x3b, 0x52, 0xed, 0x54, 0xca, 0xf4, + 0x81, 0x01, 0x2d, 0x1b, 0xc3, 0xa7, 0x00, 0x3d, 0x1a, 0x39, 0x54, 0xe1, 0xf6, 0xff, 0xed, 0x6f, 0x0b, 0x5a, 0x68, + 0xda, 0x58, 0xdd, 0xa9, 0xcf, 0x5c, 0x4a, 0xe5, 0x09, 0x4e, 0xde, 0x9d, 0xbc, 0x3e, 0xee, 0x5a, 0x00, 0x3b, 0x2c, + 0x87, 0x10, 0x65, 0x60, 0xdd, 0xd7, 0x56, 0xd1, 0x4c, 0x64, 0x45, 0xe4, 0x21, 0xec, 0x78, 0xf8, 0x25, 0x7a, 0x3e, + 0x16, 0x5d, 0x09, 0x53, 0x14, 0xbe, 0x4f, 0xae, 0x87, 0xd8, 0xd1, 0xaa, 0x3c, 0xf6, 0x3e, 0xa4, 0x70, 0x8c, 0x5e, + 0x70, 0xa4, 0xb3, 0x6b, 0x66, 0x73, 0xd3, 0xbf, 0x31, 0x06, 0x19, 0x62, 0x93, 0x15, 0xf2, 0x86, 0xe4, 0x52, 0x7e, + 0x53, 0x4c, 0x12, 0x38, 0xcc, 0x34, 0x7d, 0x57, 0xf6, 0x42, 0x93, 0x8a, 0xc4, 0xee, 0x5c, 0x8a, 0xe1, 0x52, 0x8f, + 0x56, 0x64, 0xf6, 0xa6, 0xd1, 0x91, 0x57, 0x70, 0xcd, 0x11, 0x76, 0xf5, 0x59, 0x60, 0x60, 0x3c, 0xc1, 0xc3, 0x0b, + 0x7f, 0x58, 0x1a, 0x50, 0x91, 0xf1, 0x68, 0x8f, 0x6e, 0x74, 0x74, 0xa8, 0x51, 0x0b, 0xf7, 0x7a, 0x98, 0x37, 0xf2, + 0x0a, 0x0e, 0xa4, 0x97, 0x04, 0xb8, 0x9b, 0xfd, 0xa0, 0xea, 0xf7, 0x0d, 0xe1, 0xdb, 0x03, 0xf0, 0x31, 0x29, 0xf8, + 0xdd, 0x6b, 0x8b, 0x5d, 0xd8, 0x59, 0xa9, 0x29, 0xcf, 0x9a, 0x79, 0x89, 0x19, 0x63, 0x46, 0x09, 0x79, 0x6a, 0x11, + 0xda, 0x63, 0x68, 0x48, 0x77, 0x23, 0xfb, 0x7d, 0x3a, 0x43, 0xcb, 0x02, 0x3b, 0x7a, 0x6d, 0x10, 0x2a, 0x9e, 0xac, + 0xf1, 0xd4, 0x19, 0xf8, 0x23, 0x64, 0x1d, 0x2c, 0x5f, 0xf2, 0xb0, 0x5c, 0x23, 0x27, 0xf7, 0x27, 0x30, 0x16, 0x37, + 0xb1, 0x90, 0xab, 0x38, 0xfb, 0x55, 0xcd, 0x78, 0x58, 0xd4, 0x7d, 0x43, 0xf6, 0x45, 0x5e, 0x55, 0x8d, 0xb1, 0x02, + 0x65, 0x58, 0xb4, 0x13, 0x4b, 0x36, 0xf7, 0xcc, 0xfe, 0x3d, 0x0b, 0x82, 0xe2, 0x12, 0x11, 0xbb, 0xe6, 0xb8, 0x3a, + 0x48, 0x71, 0xc7, 0x50, 0x06, 0x16, 0x3a, 0xe6, 0x7c, 0x05, 0xc7, 0xc8, 0x4d, 0x2f, 0x08, 0x6a, 0x17, 0x9a, 0x95, + 0x97, 0x50, 0x68, 0xdc, 0x28, 0x18, 0xc4, 0x61, 0x38, 0xb9, 0xe0, 0x3e, 0x78, 0xdb, 0x29, 0xe0, 0x9f, 0x52, 0xdd, + 0xf8, 0x4f, 0x91, 0xc1, 0xd0, 0x33, 0xa1, 0x7a, 0x8e, 0x30, 0x13, 0x82, 0x07, 0x9f, 0xd3, 0x31, 0x0f, 0x23, 0xbe, + 0x32, 0x5a, 0x75, 0xcf, 0x96, 0xb2, 0xec, 0xb5, 0x32, 0xac, 0x21, 0xd1, 0x82, 0x33, 0xd3, 0x15, 0x74, 0xbd, 0x90, + 0xf1, 0x2c, 0xe6, 0x5f, 0x8d, 0xe3, 0x02, 0xe8, 0xe9, 0xc4, 0xca, 0x96, 0xeb, 0x0e, 0xbc, 0x91, 0xf4, 0xb9, 0xea, + 0xd9, 0x1b, 0x75, 0xbd, 0xe1, 0xac, 0x2a, 0x05, 0x37, 0x52, 0x9b, 0x1b, 0x3f, 0x5a, 0xdc, 0x21, 0xc3, 0x98, 0xbb, + 0xaf, 0xa3, 0xf2, 0x00, 0xbf, 0x0d, 0x30, 0x89, 0x05, 0xcc, 0xa5, 0x76, 0xf5, 0x06, 0xf0, 0xc6, 0x54, 0x8a, 0x5d, + 0xd4, 0x1e, 0xc1, 0xf2, 0xce, 0xb0, 0x62, 0xc8, 0xfc, 0x59, 0x42, 0x9a, 0x90, 0x60, 0x55, 0xfe, 0x88, 0xa5, 0x8b, + 0xb8, 0x33, 0x0c, 0x23, 0x24, 0x0d, 0x15, 0x70, 0x37, 0x1e, 0x3d, 0xf6, 0xd2, 0xea, 0x92, 0x10, 0xb2, 0xc4, 0x51, + 0xac, 0xf2, 0xac, 0xf3, 0x6b, 0x6c, 0xaa, 0xcf, 0x12, 0xc5, 0x6c, 0x90, 0x50, 0xb5, 0x0c, 0xfc, 0x1a, 0x15, 0x52, + 0xe9, 0x26, 0xc6, 0x52, 0xa4, 0xe7, 0x81, 0x69, 0xe1, 0xe7, 0x9e, 0x30, 0x01, 0xec, 0x84, 0x89, 0xb2, 0x0d, 0x66, + 0xdd, 0xce, 0x28, 0x5c, 0xec, 0x98, 0x46, 0x68, 0x21, 0x9f, 0x88, 0x3f, 0x1f, 0x42, 0x77, 0xce, 0xd0, 0x61, 0xd4, + 0x20, 0xa7, 0xff, 0x53, 0xad, 0x37, 0xd0, 0x17, 0x35, 0xc9, 0xfc, 0xba, 0x0a, 0x78, 0x3f, 0xf2, 0xcc, 0x86, 0x89, + 0xe8, 0x4b, 0x3c, 0x48, 0x33, 0x09, 0x7f, 0xc6, 0xc0, 0xdd, 0xb8, 0xfd, 0x7a, 0x66, 0x66, 0x65, 0xeb, 0x47, 0xa7, + 0x04, 0x28, 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, 0x70, 0xcf, 0xd6 +}; +static CONST UINT8 EncAssoc012[] = { 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, 0x77, 0xe8, 0x21, 0xff, 0x06, + 0x59, 0x35, 0xce, 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, 0xcf, 0x70, + 0xdd, 0x0a, 0x68, 0xbf, 0xd4, 0xbc, 0x16, 0x76, 0x99, 0x36, 0x1e, 0x58, 0x79, + 0x5e, 0xd4, 0x29, 0xf7, 0x33, 0x93, 0x48, 0xdb, 0x5f, 0x01, 0xae, 0x9c, 0xb6, + 0xe4, 0x88, 0x6d, 0x2b, 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 }; +static CONST UINT8 EncNonce012[] = { 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 }; +static CONST UINT8 EncKey012[] = { 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, 0x8f, 0x59, 0x8e, + 0xc5, 0x90, 0xd5, 0x27, 0x2d, 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, + 0x44, 0x1e, 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 }; + +/* wycheproof - rfc7539 */ +static CONST UINT8 EncInput013[] = { + 0x4c, 0x61, 0x64, 0x69, 0x65, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x47, 0x65, 0x6e, 0x74, 0x6c, 0x65, 0x6d, 0x65, + 0x6e, 0x20, 0x6f, 0x66, 0x20, 0x74, 0x68, 0x65, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x20, 0x6f, 0x66, 0x20, 0x27, + 0x39, 0x39, 0x3a, 0x20, 0x49, 0x66, 0x20, 0x49, 0x20, 0x63, 0x6f, 0x75, 0x6c, 0x64, 0x20, 0x6f, 0x66, 0x66, 0x65, + 0x72, 0x20, 0x79, 0x6f, 0x75, 0x20, 0x6f, 0x6e, 0x6c, 0x79, 0x20, 0x6f, 0x6e, 0x65, 0x20, 0x74, 0x69, 0x70, 0x20, + 0x66, 0x6f, 0x72, 0x20, 0x74, 0x68, 0x65, 0x20, 0x66, 0x75, 0x74, 0x75, 0x72, 0x65, 0x2c, 0x20, 0x73, 0x75, 0x6e, + 0x73, 0x63, 0x72, 0x65, 0x65, 0x6e, 0x20, 0x77, 0x6f, 0x75, 0x6c, 0x64, 0x20, 0x62, 0x65, 0x20, 0x69, 0x74, 0x2e +}; +static CONST UINT8 EncOutput013[] = { + 0xd3, 0x1a, 0x8d, 0x34, 0x64, 0x8e, 0x60, 0xdb, 0x7b, 0x86, 0xaf, 0xbc, 0x53, 0xef, 0x7e, 0xc2, 0xa4, 0xad, 0xed, + 0x51, 0x29, 0x6e, 0x08, 0xfe, 0xa9, 0xe2, 0xb5, 0xa7, 0x36, 0xee, 0x62, 0xd6, 0x3d, 0xbe, 0xa4, 0x5e, 0x8c, 0xa9, + 0x67, 0x12, 0x82, 0xfa, 0xfb, 0x69, 0xda, 0x92, 0x72, 0x8b, 0x1a, 0x71, 0xde, 0x0a, 0x9e, 0x06, 0x0b, 0x29, 0x05, + 0xd6, 0xa5, 0xb6, 0x7e, 0xcd, 0x3b, 0x36, 0x92, 0xdd, 0xbd, 0x7f, 0x2d, 0x77, 0x8b, 0x8c, 0x98, 0x03, 0xae, 0xe3, + 0x28, 0x09, 0x1b, 0x58, 0xfa, 0xb3, 0x24, 0xe4, 0xfa, 0xd6, 0x75, 0x94, 0x55, 0x85, 0x80, 0x8b, 0x48, 0x31, 0xd7, + 0xbc, 0x3f, 0xf4, 0xde, 0xf0, 0x8e, 0x4b, 0x7a, 0x9d, 0xe5, 0x76, 0xd2, 0x65, 0x86, 0xce, 0xc6, 0x4b, 0x61, 0x16, + 0x1a, 0xe1, 0x0b, 0x59, 0x4f, 0x09, 0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60, 0x06, 0x91 +}; +static CONST UINT8 EncAssoc013[] = { 0x50, 0x51, 0x52, 0x53, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7 }; +static CONST UINT8 EncNonce013[] = { 0x07, 0x00, 0x00, 0x00, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47 }; +static CONST UINT8 EncKey013[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +// static CONST UINT8 EncInput014[] = { }; +static CONST UINT8 EncOutput014[] = { 0x76, 0xac, 0xb3, 0x42, 0xcf, 0x31, 0x66, 0xa5, + 0xb6, 0x3c, 0x0c, 0x0e, 0xa1, 0x38, 0x3c, 0x8d }; +// static CONST UINT8 EncAssoc014[] = { }; +static CONST UINT8 EncNonce014[] = { 0x4d, 0xa5, 0xbf, 0x8d, 0xfd, 0x58, 0x52, 0xc1, 0xea, 0x12, 0x37, 0x9d }; +static CONST UINT8 EncKey014[] = { 0x80, 0xba, 0x31, 0x92, 0xc8, 0x03, 0xce, 0x96, 0x5e, 0xa3, 0x71, + 0xd5, 0xff, 0x07, 0x3c, 0xf0, 0xf4, 0x3b, 0x6a, 0x2a, 0xb5, 0x76, + 0xb2, 0x08, 0x42, 0x6e, 0x11, 0x40, 0x9c, 0x09, 0xb9, 0xb0 }; + +/* wycheproof - misc */ +// static CONST UINT8 EncInput015[] = { }; +static CONST UINT8 EncOutput015[] = { 0x90, 0x6f, 0xa6, 0x28, 0x4b, 0x52, 0xf8, 0x7b, + 0x73, 0x59, 0xcb, 0xaa, 0x75, 0x63, 0xc7, 0x09 }; +static CONST UINT8 EncAssoc015[] = { 0xbd, 0x50, 0x67, 0x64, 0xf2, 0xd2, 0xc4, 0x10 }; +static CONST UINT8 EncNonce015[] = { 0xa9, 0x2e, 0xf0, 0xac, 0x99, 0x1d, 0xd5, 0x16, 0xa3, 0xc6, 0xf6, 0x89 }; +static CONST UINT8 EncKey015[] = { 0x7a, 0x4c, 0xd7, 0x59, 0x17, 0x2e, 0x02, 0xeb, 0x20, 0x4d, 0xb2, + 0xc3, 0xf5, 0xc7, 0x46, 0x22, 0x7d, 0xf5, 0x84, 0xfc, 0x13, 0x45, + 0x19, 0x63, 0x91, 0xdb, 0xb9, 0x57, 0x7a, 0x25, 0x07, 0x42 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput016[] = { 0x2a }; +static CONST UINT8 EncOutput016[] = { 0x3a, 0xca, 0xc2, 0x7d, 0xec, 0x09, 0x68, 0x80, 0x1e, + 0x9f, 0x6e, 0xde, 0xd6, 0x9d, 0x80, 0x75, 0x22 }; +// static CONST UINT8 EncAssoc016[] = { }; +static CONST UINT8 EncNonce016[] = { 0x99, 0xe2, 0x3e, 0xc4, 0x89, 0x85, 0xbc, 0xcd, 0xee, 0xab, 0x60, 0xf1 }; +static CONST UINT8 EncKey016[] = { 0xcc, 0x56, 0xb6, 0x80, 0x55, 0x2e, 0xb7, 0x50, 0x08, 0xf5, 0x48, + 0x4b, 0x4c, 0xb8, 0x03, 0xfa, 0x50, 0x63, 0xeb, 0xd6, 0xea, 0xb9, + 0x1f, 0x6a, 0xb6, 0xae, 0xf4, 0x91, 0x6a, 0x76, 0x62, 0x73 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput017[] = { 0x51 }; +static CONST UINT8 EncOutput017[] = { 0xc4, 0x16, 0x83, 0x10, 0xca, 0x45, 0xb1, 0xf7, 0xc6, + 0x6c, 0xad, 0x4e, 0x99, 0xe4, 0x3f, 0x72, 0xb9 }; +static CONST UINT8 EncAssoc017[] = { 0x91, 0xca, 0x6c, 0x59, 0x2c, 0xbc, 0xca, 0x53 }; +static CONST UINT8 EncNonce017[] = { 0xab, 0x0d, 0xca, 0x71, 0x6e, 0xe0, 0x51, 0xd2, 0x78, 0x2f, 0x44, 0x03 }; +static CONST UINT8 EncKey017[] = { 0x46, 0xf0, 0x25, 0x49, 0x65, 0xf7, 0x69, 0xd5, 0x2b, 0xdb, 0x4a, + 0x70, 0xb4, 0x43, 0x19, 0x9f, 0x8e, 0xf2, 0x07, 0x52, 0x0d, 0x12, + 0x20, 0xc5, 0x5e, 0x4b, 0x70, 0xf0, 0xfd, 0xa6, 0x20, 0xee }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput018[] = { 0x5c, 0x60 }; +static CONST UINT8 EncOutput018[] = { 0x4d, 0x13, 0x91, 0xe8, 0xb6, 0x1e, 0xfb, 0x39, 0xc1, + 0x22, 0x19, 0x54, 0x53, 0x07, 0x7b, 0x22, 0xe5, 0xe2 }; +// static CONST UINT8 EncAssoc018[] = { }; +static CONST UINT8 EncNonce018[] = { 0x46, 0x1a, 0xf1, 0x22, 0xe9, 0xf2, 0xe0, 0x34, 0x7e, 0x03, 0xf2, 0xdb }; +static CONST UINT8 EncKey018[] = { 0x2f, 0x7f, 0x7e, 0x4f, 0x59, 0x2b, 0xb3, 0x89, 0x19, 0x49, 0x89, + 0x74, 0x35, 0x07, 0xbf, 0x3e, 0xe9, 0xcb, 0xde, 0x17, 0x86, 0xb6, + 0x69, 0x5f, 0xe6, 0xc0, 0x25, 0xfd, 0x9b, 0xa4, 0xc1, 0x00 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput019[] = { 0xdd, 0xf2 }; +static CONST UINT8 EncOutput019[] = { 0xb6, 0x0d, 0xea, 0xd0, 0xfd, 0x46, 0x97, 0xec, 0x2e, + 0x55, 0x58, 0x23, 0x77, 0x19, 0xd0, 0x24, 0x37, 0xa2 }; +static CONST UINT8 EncAssoc019[] = { 0x88, 0x36, 0x4f, 0xc8, 0x06, 0x05, 0x18, 0xbf }; +static CONST UINT8 EncNonce019[] = { 0x61, 0x54, 0x6b, 0xa5, 0xf1, 0x72, 0x05, 0x90, 0xb6, 0x04, 0x0a, 0xc6 }; +static CONST UINT8 EncKey019[] = { 0xc8, 0x83, 0x3d, 0xce, 0x5e, 0xa9, 0xf2, 0x48, 0xaa, 0x20, 0x30, + 0xea, 0xcf, 0xe7, 0x2b, 0xff, 0xe6, 0x9a, 0x62, 0x0c, 0xaf, 0x79, + 0x33, 0x44, 0xe5, 0x71, 0x8f, 0xe0, 0xd7, 0xab, 0x1a, 0x58 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput020[] = { 0xab, 0x85, 0xe9, 0xc1, 0x57, 0x17, 0x31 }; +static CONST UINT8 EncOutput020[] = { 0x5d, 0xfe, 0x34, 0x40, 0xdb, 0xb3, 0xc3, 0xed, 0x7a, 0x43, 0x4e, 0x26, + 0x02, 0xd3, 0x94, 0x28, 0x1e, 0x0a, 0xfa, 0x9f, 0xb7, 0xaa, 0x42 }; +// static CONST UINT8 EncAssoc020[] = { }; +static CONST UINT8 EncNonce020[] = { 0x3c, 0x4e, 0x65, 0x4d, 0x66, 0x3f, 0xa4, 0x59, 0x6d, 0xc5, 0x5b, 0xb7 }; +static CONST UINT8 EncKey020[] = { 0x55, 0x56, 0x81, 0x58, 0xd3, 0xa6, 0x48, 0x3f, 0x1f, 0x70, 0x21, + 0xea, 0xb6, 0x9b, 0x70, 0x3f, 0x61, 0x42, 0x51, 0xca, 0xdc, 0x1a, + 0xf5, 0xd3, 0x4a, 0x37, 0x4f, 0xdb, 0xfc, 0x5a, 0xda, 0xc7 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput021[] = { 0x4e, 0xe5, 0xcd, 0xa2, 0x0d, 0x42, 0x90 }; +static CONST UINT8 EncOutput021[] = { 0x4b, 0xd4, 0x72, 0x12, 0x94, 0x1c, 0xe3, 0x18, 0x5f, 0x14, 0x08, 0xee, + 0x7f, 0xbf, 0x18, 0xf5, 0xab, 0xad, 0x6e, 0x22, 0x53, 0xa1, 0xba }; +static CONST UINT8 EncAssoc021[] = { 0x84, 0xe4, 0x6b, 0xe8, 0xc0, 0x91, 0x90, 0x53 }; +static CONST UINT8 EncNonce021[] = { 0x58, 0x38, 0x93, 0x75, 0xc6, 0x9e, 0xe3, 0x98, 0xde, 0x94, 0x83, 0x96 }; +static CONST UINT8 EncKey021[] = { 0xe3, 0xc0, 0x9e, 0x7f, 0xab, 0x1a, 0xef, 0xb5, 0x16, 0xda, 0x6a, + 0x33, 0x02, 0x2a, 0x1d, 0xd4, 0xeb, 0x27, 0x2c, 0x80, 0xd5, 0x40, + 0xc5, 0xda, 0x52, 0xa7, 0x30, 0xf3, 0x4d, 0x84, 0x0d, 0x7f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput022[] = { 0xbe, 0x33, 0x08, 0xf7, 0x2a, 0x2c, 0x6a, 0xed }; +static CONST UINT8 EncOutput022[] = { 0x8e, 0x94, 0x39, 0xa5, 0x6e, 0xee, 0xc8, 0x17, 0xfb, 0xe8, 0xa6, 0xed, + 0x8f, 0xab, 0xb1, 0x93, 0x75, 0x39, 0xdd, 0x6c, 0x00, 0xe9, 0x00, 0x21 }; +// static CONST UINT8 EncAssoc022[] = { }; +static CONST UINT8 EncNonce022[] = { 0x4f, 0x07, 0xaf, 0xed, 0xfd, 0xc3, 0xb6, 0xc2, 0x36, 0x18, 0x23, 0xd3 }; +static CONST UINT8 EncKey022[] = { 0x51, 0xe4, 0xbf, 0x2b, 0xad, 0x92, 0xb7, 0xaf, 0xf1, 0xa4, 0xbc, + 0x05, 0x55, 0x0b, 0xa8, 0x1d, 0xf4, 0xb9, 0x6f, 0xab, 0xf4, 0x1c, + 0x12, 0xc7, 0xb0, 0x0e, 0x60, 0xe4, 0x8d, 0xb7, 0xe1, 0x52 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput023[] = { 0xa4, 0xc9, 0xc2, 0x80, 0x1b, 0x71, 0xf7, 0xdf }; +static CONST UINT8 EncOutput023[] = { 0xb9, 0xb9, 0x10, 0x43, 0x3a, 0xf0, 0x52, 0xb0, 0x45, 0x30, 0xf5, 0x1a, + 0xee, 0xe0, 0x24, 0xe0, 0xa4, 0x45, 0xa6, 0x32, 0x8f, 0xa6, 0x7a, 0x18 }; +static CONST UINT8 EncAssoc023[] = { 0x66, 0xc0, 0xae, 0x70, 0x07, 0x6c, 0xb1, 0x4d }; +static CONST UINT8 EncNonce023[] = { 0xb4, 0xea, 0x66, 0x6e, 0xe1, 0x19, 0x56, 0x33, 0x66, 0x48, 0x4a, 0x78 }; +static CONST UINT8 EncKey023[] = { 0x11, 0x31, 0xc1, 0x41, 0x85, 0x77, 0xa0, 0x54, 0xde, 0x7a, 0x4a, + 0xc5, 0x51, 0x95, 0x0f, 0x1a, 0x05, 0x3f, 0x9a, 0xe4, 0x6e, 0x5b, + 0x75, 0xfe, 0x4a, 0xbd, 0x56, 0x08, 0xd7, 0xcd, 0xda, 0xdd }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput024[] = { 0x42, 0xba, 0xae, 0x59, 0x78, 0xfe, 0xaf, 0x5c, 0x36, 0x8d, 0x14, 0xe0 }; +static CONST UINT8 EncOutput024[] = { 0xff, 0x7d, 0xc2, 0x03, 0xb2, 0x6c, 0x46, 0x7a, 0x6b, 0x50, + 0xdb, 0x33, 0x57, 0x8c, 0x0f, 0x27, 0x58, 0xc2, 0xe1, 0x4e, + 0x36, 0xd4, 0xfc, 0x10, 0x6d, 0xcb, 0x29, 0xb4 }; +// static CONST UINT8 EncAssoc024[] = { }; +static CONST UINT8 EncNonce024[] = { 0x9a, 0x59, 0xfc, 0xe2, 0x6d, 0xf0, 0x00, 0x5e, 0x07, 0x53, 0x86, 0x56 }; +static CONST UINT8 EncKey024[] = { 0x99, 0xb6, 0x2b, 0xd5, 0xaf, 0xbe, 0x3f, 0xb0, 0x15, 0xbd, 0xe9, + 0x3f, 0x0a, 0xbf, 0x48, 0x39, 0x57, 0xa1, 0xc3, 0xeb, 0x3c, 0xa5, + 0x9c, 0xb5, 0x0b, 0x39, 0xf7, 0xf8, 0xa9, 0xcc, 0x51, 0xbe }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput025[] = { 0xfd, 0xc8, 0x5b, 0x94, 0xa4, 0xb2, 0xa6, 0xb7, 0x59, 0xb1, 0xa0, 0xda }; +static CONST UINT8 EncOutput025[] = { 0x9f, 0x88, 0x16, 0xde, 0x09, 0x94, 0xe9, 0x38, 0xd9, 0xe5, + 0x3f, 0x95, 0xd0, 0x86, 0xfc, 0x6c, 0x9d, 0x8f, 0xa9, 0x15, + 0xfd, 0x84, 0x23, 0xa7, 0xcf, 0x05, 0x07, 0x2f }; +static CONST UINT8 EncAssoc025[] = { 0xa5, 0x06, 0xe1, 0xa5, 0xc6, 0x90, 0x93, 0xf9 }; +static CONST UINT8 EncNonce025[] = { 0x58, 0xdb, 0xd4, 0xad, 0x2c, 0x4a, 0xd3, 0x5d, 0xd9, 0x06, 0xe9, 0xce }; +static CONST UINT8 EncKey025[] = { 0x85, 0xf3, 0x5b, 0x62, 0x82, 0xcf, 0xf4, 0x40, 0xbc, 0x10, 0x20, + 0xc8, 0x13, 0x6f, 0xf2, 0x70, 0x31, 0x11, 0x0f, 0xa6, 0x3e, 0xc1, + 0x6f, 0x1e, 0x82, 0x51, 0x18, 0xb0, 0x06, 0xb9, 0x12, 0x57 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput026[] = { 0x51, 0xf8, 0xc1, 0xf7, 0x31, 0xea, 0x14, 0xac, + 0xdb, 0x21, 0x0a, 0x6d, 0x97, 0x3e, 0x07 }; +static CONST UINT8 EncOutput026[] = { 0x0b, 0x29, 0x63, 0x8e, 0x1f, 0xbd, 0xd6, 0xdf, 0x53, 0x97, 0x0b, + 0xe2, 0x21, 0x00, 0x42, 0x2a, 0x91, 0x34, 0x08, 0x7d, 0x67, 0xa4, + 0x6e, 0x79, 0x17, 0x8d, 0x0a, 0x93, 0xf5, 0xe1, 0xd2 }; +// static CONST UINT8 EncAssoc026[] = { }; +static CONST UINT8 EncNonce026[] = { 0x68, 0xab, 0x7f, 0xdb, 0xf6, 0x19, 0x01, 0xda, 0xd4, 0x61, 0xd2, 0x3c }; +static CONST UINT8 EncKey026[] = { 0x67, 0x11, 0x96, 0x27, 0xbd, 0x98, 0x8e, 0xda, 0x90, 0x62, 0x19, + 0xe0, 0x8c, 0x0d, 0x0d, 0x77, 0x9a, 0x07, 0xd2, 0x08, 0xce, 0x8a, + 0x4f, 0xe0, 0x70, 0x9a, 0xf7, 0x55, 0xee, 0xec, 0x6d, 0xcb }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput027[] = { 0x97, 0x46, 0x9d, 0xa6, 0x67, 0xd6, 0x11, 0x0f, + 0x9c, 0xbd, 0xa1, 0xd1, 0xa2, 0x06, 0x73 }; +static CONST UINT8 EncOutput027[] = { 0x32, 0xdb, 0x66, 0xc4, 0xa3, 0x81, 0x9d, 0x81, 0x55, 0x74, 0x55, + 0xe5, 0x98, 0x0f, 0xed, 0xfe, 0xae, 0x30, 0xde, 0xc9, 0x4e, 0x6a, + 0xd3, 0xa9, 0xee, 0xa0, 0x6a, 0x0d, 0x70, 0x39, 0x17 }; +static CONST UINT8 EncAssoc027[] = { 0x64, 0x53, 0xa5, 0x33, 0x84, 0x63, 0x22, 0x12 }; +static CONST UINT8 EncNonce027[] = { 0xd9, 0x5b, 0x32, 0x43, 0xaf, 0xae, 0xf7, 0x14, 0xc5, 0x03, 0x5b, 0x6a }; +static CONST UINT8 EncKey027[] = { 0xe6, 0xf1, 0x11, 0x8d, 0x41, 0xe4, 0xb4, 0x3f, 0xb5, 0x82, 0x21, + 0xb7, 0xed, 0x79, 0x67, 0x38, 0x34, 0xe0, 0xd8, 0xac, 0x5c, 0x4f, + 0xa6, 0x0b, 0xbc, 0x8b, 0xc4, 0x89, 0x3a, 0x58, 0x89, 0x4d }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput028[] = { 0x54, 0x9b, 0x36, 0x5a, 0xf9, 0x13, 0xf3, 0xb0, + 0x81, 0x13, 0x1c, 0xcb, 0x6b, 0x82, 0x55, 0x88 }; +static CONST UINT8 EncOutput028[] = { 0xe9, 0x11, 0x0e, 0x9f, 0x56, 0xab, 0x3c, 0xa4, 0x83, 0x50, 0x0c, + 0xea, 0xba, 0xb6, 0x7a, 0x13, 0x83, 0x6c, 0xca, 0xbf, 0x15, 0xa6, + 0xa2, 0x2a, 0x51, 0xc1, 0x07, 0x1c, 0xfa, 0x68, 0xfa, 0x0c }; +// static CONST UINT8 EncAssoc028[] = { }; +static CONST UINT8 EncNonce028[] = { 0x2f, 0xcb, 0x1b, 0x38, 0xa9, 0x9e, 0x71, 0xb8, 0x47, 0x40, 0xad, 0x9b }; +static CONST UINT8 EncKey028[] = { 0x59, 0xd4, 0xea, 0xfb, 0x4d, 0xe0, 0xcf, 0xc7, 0xd3, 0xdb, 0x99, + 0xa8, 0xf5, 0x4b, 0x15, 0xd7, 0xb3, 0x9f, 0x0a, 0xcc, 0x8d, 0xa6, + 0x97, 0x63, 0xb0, 0x19, 0xc1, 0x69, 0x9f, 0x87, 0x67, 0x4a }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput029[] = { 0x55, 0xa4, 0x65, 0x64, 0x4f, 0x5b, 0x65, 0x09, + 0x28, 0xcb, 0xee, 0x7c, 0x06, 0x32, 0x14, 0xd6 }; +static CONST UINT8 EncOutput029[] = { 0xe4, 0xb1, 0x13, 0xcb, 0x77, 0x59, 0x45, 0xf3, 0xd3, 0xa8, 0xae, + 0x9e, 0xc1, 0x41, 0xc0, 0x0c, 0x7c, 0x43, 0xf1, 0x6c, 0xe0, 0x96, + 0xd0, 0xdc, 0x27, 0xc9, 0x58, 0x49, 0xdc, 0x38, 0x3b, 0x7d }; +static CONST UINT8 EncAssoc029[] = { 0x03, 0x45, 0x85, 0x62, 0x1a, 0xf8, 0xd7, 0xff }; +static CONST UINT8 EncNonce029[] = { 0x11, 0x8a, 0x69, 0x64, 0xc2, 0xd3, 0xe3, 0x80, 0x07, 0x1f, 0x52, 0x66 }; +static CONST UINT8 EncKey029[] = { 0xb9, 0x07, 0xa4, 0x50, 0x75, 0x51, 0x3f, 0xe8, 0xa8, 0x01, 0x9e, + 0xde, 0xe3, 0xf2, 0x59, 0x14, 0x87, 0xb2, 0xa0, 0x30, 0xb0, 0x3c, + 0x6e, 0x1d, 0x77, 0x1c, 0x86, 0x25, 0x71, 0xd2, 0xea, 0x1e }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput030[] = { 0x3f, 0xf1, 0x51, 0x4b, 0x1c, 0x50, 0x39, 0x15, 0x91, + 0x8f, 0x0c, 0x0c, 0x31, 0x09, 0x4a, 0x6e, 0x1f }; +static CONST UINT8 EncOutput030[] = { 0x02, 0xcc, 0x3a, 0xcb, 0x5e, 0xe1, 0xfc, 0xdd, 0x12, 0xa0, 0x3b, + 0xb8, 0x57, 0x97, 0x64, 0x74, 0xd3, 0xd8, 0x3b, 0x74, 0x63, 0xa2, + 0xc3, 0x80, 0x0f, 0xe9, 0x58, 0xc2, 0x8e, 0xaa, 0x29, 0x08, 0x13 }; +// static CONST UINT8 EncAssoc030[] = { }; +static CONST UINT8 EncNonce030[] = { 0x45, 0xaa, 0xa3, 0xe5, 0xd1, 0x6d, 0x2d, 0x42, 0xdc, 0x03, 0x44, 0x5d }; +static CONST UINT8 EncKey030[] = { 0x3b, 0x24, 0x58, 0xd8, 0x17, 0x6e, 0x16, 0x21, 0xc0, 0xcc, 0x24, + 0xc0, 0xc0, 0xe2, 0x4c, 0x1e, 0x80, 0xd7, 0x2f, 0x7e, 0xe9, 0x14, + 0x9a, 0x4b, 0x16, 0x61, 0x76, 0x62, 0x96, 0x16, 0xd0, 0x11 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput031[] = { 0x63, 0x85, 0x8c, 0xa3, 0xe2, 0xce, 0x69, 0x88, 0x7b, + 0x57, 0x8a, 0x3c, 0x16, 0x7b, 0x42, 0x1c, 0x9c }; +static CONST UINT8 EncOutput031[] = { 0x35, 0x76, 0x64, 0x88, 0xd2, 0xbc, 0x7c, 0x2b, 0x8d, 0x17, 0xcb, + 0xbb, 0x9a, 0xbf, 0xad, 0x9e, 0x6d, 0x1f, 0x39, 0x1e, 0x65, 0x7b, + 0x27, 0x38, 0xdd, 0xa0, 0x84, 0x48, 0xcb, 0xa2, 0x81, 0x1c, 0xeb }; +static CONST UINT8 EncAssoc031[] = { 0x9a, 0xaf, 0x29, 0x9e, 0xee, 0xa7, 0x8f, 0x79 }; +static CONST UINT8 EncNonce031[] = { 0xf0, 0x38, 0x4f, 0xb8, 0x76, 0x12, 0x14, 0x10, 0x63, 0x3d, 0x99, 0x3d }; +static CONST UINT8 EncKey031[] = { 0xf6, 0x0c, 0x6a, 0x1b, 0x62, 0x57, 0x25, 0xf7, 0x6c, 0x70, 0x37, + 0xb4, 0x8f, 0xe3, 0x57, 0x7f, 0xa7, 0xf7, 0xb8, 0x7b, 0x1b, 0xd5, + 0xa9, 0x82, 0x17, 0x6d, 0x18, 0x23, 0x06, 0xff, 0xb8, 0x70 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput032[] = { 0x10, 0xf1, 0xec, 0xf9, 0xc6, 0x05, 0x84, 0x66, 0x5d, 0x9a, 0xe5, 0xef, + 0xe2, 0x79, 0xe7, 0xf7, 0x37, 0x7e, 0xea, 0x69, 0x16, 0xd2, 0xb1, 0x11 }; +static CONST UINT8 EncOutput032[] = { 0x42, 0xf2, 0x6c, 0x56, 0xcb, 0x4b, 0xe2, 0x1d, 0x9d, 0x8d, + 0x0c, 0x80, 0xfc, 0x99, 0xdd, 0xe0, 0x0d, 0x75, 0xf3, 0x80, + 0x74, 0xbf, 0xe7, 0x64, 0x54, 0xaa, 0x7e, 0x13, 0xd4, 0x8f, + 0xff, 0x7d, 0x75, 0x57, 0x03, 0x94, 0x57, 0x04, 0x0a, 0x3a }; +// static CONST UINT8 EncAssoc032[] = { }; +static CONST UINT8 EncNonce032[] = { 0xe6, 0xb1, 0xad, 0xf2, 0xfd, 0x58, 0xa8, 0x76, 0x2c, 0x65, 0xf3, 0x1b }; +static CONST UINT8 EncKey032[] = { 0x02, 0x12, 0xa8, 0xde, 0x50, 0x07, 0xed, 0x87, 0xb3, 0x3f, 0x1a, + 0x70, 0x90, 0xb6, 0x11, 0x4f, 0x9e, 0x08, 0xce, 0xfd, 0x96, 0x07, + 0xf2, 0xc2, 0x76, 0xbd, 0xcf, 0xdb, 0xc5, 0xce, 0x9c, 0xd7 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput033[] = { 0x92, 0x22, 0xf9, 0x01, 0x8e, 0x54, 0xfd, 0x6d, 0xe1, 0x20, 0x08, 0x06, + 0xa9, 0xee, 0x8e, 0x4c, 0xc9, 0x04, 0xd2, 0x9f, 0x25, 0xcb, 0xa1, 0x93 }; +static CONST UINT8 EncOutput033[] = { 0x12, 0x30, 0x32, 0x43, 0x7b, 0x4b, 0xfd, 0x69, 0x20, 0xe8, + 0xf7, 0xe7, 0xe0, 0x08, 0x7a, 0xe4, 0x88, 0x9e, 0xbe, 0x7a, + 0x0a, 0xd0, 0xe9, 0x00, 0x3c, 0xf6, 0x8f, 0x17, 0x95, 0x50, + 0xda, 0x63, 0xd3, 0xb9, 0x6c, 0x2d, 0x55, 0x41, 0x18, 0x65 }; +static CONST UINT8 EncAssoc033[] = { 0x3e, 0x8b, 0xc5, 0xad, 0xe1, 0x82, 0xff, 0x08 }; +static CONST UINT8 EncNonce033[] = { 0x6b, 0x28, 0x2e, 0xbe, 0xcc, 0x54, 0x1b, 0xcd, 0x78, 0x34, 0xed, 0x55 }; +static CONST UINT8 EncKey033[] = { 0xc5, 0xbc, 0x09, 0x56, 0x56, 0x46, 0xe7, 0xed, 0xda, 0x95, 0x4f, + 0x1f, 0x73, 0x92, 0x23, 0xda, 0xda, 0x20, 0xb9, 0x5c, 0x44, 0xab, + 0x03, 0x3d, 0x0f, 0xae, 0x4b, 0x02, 0x83, 0xd1, 0x8b, 0xe3 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput034[] = { 0xb0, 0x53, 0x99, 0x92, 0x86, 0xa2, 0x82, 0x4f, 0x42, 0xcc, 0x8c, + 0x20, 0x3a, 0xb2, 0x4e, 0x2c, 0x97, 0xa6, 0x85, 0xad, 0xcc, 0x2a, + 0xd3, 0x26, 0x62, 0x55, 0x8e, 0x55, 0xa5, 0xc7, 0x29 }; +static CONST UINT8 EncOutput034[] = { 0x45, 0xc7, 0xd6, 0xb5, 0x3a, 0xca, 0xd4, 0xab, 0xb6, 0x88, 0x76, 0xa6, + 0xe9, 0x6a, 0x48, 0xfb, 0x59, 0x52, 0x4d, 0x2c, 0x92, 0xc9, 0xd8, 0xa1, + 0x89, 0xc9, 0xfd, 0x2d, 0xb9, 0x17, 0x46, 0x56, 0x6d, 0x3c, 0xa1, 0x0e, + 0x31, 0x1b, 0x69, 0x5f, 0x3e, 0xae, 0x15, 0x51, 0x65, 0x24, 0x93 }; +// static CONST UINT8 EncAssoc034[] = { }; +static CONST UINT8 EncNonce034[] = { 0x04, 0xa9, 0xbe, 0x03, 0x50, 0x8a, 0x5f, 0x31, 0x37, 0x1a, 0x6f, 0xd2 }; +static CONST UINT8 EncKey034[] = { 0x2e, 0xb5, 0x1c, 0x46, 0x9a, 0xa8, 0xeb, 0x9e, 0x6c, 0x54, 0xa8, + 0x34, 0x9b, 0xae, 0x50, 0xa2, 0x0f, 0x0e, 0x38, 0x27, 0x11, 0xbb, + 0xa1, 0x15, 0x2c, 0x42, 0x4f, 0x03, 0xb6, 0x67, 0x1d, 0x71 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput035[] = { 0xf4, 0x52, 0x06, 0xab, 0xc2, 0x55, 0x52, 0xb2, 0xab, 0xc9, 0xab, + 0x7f, 0xa2, 0x43, 0x03, 0x5f, 0xed, 0xaa, 0xdd, 0xc3, 0xb2, 0x29, + 0x39, 0x56, 0xf1, 0xea, 0x6e, 0x71, 0x56, 0xe7, 0xeb }; +static CONST UINT8 EncOutput035[] = { 0x46, 0xa8, 0x0c, 0x41, 0x87, 0x02, 0x47, 0x20, 0x08, 0x46, 0x27, 0x58, + 0x00, 0x80, 0xdd, 0xe5, 0xa3, 0xf4, 0xa1, 0x10, 0x93, 0xa7, 0x07, 0x6e, + 0xd6, 0xf3, 0xd3, 0x26, 0xbc, 0x7b, 0x70, 0x53, 0x4d, 0x4a, 0xa2, 0x83, + 0x5a, 0x52, 0xe7, 0x2d, 0x14, 0xdf, 0x0e, 0x4f, 0x47, 0xf2, 0x5f }; +static CONST UINT8 EncAssoc035[] = { 0x37, 0x46, 0x18, 0xa0, 0x6e, 0xa9, 0x8a, 0x48 }; +static CONST UINT8 EncNonce035[] = { 0x47, 0x0a, 0x33, 0x9e, 0xcb, 0x32, 0x19, 0xb8, 0xb8, 0x1a, 0x1f, 0x8b }; +static CONST UINT8 EncKey035[] = { 0x7f, 0x5b, 0x74, 0xc0, 0x7e, 0xd1, 0xb4, 0x0f, 0xd1, 0x43, 0x58, + 0xfe, 0x2f, 0xf2, 0xa7, 0x40, 0xc1, 0x16, 0xc7, 0x70, 0x65, 0x10, + 0xe6, 0xa4, 0x37, 0xf1, 0x9e, 0xa4, 0x99, 0x11, 0xce, 0xc4 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput036[] = { 0xb9, 0xc5, 0x54, 0xcb, 0xc3, 0x6a, 0xc1, 0x8a, 0xe8, 0x97, 0xdf, + 0x7b, 0xee, 0xca, 0xc1, 0xdb, 0xeb, 0x4e, 0xaf, 0xa1, 0x56, 0xbb, + 0x60, 0xce, 0x2e, 0x5d, 0x48, 0xf0, 0x57, 0x15, 0xe6, 0x78 }; +static CONST UINT8 EncOutput036[] = { 0xea, 0x29, 0xaf, 0xa4, 0x9d, 0x36, 0xe8, 0x76, 0x0f, 0x5f, 0xe1, 0x97, + 0x23, 0xb9, 0x81, 0x1e, 0xd5, 0xd5, 0x19, 0x93, 0x4a, 0x44, 0x0f, 0x50, + 0x81, 0xac, 0x43, 0x0b, 0x95, 0x3b, 0x0e, 0x21, 0x22, 0x25, 0x41, 0xaf, + 0x46, 0xb8, 0x65, 0x33, 0xc6, 0xb6, 0x8d, 0x2f, 0xf1, 0x08, 0xa7, 0xea }; +// static CONST UINT8 EncAssoc036[] = { }; +static CONST UINT8 EncNonce036[] = { 0x72, 0xcf, 0xd9, 0x0e, 0xf3, 0x02, 0x6c, 0xa2, 0x2b, 0x7e, 0x6e, 0x6a }; +static CONST UINT8 EncKey036[] = { 0xe1, 0x73, 0x1d, 0x58, 0x54, 0xe1, 0xb7, 0x0c, 0xb3, 0xff, 0xe8, + 0xb7, 0x86, 0xa2, 0xb3, 0xeb, 0xf0, 0x99, 0x43, 0x70, 0x95, 0x47, + 0x57, 0xb9, 0xdc, 0x8c, 0x7b, 0xc5, 0x35, 0x46, 0x34, 0xa3 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput037[] = { 0x6b, 0x26, 0x04, 0x99, 0x6c, 0xd3, 0x0c, 0x14, 0xa1, 0x3a, 0x52, + 0x57, 0xed, 0x6c, 0xff, 0xd3, 0xbc, 0x5e, 0x29, 0xd6, 0xb9, 0x7e, + 0xb1, 0x79, 0x9e, 0xb3, 0x35, 0xe2, 0x81, 0xea, 0x45, 0x1e }; +static CONST UINT8 EncOutput037[] = { 0x6d, 0xad, 0x63, 0x78, 0x97, 0x54, 0x4d, 0x8b, 0xf6, 0xbe, 0x95, 0x07, + 0xed, 0x4d, 0x1b, 0xb2, 0xe9, 0x54, 0xbc, 0x42, 0x7e, 0x5d, 0xe7, 0x29, + 0xda, 0xf5, 0x07, 0x62, 0x84, 0x6f, 0xf2, 0xf4, 0x7b, 0x99, 0x7d, 0x93, + 0xc9, 0x82, 0x18, 0x9d, 0x70, 0x95, 0xdc, 0x79, 0x4c, 0x74, 0x62, 0x32 }; +static CONST UINT8 EncAssoc037[] = { 0x23, 0x33, 0xe5, 0xce, 0x0f, 0x93, 0xb0, 0x59 }; +static CONST UINT8 EncNonce037[] = { 0x26, 0x28, 0x80, 0xd4, 0x75, 0xf3, 0xda, 0xc5, 0x34, 0x0d, 0xd1, 0xb8 }; +static CONST UINT8 EncKey037[] = { 0x27, 0xd8, 0x60, 0x63, 0x1b, 0x04, 0x85, 0xa4, 0x10, 0x70, 0x2f, + 0xea, 0x61, 0xbc, 0x87, 0x3f, 0x34, 0x42, 0x26, 0x0c, 0xad, 0xed, + 0x4a, 0xbd, 0xe2, 0x5b, 0x78, 0x6a, 0x2d, 0x97, 0xf1, 0x45 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput038[] = { 0x97, 0x3d, 0x0c, 0x75, 0x38, 0x26, 0xba, 0xe4, 0x66, 0xcf, 0x9a, 0xbb, + 0x34, 0x93, 0x15, 0x2e, 0x9d, 0xe7, 0x81, 0x9e, 0x2b, 0xd0, 0xc7, 0x11, + 0x71, 0x34, 0x6b, 0x4d, 0x2c, 0xeb, 0xf8, 0x04, 0x1a, 0xa3, 0xce, 0xdc, + 0x0d, 0xfd, 0x7b, 0x46, 0x7e, 0x26, 0x22, 0x8b, 0xc8, 0x6c, 0x9a }; +static CONST UINT8 EncOutput038[] = { 0xfb, 0xa7, 0x8a, 0xe4, 0xf9, 0xd8, 0x08, 0xa6, 0x2e, 0x3d, 0xa4, 0x0b, 0xe2, + 0xcb, 0x77, 0x00, 0xc3, 0x61, 0x3d, 0x9e, 0xb2, 0xc5, 0x29, 0xc6, 0x52, 0xe7, + 0x6a, 0x43, 0x2c, 0x65, 0x8d, 0x27, 0x09, 0x5f, 0x0e, 0xb8, 0xf9, 0x40, 0xc3, + 0x24, 0x98, 0x1e, 0xa9, 0x35, 0xe5, 0x07, 0xf9, 0x8f, 0x04, 0x69, 0x56, 0xdb, + 0x3a, 0x51, 0x29, 0x08, 0xbd, 0x7a, 0xfc, 0x8f, 0x2a, 0xb0, 0xa9 }; +// static CONST UINT8 EncAssoc038[] = { }; +static CONST UINT8 EncNonce038[] = { 0xe7, 0x4a, 0x51, 0x5e, 0x7e, 0x21, 0x02, 0xb9, 0x0b, 0xef, 0x55, 0xd2 }; +static CONST UINT8 EncKey038[] = { 0xcf, 0x0d, 0x40, 0xa4, 0x64, 0x4e, 0x5f, 0x51, 0x81, 0x51, 0x65, + 0xd5, 0x30, 0x1b, 0x22, 0x63, 0x1f, 0x45, 0x44, 0xc4, 0x9a, 0x18, + 0x78, 0xe3, 0xa0, 0xa5, 0xe8, 0xe1, 0xaa, 0xe0, 0xf2, 0x64 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput039[] = { 0xa9, 0x89, 0x95, 0x50, 0x4d, 0xf1, 0x6f, 0x74, 0x8b, 0xfb, 0x77, 0x85, + 0xff, 0x91, 0xee, 0xb3, 0xb6, 0x60, 0xea, 0x9e, 0xd3, 0x45, 0x0c, 0x3d, + 0x5e, 0x7b, 0x0e, 0x79, 0xef, 0x65, 0x36, 0x59, 0xa9, 0x97, 0x8d, 0x75, + 0x54, 0x2e, 0xf9, 0x1c, 0x45, 0x67, 0x62, 0x21, 0x56, 0x40, 0xb9 }; +static CONST UINT8 EncOutput039[] = { 0xa1, 0xff, 0xed, 0x80, 0x76, 0x18, 0x29, 0xec, 0xce, 0x24, 0x2e, 0x0e, 0x88, + 0xb1, 0x38, 0x04, 0x90, 0x16, 0xbc, 0xa0, 0x18, 0xda, 0x2b, 0x6e, 0x19, 0x98, + 0x6b, 0x3e, 0x31, 0x8c, 0xae, 0x8d, 0x80, 0x61, 0x98, 0xfb, 0x4c, 0x52, 0x7c, + 0xc3, 0x93, 0x50, 0xeb, 0xdd, 0xea, 0xc5, 0x73, 0xc4, 0xcb, 0xf0, 0xbe, 0xfd, + 0xa0, 0xb7, 0x02, 0x42, 0xc6, 0x40, 0xd7, 0xcd, 0x02, 0xd7, 0xa3 }; +static CONST UINT8 EncAssoc039[] = { 0xb3, 0xe4, 0x06, 0x46, 0x83, 0xb0, 0x2d, 0x84 }; +static CONST UINT8 EncNonce039[] = { 0xd4, 0xd8, 0x07, 0x34, 0x16, 0x83, 0x82, 0x5b, 0x31, 0xcd, 0x4d, 0x95 }; +static CONST UINT8 EncKey039[] = { 0x6c, 0xbf, 0xd7, 0x1c, 0x64, 0x5d, 0x18, 0x4c, 0xf5, 0xd2, 0x3c, + 0x40, 0x2b, 0xdb, 0x0d, 0x25, 0xec, 0x54, 0x89, 0x8c, 0x8a, 0x02, + 0x73, 0xd4, 0x2e, 0xb5, 0xbe, 0x10, 0x9f, 0xdc, 0xb2, 0xac }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput040[] = { 0xd0, 0x96, 0x80, 0x31, 0x81, 0xbe, 0xef, 0x9e, 0x00, 0x8f, 0xf8, 0x5d, 0x5d, + 0xdc, 0x38, 0xdd, 0xac, 0xf0, 0xf0, 0x9e, 0xe5, 0xf7, 0xe0, 0x7f, 0x1e, 0x40, + 0x79, 0xcb, 0x64, 0xd0, 0xdc, 0x8f, 0x5e, 0x67, 0x11, 0xcd, 0x49, 0x21, 0xa7, + 0x88, 0x7d, 0xe7, 0x6e, 0x26, 0x78, 0xfd, 0xc6, 0x76, 0x18, 0xf1, 0x18, 0x55, + 0x86, 0xbf, 0xea, 0x9d, 0x4c, 0x68, 0x5d, 0x50, 0xe4, 0xbb, 0x9a, 0x82 }; +static CONST UINT8 EncOutput040[] = { 0x9a, 0x4e, 0xf2, 0x2b, 0x18, 0x16, 0x77, 0xb5, 0x75, 0x5c, 0x08, 0xf7, + 0x47, 0xc0, 0xf8, 0xd8, 0xe8, 0xd4, 0xc1, 0x8a, 0x9c, 0xc2, 0x40, 0x5c, + 0x12, 0xbb, 0x51, 0xbb, 0x18, 0x72, 0xc8, 0xe8, 0xb8, 0x77, 0x67, 0x8b, + 0xec, 0x44, 0x2c, 0xfc, 0xbb, 0x0f, 0xf4, 0x64, 0xa6, 0x4b, 0x74, 0x33, + 0x2c, 0xf0, 0x72, 0x89, 0x8c, 0x7e, 0x0e, 0xdd, 0xf6, 0x23, 0x2e, 0xa6, + 0xe2, 0x7e, 0xfe, 0x50, 0x9f, 0xf3, 0x42, 0x7a, 0x0f, 0x32, 0xfa, 0x56, + 0x6d, 0x9c, 0xa0, 0xa7, 0x8a, 0xef, 0xc0, 0x13 }; +// static CONST UINT8 EncAssoc040[] = { }; +static CONST UINT8 EncNonce040[] = { 0xd6, 0x10, 0x40, 0xa3, 0x13, 0xed, 0x49, 0x28, 0x23, 0xcc, 0x06, 0x5b }; +static CONST UINT8 EncKey040[] = { 0x5b, 0x1d, 0x10, 0x35, 0xc0, 0xb1, 0x7e, 0xe0, 0xb0, 0x44, 0x47, + 0x67, 0xf8, 0x0a, 0x25, 0xb8, 0xc1, 0xb7, 0x41, 0xf4, 0xb5, 0x0a, + 0x4d, 0x30, 0x52, 0x22, 0x6b, 0xaa, 0x1c, 0x6f, 0xb7, 0x01 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput041[] = { 0x94, 0xee, 0x16, 0x6d, 0x6d, 0x6e, 0xcf, 0x88, 0x32, 0x43, 0x71, 0x36, 0xb4, + 0xae, 0x80, 0x5d, 0x42, 0x88, 0x64, 0x35, 0x95, 0x86, 0xd9, 0x19, 0x3a, 0x25, + 0x01, 0x62, 0x93, 0xed, 0xba, 0x44, 0x3c, 0x58, 0xe0, 0x7e, 0x7b, 0x71, 0x95, + 0xec, 0x5b, 0xd8, 0x45, 0x82, 0xa9, 0xd5, 0x6c, 0x8d, 0x4a, 0x10, 0x8c, 0x7d, + 0x7c, 0xe3, 0x4e, 0x6c, 0x6f, 0x8e, 0xa1, 0xbe, 0xc0, 0x56, 0x73, 0x17 }; +static CONST UINT8 EncOutput041[] = { 0x5f, 0xbb, 0xde, 0xcc, 0x34, 0xbe, 0x20, 0x16, 0x14, 0xf6, 0x36, 0x03, + 0x1e, 0xeb, 0x42, 0xf1, 0xca, 0xce, 0x3c, 0x79, 0xa1, 0x2c, 0xff, 0xd8, + 0x71, 0xee, 0x8e, 0x73, 0x82, 0x0c, 0x82, 0x97, 0x49, 0xf1, 0xab, 0xb4, + 0x29, 0x43, 0x67, 0x84, 0x9f, 0xb6, 0xc2, 0xaa, 0x56, 0xbd, 0xa8, 0xa3, + 0x07, 0x8f, 0x72, 0x3d, 0x7c, 0x1c, 0x85, 0x20, 0x24, 0xb0, 0x17, 0xb5, + 0x89, 0x73, 0xfb, 0x1e, 0x09, 0x26, 0x3d, 0xa7, 0xb4, 0xcb, 0x92, 0x14, + 0x52, 0xf9, 0x7d, 0xca, 0x40, 0xf5, 0x80, 0xec }; +static CONST UINT8 EncAssoc041[] = { 0x71, 0x93, 0xf6, 0x23, 0x66, 0x33, 0x21, 0xa2 }; +static CONST UINT8 EncNonce041[] = { 0xd3, 0x1c, 0x21, 0xab, 0xa1, 0x75, 0xb7, 0x0d, 0xe4, 0xeb, 0xb1, 0x9c }; +static CONST UINT8 EncKey041[] = { 0x97, 0xd6, 0x35, 0xc4, 0xf4, 0x75, 0x74, 0xd9, 0x99, 0x8a, 0x90, + 0x87, 0x5d, 0xa1, 0xd3, 0xa2, 0x84, 0xb7, 0x55, 0xb2, 0xd3, 0x92, + 0x97, 0xa5, 0x72, 0x52, 0x35, 0x19, 0x0e, 0x10, 0xa9, 0x7e }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput042[] = { 0xb4, 0x29, 0xeb, 0x80, 0xfb, 0x8f, 0xe8, 0xba, 0xed, 0xa0, 0xc8, 0x5b, 0x9c, 0x33, + 0x34, 0x58, 0xe7, 0xc2, 0x99, 0x2e, 0x55, 0x84, 0x75, 0x06, 0x9d, 0x12, 0xd4, 0x5c, + 0x22, 0x21, 0x75, 0x64, 0x12, 0x15, 0x88, 0x03, 0x22, 0x97, 0xef, 0xf5, 0x67, 0x83, + 0x74, 0x2a, 0x5f, 0xc2, 0x2d, 0x74, 0x10, 0xff, 0xb2, 0x9d, 0x66, 0x09, 0x86, 0x61, + 0xd7, 0x6f, 0x12, 0x6c, 0x3c, 0x27, 0x68, 0x9e, 0x43, 0xb3, 0x72, 0x67, 0xca, 0xc5, + 0xa3, 0xa6, 0xd3, 0xab, 0x49, 0xe3, 0x91, 0xda, 0x29, 0xcd, 0x30, 0x54, 0xa5, 0x69, + 0x2e, 0x28, 0x07, 0xe4, 0xc3, 0xea, 0x46, 0xc8, 0x76, 0x1d, 0x50, 0xf5, 0x92 }; +static CONST UINT8 EncOutput042[] = { + 0xd0, 0x10, 0x2f, 0x6c, 0x25, 0x8b, 0xf4, 0x97, 0x42, 0xce, 0xc3, 0x4c, 0xf2, 0xd0, 0xfe, 0xdf, 0x23, 0xd1, 0x05, + 0xfb, 0x4c, 0x84, 0xcf, 0x98, 0x51, 0x5e, 0x1b, 0xc9, 0xa6, 0x4f, 0x8a, 0xd5, 0xbe, 0x8f, 0x07, 0x21, 0xbd, 0xe5, + 0x06, 0x45, 0xd0, 0x00, 0x83, 0xc3, 0xa2, 0x63, 0xa3, 0x10, 0x53, 0xb7, 0x60, 0x24, 0x5f, 0x52, 0xae, 0x28, 0x66, + 0xa5, 0xec, 0x83, 0xb1, 0x9f, 0x61, 0xbe, 0x1d, 0x30, 0xd5, 0xc5, 0xd9, 0xfe, 0xcc, 0x4c, 0xbb, 0xe0, 0x8f, 0xd3, + 0x85, 0x81, 0x3a, 0x2a, 0xa3, 0x9a, 0x00, 0xff, 0x9c, 0x10, 0xf7, 0xf2, 0x37, 0x02, 0xad, 0xd1, 0xe4, 0xb2, 0xff, + 0xa3, 0x1c, 0x41, 0x86, 0x5f, 0xc7, 0x1d, 0xe1, 0x2b, 0x19, 0x61, 0x21, 0x27, 0xce, 0x49, 0x99, 0x3b, 0xb0 +}; +// static CONST UINT8 EncAssoc042[] = { }; +static CONST UINT8 EncNonce042[] = { 0x17, 0xc8, 0x6a, 0x8a, 0xbb, 0xb7, 0xe0, 0x03, 0xac, 0xde, 0x27, 0x99 }; +static CONST UINT8 EncKey042[] = { 0xfe, 0x6e, 0x55, 0xbd, 0xae, 0xd1, 0xf7, 0x28, 0x4c, 0xa5, 0xfc, + 0x0f, 0x8c, 0x5f, 0x2b, 0x8d, 0xf5, 0x6d, 0xc0, 0xf4, 0x9e, 0x8c, + 0xa6, 0x6a, 0x41, 0x99, 0x5e, 0x78, 0x33, 0x51, 0xf9, 0x01 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput043[] = { 0xce, 0xb5, 0x34, 0xce, 0x50, 0xdc, 0x23, 0xff, 0x63, 0x8a, 0xce, 0x3e, 0xf6, 0x3a, + 0xb2, 0xcc, 0x29, 0x73, 0xee, 0xad, 0xa8, 0x07, 0x85, 0xfc, 0x16, 0x5d, 0x06, 0xc2, + 0xf5, 0x10, 0x0f, 0xf5, 0xe8, 0xab, 0x28, 0x82, 0xc4, 0x75, 0xaf, 0xcd, 0x05, 0xcc, + 0xd4, 0x9f, 0x2e, 0x7d, 0x8f, 0x55, 0xef, 0x3a, 0x72, 0xe3, 0xdc, 0x51, 0xd6, 0x85, + 0x2b, 0x8e, 0x6b, 0x9e, 0x7a, 0xec, 0xe5, 0x7b, 0xe6, 0x55, 0x6b, 0x0b, 0x6d, 0x94, + 0x13, 0xe3, 0x3f, 0xc5, 0xfc, 0x24, 0xa9, 0xa2, 0x05, 0xad, 0x59, 0x57, 0x4b, 0xb3, + 0x9d, 0x94, 0x4a, 0x92, 0xdc, 0x47, 0x97, 0x0d, 0x84, 0xa6, 0xad, 0x31, 0x76 }; +static CONST UINT8 EncOutput043[] = { + 0x75, 0x45, 0x39, 0x1b, 0x51, 0xde, 0x01, 0xd5, 0xc5, 0x3d, 0xfa, 0xca, 0x77, 0x79, 0x09, 0x06, 0x3e, 0x58, 0xed, + 0xee, 0x4b, 0xb1, 0x22, 0x7e, 0x71, 0x10, 0xac, 0x4d, 0x26, 0x20, 0xc2, 0xae, 0xc2, 0xf8, 0x48, 0xf5, 0x6d, 0xee, + 0xb0, 0x37, 0xa8, 0xdc, 0xed, 0x75, 0xaf, 0xa8, 0xa6, 0xc8, 0x90, 0xe2, 0xde, 0xe4, 0x2f, 0x95, 0x0b, 0xb3, 0x3d, + 0x9e, 0x24, 0x24, 0xd0, 0x8a, 0x50, 0x5d, 0x89, 0x95, 0x63, 0x97, 0x3e, 0xd3, 0x88, 0x70, 0xf3, 0xde, 0x6e, 0xe2, + 0xad, 0xc7, 0xfe, 0x07, 0x2c, 0x36, 0x6c, 0x14, 0xe2, 0xcf, 0x7c, 0xa6, 0x2f, 0xb3, 0xd3, 0x6b, 0xee, 0x11, 0x68, + 0x54, 0x61, 0xb7, 0x0d, 0x44, 0xef, 0x8c, 0x66, 0xc5, 0xc7, 0xbb, 0xf1, 0x0d, 0xca, 0xdd, 0x7f, 0xac, 0xf6 +}; +static CONST UINT8 EncAssoc043[] = { 0xa1, 0x1c, 0x40, 0xb6, 0x03, 0x76, 0x73, 0x30 }; +static CONST UINT8 EncNonce043[] = { 0x46, 0x36, 0x2f, 0x45, 0xd6, 0x37, 0x9e, 0x63, 0xe5, 0x22, 0x94, 0x60 }; +static CONST UINT8 EncKey043[] = { 0xaa, 0xbc, 0x06, 0x34, 0x74, 0xe6, 0x5c, 0x4c, 0x3e, 0x9b, 0xdc, + 0x48, 0x0d, 0xea, 0x97, 0xb4, 0x51, 0x10, 0xc8, 0x61, 0x88, 0x46, + 0xff, 0x6b, 0x15, 0xbd, 0xd2, 0xa4, 0xa5, 0x68, 0x2c, 0x4e }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput044[] = { 0xe5, 0xcc, 0xaa, 0x44, 0x1b, 0xc8, 0x14, 0x68, + 0x8f, 0x8f, 0x6e, 0x8f, 0x28, 0xb5, 0x00, 0xb2 }; +static CONST UINT8 EncOutput044[] = { 0x7e, 0x72, 0xf5, 0xa1, 0x85, 0xaf, 0x16, 0xa6, 0x11, 0x92, 0x1b, + 0x43, 0x8f, 0x74, 0x9f, 0x0b, 0x12, 0x42, 0xc6, 0x70, 0x73, 0x23, + 0x34, 0x02, 0x9a, 0xdf, 0xe1, 0xc5, 0x00, 0x16, 0x51, 0xe4 }; +static CONST UINT8 EncAssoc044[] = { 0x02 }; +static CONST UINT8 EncNonce044[] = { 0x87, 0x34, 0x5f, 0x10, 0x55, 0xfd, 0x9e, 0x21, 0x02, 0xd5, 0x06, 0x56 }; +static CONST UINT8 EncKey044[] = { 0x7d, 0x00, 0xb4, 0x80, 0x95, 0xad, 0xfa, 0x32, 0x72, 0x05, 0x06, + 0x07, 0xb2, 0x64, 0x18, 0x50, 0x02, 0xba, 0x99, 0x95, 0x7c, 0x49, + 0x8b, 0xe0, 0x22, 0x77, 0x0f, 0x2c, 0xe2, 0xf3, 0x14, 0x3c }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput045[] = { 0x02, 0xcd, 0xe1, 0x68, 0xfb, 0xa3, 0xf5, 0x44, + 0xbb, 0xd0, 0x33, 0x2f, 0x7a, 0xde, 0xad, 0xa8 }; +static CONST UINT8 EncOutput045[] = { 0x85, 0xf2, 0x9a, 0x71, 0x95, 0x57, 0xcd, 0xd1, 0x4d, 0x1f, 0x8f, + 0xff, 0xab, 0x6d, 0x9e, 0x60, 0x73, 0x2c, 0xa3, 0x2b, 0xec, 0xd5, + 0x15, 0xa1, 0xed, 0x35, 0x3f, 0x54, 0x2e, 0x99, 0x98, 0x58 }; +static CONST UINT8 EncAssoc045[] = { 0xb6, 0x48 }; +static CONST UINT8 EncNonce045[] = { 0x87, 0xa3, 0x16, 0x3e, 0xc0, 0x59, 0x8a, 0xd9, 0x5b, 0x3a, 0xa7, 0x13 }; +static CONST UINT8 EncKey045[] = { 0x64, 0x32, 0x71, 0x7f, 0x1d, 0xb8, 0x5e, 0x41, 0xac, 0x78, 0x36, + 0xbc, 0xe2, 0x51, 0x85, 0xa0, 0x80, 0xd5, 0x76, 0x2b, 0x9e, 0x2b, + 0x18, 0x44, 0x4b, 0x6e, 0xc7, 0x2c, 0x3b, 0xd8, 0xe4, 0xdc }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput046[] = { 0x16, 0xdd, 0xd2, 0x3f, 0xf5, 0x3f, 0x3d, 0x23, + 0xc0, 0x63, 0x34, 0x48, 0x70, 0x40, 0xeb, 0x47 }; +static CONST UINT8 EncOutput046[] = { 0xc1, 0xb2, 0x95, 0x93, 0x6d, 0x56, 0xfa, 0xda, 0xc0, 0x3e, 0x5f, + 0x74, 0x2b, 0xff, 0x73, 0xa1, 0x39, 0xc4, 0x57, 0xdb, 0xab, 0x66, + 0x38, 0x2b, 0xab, 0xb3, 0xb5, 0x58, 0x00, 0xcd, 0xa5, 0xb8 }; +static CONST UINT8 EncAssoc046[] = { 0xbd, 0x4c, 0xd0, 0x2f, 0xc7, 0x50, 0x2b, 0xbd, + 0xbd, 0xf6, 0xc9, 0xa3, 0xcb, 0xe8, 0xf0 }; +static CONST UINT8 EncNonce046[] = { 0x6f, 0x57, 0x3a, 0xa8, 0x6b, 0xaa, 0x49, 0x2b, 0xa4, 0x65, 0x96, 0xdf }; +static CONST UINT8 EncKey046[] = { 0x8e, 0x34, 0xcf, 0x73, 0xd2, 0x45, 0xa1, 0x08, 0x2a, 0x92, 0x0b, + 0x86, 0x36, 0x4e, 0xb8, 0x96, 0xc4, 0x94, 0x64, 0x67, 0xbc, 0xb3, + 0xd5, 0x89, 0x29, 0xfc, 0xb3, 0x66, 0x90, 0xe6, 0x39, 0x4f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput047[] = { 0x62, 0x3b, 0x78, 0x50, 0xc3, 0x21, 0xe2, 0xcf, + 0x0c, 0x6f, 0xbc, 0xc8, 0xdf, 0xd1, 0xaf, 0xf2 }; +static CONST UINT8 EncOutput047[] = { 0xc8, 0x4c, 0x9b, 0xb7, 0xc6, 0x1c, 0x1b, 0xcb, 0x17, 0x77, 0x2a, + 0x1c, 0x50, 0x0c, 0x50, 0x95, 0xdb, 0xad, 0xf7, 0xa5, 0x13, 0x8c, + 0xa0, 0x34, 0x59, 0xa2, 0xcd, 0x65, 0x83, 0x1e, 0x09, 0x2f }; +static CONST UINT8 EncAssoc047[] = { 0x89, 0xcc, 0xe9, 0xfb, 0x47, 0x44, 0x1d, 0x07, + 0xe0, 0x24, 0x5a, 0x66, 0xfe, 0x8b, 0x77, 0x8b }; +static CONST UINT8 EncNonce047[] = { 0x1a, 0x65, 0x18, 0xf0, 0x2e, 0xde, 0x1d, 0xa6, 0x80, 0x92, 0x66, 0xd9 }; +static CONST UINT8 EncKey047[] = { 0xcb, 0x55, 0x75, 0xf5, 0xc7, 0xc4, 0x5c, 0x91, 0xcf, 0x32, 0x0b, + 0x13, 0x9f, 0xb5, 0x94, 0x23, 0x75, 0x60, 0xd0, 0xa3, 0xe6, 0xf8, + 0x65, 0xa6, 0x7d, 0x4f, 0x63, 0x3f, 0x2c, 0x08, 0xf0, 0x16 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput048[] = { 0x87, 0xb3, 0xa4, 0xd7, 0xb2, 0x6d, 0x8d, 0x32, + 0x03, 0xa0, 0xde, 0x1d, 0x64, 0xef, 0x82, 0xe3 }; +static CONST UINT8 EncOutput048[] = { 0x94, 0xbc, 0x80, 0x62, 0x1e, 0xd1, 0xe7, 0x1b, 0x1f, 0xd2, 0xb5, + 0xc3, 0xa1, 0x5e, 0x35, 0x68, 0x33, 0x35, 0x11, 0x86, 0x17, 0x96, + 0x97, 0x84, 0x01, 0x59, 0x8b, 0x96, 0x37, 0x22, 0xf5, 0xb3 }; +static CONST UINT8 EncAssoc048[] = { 0xd1, 0x9f, 0x2d, 0x98, 0x90, 0x95, 0xf7, 0xab, 0x03, + 0xa5, 0xfd, 0xe8, 0x44, 0x16, 0xe0, 0x0c, 0x0e }; +static CONST UINT8 EncNonce048[] = { 0x56, 0x4d, 0xee, 0x49, 0xab, 0x00, 0xd2, 0x40, 0xfc, 0x10, 0x68, 0xc3 }; +static CONST UINT8 EncKey048[] = { 0xa5, 0x56, 0x9e, 0x72, 0x9a, 0x69, 0xb2, 0x4b, 0xa6, 0xe0, 0xff, + 0x15, 0xc4, 0x62, 0x78, 0x97, 0x43, 0x68, 0x24, 0xc9, 0x41, 0xe9, + 0xd0, 0x0b, 0x2e, 0x93, 0xfd, 0xdc, 0x4b, 0xa7, 0x76, 0x57 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput049[] = { 0xe6, 0x01, 0xb3, 0x85, 0x57, 0x79, 0x7d, 0xa2, + 0xf8, 0xa4, 0x10, 0x6a, 0x08, 0x9d, 0x1d, 0xa6 }; +static CONST UINT8 EncOutput049[] = { 0x29, 0x9b, 0x5d, 0x3f, 0x3d, 0x03, 0xc0, 0x87, 0x20, 0x9a, 0x16, + 0xe2, 0x85, 0x14, 0x31, 0x11, 0x4b, 0x45, 0x4e, 0xd1, 0x98, 0xde, + 0x11, 0x7e, 0x83, 0xec, 0x49, 0xfa, 0x8d, 0x85, 0x08, 0xd6 }; +static CONST UINT8 EncAssoc049[] = { 0x5e, 0x64, 0x70, 0xfa, 0xcd, 0x99, 0xc1, 0xd8, 0x1e, 0x37, + 0xcd, 0x44, 0x01, 0x5f, 0xe1, 0x94, 0x80, 0xa2, 0xa4, 0xd3, + 0x35, 0x2a, 0x4f, 0xf5, 0x60, 0xc0, 0x64, 0x0f, 0xdb, 0xda }; +static CONST UINT8 EncNonce049[] = { 0xdf, 0x87, 0x13, 0xe8, 0x7e, 0xc3, 0xdb, 0xcf, 0xad, 0x14, 0xd5, 0x3e }; +static CONST UINT8 EncKey049[] = { 0x56, 0x20, 0x74, 0x65, 0xb4, 0xe4, 0x8e, 0x6d, 0x04, 0x63, 0x0f, + 0x4a, 0x42, 0xf3, 0x5c, 0xfc, 0x16, 0x3a, 0xb2, 0x89, 0xc2, 0x2a, + 0x2b, 0x47, 0x84, 0xf6, 0xf9, 0x29, 0x03, 0x30, 0xbe, 0xe0 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput050[] = { 0xdc, 0x9e, 0x9e, 0xaf, 0x11, 0xe3, 0x14, 0x18, + 0x2d, 0xf6, 0xa4, 0xeb, 0xa1, 0x7a, 0xec, 0x9c }; +static CONST UINT8 EncOutput050[] = { 0x60, 0x5b, 0xbf, 0x90, 0xae, 0xb9, 0x74, 0xf6, 0x60, 0x2b, 0xc7, + 0x78, 0x05, 0x6f, 0x0d, 0xca, 0x38, 0xea, 0x23, 0xd9, 0x90, 0x54, + 0xb4, 0x6b, 0x42, 0xff, 0xe0, 0x04, 0x12, 0x9d, 0x22, 0x04 }; +static CONST UINT8 EncAssoc050[] = { 0xba, 0x44, 0x6f, 0x6f, 0x9a, 0x0c, 0xed, 0x22, 0x45, 0x0f, 0xeb, + 0x10, 0x73, 0x7d, 0x90, 0x07, 0xfd, 0x69, 0xab, 0xc1, 0x9b, 0x1d, + 0x4d, 0x90, 0x49, 0xa5, 0x55, 0x1e, 0x86, 0xec, 0x2b, 0x37 }; +static CONST UINT8 EncNonce050[] = { 0x8d, 0xf4, 0xb1, 0x5a, 0x88, 0x8c, 0x33, 0x28, 0x6a, 0x7b, 0x76, 0x51 }; +static CONST UINT8 EncKey050[] = { 0x39, 0x37, 0x98, 0x6a, 0xf8, 0x6d, 0xaf, 0xc1, 0xba, 0x0c, 0x46, + 0x72, 0xd8, 0xab, 0xc4, 0x6c, 0x20, 0x70, 0x62, 0x68, 0x2d, 0x9c, + 0x26, 0x4a, 0xb0, 0x6d, 0x6c, 0x58, 0x07, 0x20, 0x51, 0x30 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput051[] = { 0x81, 0xce, 0x84, 0xed, 0xe9, 0xb3, 0x58, 0x59, + 0xcc, 0x8c, 0x49, 0xa8, 0xf6, 0xbe, 0x7d, 0xc6 }; +static CONST UINT8 EncOutput051[] = { 0x7b, 0x7c, 0xe0, 0xd8, 0x24, 0x80, 0x9a, 0x70, 0xde, 0x32, 0x56, + 0x2c, 0xcf, 0x2c, 0x2b, 0xbd, 0x15, 0xd4, 0x4a, 0x00, 0xce, 0x0d, + 0x19, 0xb4, 0x23, 0x1f, 0x92, 0x1e, 0x22, 0xbc, 0x0a, 0x43 }; +static CONST UINT8 EncAssoc051[] = { 0xd4, 0x1a, 0x82, 0x8d, 0x5e, 0x71, 0x82, 0x92, 0x47, 0x02, 0x19, + 0x05, 0x40, 0x2e, 0xa2, 0x57, 0xdc, 0xcb, 0xc3, 0xb8, 0x0f, 0xcd, + 0x56, 0x75, 0x05, 0x6b, 0x68, 0xbb, 0x59, 0xe6, 0x2e, 0x88, 0x73 }; +static CONST UINT8 EncNonce051[] = { 0xbe, 0x40, 0xe5, 0xf1, 0xa1, 0x18, 0x17, 0xa0, 0xa8, 0xfa, 0x89, 0x49 }; +static CONST UINT8 EncKey051[] = { 0x36, 0x37, 0x2a, 0xbc, 0xdb, 0x78, 0xe0, 0x27, 0x96, 0x46, 0xac, + 0x3d, 0x17, 0x6b, 0x96, 0x74, 0xe9, 0x15, 0x4e, 0xec, 0xf0, 0xd5, + 0x46, 0x9c, 0x65, 0x1e, 0xc7, 0xe1, 0x6b, 0x4c, 0x11, 0x99 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput052[] = { 0xa6, 0x67, 0x47, 0xc8, 0x9e, 0x85, 0x7a, 0xf3, + 0xa1, 0x8e, 0x2c, 0x79, 0x50, 0x00, 0x87, 0xed }; +static CONST UINT8 EncOutput052[] = { 0xca, 0x82, 0xbf, 0xf3, 0xe2, 0xf3, 0x10, 0xcc, 0xc9, 0x76, 0x67, + 0x2c, 0x44, 0x15, 0xe6, 0x9b, 0x57, 0x63, 0x8c, 0x62, 0xa5, 0xd8, + 0x5d, 0xed, 0x77, 0x4f, 0x91, 0x3c, 0x81, 0x3e, 0xa0, 0x32 }; +static CONST UINT8 EncAssoc052[] = { 0x3f, 0x2d, 0xd4, 0x9b, 0xbf, 0x09, 0xd6, 0x9a, 0x78, 0xa3, 0xd8, 0x0e, + 0xa2, 0x56, 0x66, 0x14, 0xfc, 0x37, 0x94, 0x74, 0x19, 0x6c, 0x1a, 0xae, + 0x84, 0x58, 0x3d, 0xa7, 0x3d, 0x7f, 0xf8, 0x5c, 0x6f, 0x42, 0xca, 0x42, + 0x05, 0x6a, 0x97, 0x92, 0xcc, 0x1b, 0x9f, 0xb3, 0xc7, 0xd2, 0x61 }; +static CONST UINT8 EncNonce052[] = { 0x84, 0xc8, 0x7d, 0xae, 0x4e, 0xee, 0x27, 0x73, 0x0e, 0xc3, 0x5d, 0x12 }; +static CONST UINT8 EncKey052[] = { 0x9f, 0x14, 0x79, 0xed, 0x09, 0x7d, 0x7f, 0xe5, 0x29, 0xc1, 0x1f, + 0x2f, 0x5a, 0xdd, 0x9a, 0xaf, 0xf4, 0xa1, 0xca, 0x0b, 0x68, 0x99, + 0x7a, 0x2c, 0xb7, 0xf7, 0x97, 0x49, 0xbd, 0x90, 0xaa, 0xf4 }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput053[] = { 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, 0x55, 0xd3, 0x04, + 0x84, 0x64, 0x43, 0xfe, 0xe8, 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, + 0x3b, 0x7b, 0x80, 0xe0, 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe }; +static CONST UINT8 EncOutput053[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe6, 0xd3, 0xd7, 0x32, + 0x4a, 0x1c, 0xbb, 0xa7, 0x77, 0xbb, 0xb0, 0xec, 0xdd, 0xa3, 0x78, 0x07 }; +static CONST UINT8 EncAssoc053[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; +static CONST UINT8 EncNonce053[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey053[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput054[] = { 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, 0x55, 0xd3, 0x04, 0x84, 0x64, + 0x43, 0xfe, 0xe8, 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, 0x80, 0xe0, + 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe, 0xe3, 0xbc, 0xdb, 0x5b, 0x1e, 0xde, 0xfc, + 0xfe, 0x8b, 0xcd, 0xa1, 0xb6, 0xa1, 0x5c, 0x8c, 0x2b, 0x08, 0x69, 0xff, 0xd2, + 0xec, 0x5e, 0x26, 0xe5, 0x53, 0xb7, 0xb2, 0x27, 0xfe, 0x87, 0xfd, 0xbd }; +static CONST UINT8 EncOutput054[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x06, 0x2d, 0xe6, 0x79, 0x5f, 0x27, 0x4f, 0xd2, + 0xa3, 0x05, 0xd7, 0x69, 0x80, 0xbc, 0x9c, 0xce }; +static CONST UINT8 EncAssoc054[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; +static CONST UINT8 EncNonce054[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey054[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput055[] = { + 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, 0xdf, 0x99, 0x47, + 0x03, 0x03, 0xfb, 0x3b, 0x7b, 0x80, 0xe0, 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe, 0xe3, 0xbc, 0xdb, 0x5b, 0x1e, 0xde, + 0xfc, 0xfe, 0x8b, 0xcd, 0xa1, 0xb6, 0xa1, 0x5c, 0x8c, 0x2b, 0x08, 0x69, 0xff, 0xd2, 0xec, 0x5e, 0x26, 0xe5, 0x53, + 0xb7, 0xb2, 0x27, 0xfe, 0x87, 0xfd, 0xbd, 0x7a, 0xda, 0x44, 0x42, 0x42, 0x69, 0xbf, 0xfa, 0x55, 0x27, 0xf2, 0x70, + 0xac, 0xf6, 0x85, 0x02, 0xb7, 0x4c, 0x5a, 0xe2, 0xe6, 0x0c, 0x05, 0x80, 0x98, 0x1a, 0x49, 0x38, 0x45, 0x93, 0x92, + 0xc4, 0x9b, 0xb2, 0xf2, 0x84, 0xb6, 0x46, 0xef, 0xc7, 0xf3, 0xf0, 0xb1, 0x36, 0x1d, 0xc3, 0x48, 0xed, 0x77, 0xd3, + 0x0b, 0xc5, 0x76, 0x92, 0xed, 0x38, 0xfb, 0xac, 0x01, 0x88, 0x38, 0x04, 0x88, 0xc7 +}; +static CONST UINT8 EncOutput055[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xd8, 0xb4, 0x79, 0x02, 0xba, 0xae, 0xaf, 0xb3, 0x42, 0x03, 0x05, 0x15, 0x29, 0xaf, 0x28, 0x2e +}; +static CONST UINT8 EncAssoc055[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; +static CONST UINT8 EncNonce055[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey055[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput056[] = { 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, 0xaa, 0x2c, 0xfb, + 0x7b, 0x9b, 0xbc, 0x01, 0x17, 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, + 0xc4, 0x84, 0x7f, 0x1f, 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41 }; +static CONST UINT8 EncOutput056[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xb3, 0x89, 0x1c, 0x84, + 0x9c, 0xb5, 0x2c, 0x27, 0x74, 0x7e, 0xdf, 0xcf, 0x31, 0x21, 0x3b, 0xb6 }; +static CONST UINT8 EncAssoc056[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce056[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey056[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput057[] = { 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, + 0xbc, 0x01, 0x17, 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, 0x7f, 0x1f, + 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41, 0x1c, 0x43, 0x24, 0xa4, 0xe1, 0x21, 0x03, + 0x01, 0x74, 0x32, 0x5e, 0x49, 0x5e, 0xa3, 0x73, 0xd4, 0xf7, 0x96, 0x00, 0x2d, + 0x13, 0xa1, 0xd9, 0x1a, 0xac, 0x48, 0x4d, 0xd8, 0x01, 0x78, 0x02, 0x42 }; +static CONST UINT8 EncOutput057[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xf0, 0xc1, 0x2d, 0x26, 0xef, 0x03, 0x02, 0x9b, + 0x62, 0xc0, 0x08, 0xda, 0x27, 0xc5, 0xdc, 0x68 }; +static CONST UINT8 EncAssoc057[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce057[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey057[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput058[] = { + 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, 0x20, 0x66, 0xb8, + 0xfc, 0xfc, 0x04, 0xc4, 0x84, 0x7f, 0x1f, 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41, 0x1c, 0x43, 0x24, 0xa4, 0xe1, 0x21, + 0x03, 0x01, 0x74, 0x32, 0x5e, 0x49, 0x5e, 0xa3, 0x73, 0xd4, 0xf7, 0x96, 0x00, 0x2d, 0x13, 0xa1, 0xd9, 0x1a, 0xac, + 0x48, 0x4d, 0xd8, 0x01, 0x78, 0x02, 0x42, 0x85, 0x25, 0xbb, 0xbd, 0xbd, 0x96, 0x40, 0x05, 0xaa, 0xd8, 0x0d, 0x8f, + 0x53, 0x09, 0x7a, 0xfd, 0x48, 0xb3, 0xa5, 0x1d, 0x19, 0xf3, 0xfa, 0x7f, 0x67, 0xe5, 0xb6, 0xc7, 0xba, 0x6c, 0x6d, + 0x3b, 0x64, 0x4d, 0x0d, 0x7b, 0x49, 0xb9, 0x10, 0x38, 0x0c, 0x0f, 0x4e, 0xc9, 0xe2, 0x3c, 0xb7, 0x12, 0x88, 0x2c, + 0xf4, 0x3a, 0x89, 0x6d, 0x12, 0xc7, 0x04, 0x53, 0xfe, 0x77, 0xc7, 0xfb, 0x77, 0x38 +}; +static CONST UINT8 EncOutput058[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xee, 0x65, 0x78, 0x30, 0x01, 0xc2, 0x56, 0x91, 0xfa, 0x28, 0xd0, 0xf5, 0xf1, 0xc1, 0xd7, 0x62 +}; +static CONST UINT8 EncAssoc058[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce058[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey058[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput059[] = { 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, 0x55, 0xd3, 0x04, + 0x04, 0x64, 0x43, 0xfe, 0x68, 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, + 0x3b, 0xfb, 0x80, 0xe0, 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e }; +static CONST UINT8 EncOutput059[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x79, 0xba, 0x7a, 0x29, + 0xf5, 0xa7, 0xbb, 0x75, 0x79, 0x7a, 0xf8, 0x7a, 0x61, 0x01, 0x29, 0xa4 }; +static CONST UINT8 EncAssoc059[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; +static CONST UINT8 EncNonce059[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey059[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput060[] = { 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, 0x55, 0xd3, 0x04, 0x04, 0x64, + 0x43, 0xfe, 0x68, 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, 0x80, 0xe0, + 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e, 0xe3, 0xbc, 0xdb, 0xdb, 0x1e, 0xde, 0xfc, + 0x7e, 0x8b, 0xcd, 0xa1, 0x36, 0xa1, 0x5c, 0x8c, 0xab, 0x08, 0x69, 0xff, 0x52, + 0xec, 0x5e, 0x26, 0x65, 0x53, 0xb7, 0xb2, 0xa7, 0xfe, 0x87, 0xfd, 0x3d }; +static CONST UINT8 EncOutput060[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x36, 0xb1, 0x74, 0x38, 0x19, 0xe1, 0xb9, 0xba, + 0x15, 0x51, 0xe8, 0xed, 0x92, 0x2a, 0x95, 0x9a }; +static CONST UINT8 EncAssoc060[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; +static CONST UINT8 EncNonce060[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey060[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput061[] = { + 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, 0xdf, 0x99, 0x47, + 0x83, 0x03, 0xfb, 0x3b, 0xfb, 0x80, 0xe0, 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e, 0xe3, 0xbc, 0xdb, 0xdb, 0x1e, 0xde, + 0xfc, 0x7e, 0x8b, 0xcd, 0xa1, 0x36, 0xa1, 0x5c, 0x8c, 0xab, 0x08, 0x69, 0xff, 0x52, 0xec, 0x5e, 0x26, 0x65, 0x53, + 0xb7, 0xb2, 0xa7, 0xfe, 0x87, 0xfd, 0x3d, 0x7a, 0xda, 0x44, 0xc2, 0x42, 0x69, 0xbf, 0x7a, 0x55, 0x27, 0xf2, 0xf0, + 0xac, 0xf6, 0x85, 0x82, 0xb7, 0x4c, 0x5a, 0x62, 0xe6, 0x0c, 0x05, 0x00, 0x98, 0x1a, 0x49, 0xb8, 0x45, 0x93, 0x92, + 0x44, 0x9b, 0xb2, 0xf2, 0x04, 0xb6, 0x46, 0xef, 0x47, 0xf3, 0xf0, 0xb1, 0xb6, 0x1d, 0xc3, 0x48, 0x6d, 0x77, 0xd3, + 0x0b, 0x45, 0x76, 0x92, 0xed, 0xb8, 0xfb, 0xac, 0x01, 0x08, 0x38, 0x04, 0x88, 0x47 +}; +static CONST UINT8 EncOutput061[] = { + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, + 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, + 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, + 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, + 0x00, 0x80, 0xfe, 0xac, 0x49, 0x55, 0x55, 0x4e, 0x80, 0x6f, 0x3a, 0x19, 0x02, 0xe2, 0x44, 0x32, 0xc0, 0x8a +}; +static CONST UINT8 EncAssoc061[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; +static CONST UINT8 EncNonce061[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey061[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput062[] = { 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, 0xaa, 0x2c, 0xfb, + 0xfb, 0x9b, 0xbc, 0x01, 0x97, 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, + 0xc4, 0x04, 0x7f, 0x1f, 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1 }; +static CONST UINT8 EncOutput062[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0x20, 0xa3, 0x79, 0x8d, + 0xf1, 0x29, 0x2c, 0x59, 0x72, 0xbf, 0x97, 0x41, 0xae, 0xc3, 0x8a, 0x19 }; +static CONST UINT8 EncAssoc062[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; +static CONST UINT8 EncNonce062[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey062[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput063[] = { 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, + 0xbc, 0x01, 0x97, 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, 0x7f, 0x1f, + 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1, 0x1c, 0x43, 0x24, 0x24, 0xe1, 0x21, 0x03, + 0x81, 0x74, 0x32, 0x5e, 0xc9, 0x5e, 0xa3, 0x73, 0x54, 0xf7, 0x96, 0x00, 0xad, + 0x13, 0xa1, 0xd9, 0x9a, 0xac, 0x48, 0x4d, 0x58, 0x01, 0x78, 0x02, 0xc2 }; +static CONST UINT8 EncOutput063[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xc0, 0x3d, 0x9f, 0x67, 0x35, 0x4a, 0x97, 0xb2, + 0xf0, 0x74, 0xf7, 0x55, 0x15, 0x57, 0xe4, 0x9c }; +static CONST UINT8 EncAssoc063[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; +static CONST UINT8 EncNonce063[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey063[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput064[] = { + 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, 0x20, 0x66, 0xb8, + 0x7c, 0xfc, 0x04, 0xc4, 0x04, 0x7f, 0x1f, 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1, 0x1c, 0x43, 0x24, 0x24, 0xe1, 0x21, + 0x03, 0x81, 0x74, 0x32, 0x5e, 0xc9, 0x5e, 0xa3, 0x73, 0x54, 0xf7, 0x96, 0x00, 0xad, 0x13, 0xa1, 0xd9, 0x9a, 0xac, + 0x48, 0x4d, 0x58, 0x01, 0x78, 0x02, 0xc2, 0x85, 0x25, 0xbb, 0x3d, 0xbd, 0x96, 0x40, 0x85, 0xaa, 0xd8, 0x0d, 0x0f, + 0x53, 0x09, 0x7a, 0x7d, 0x48, 0xb3, 0xa5, 0x9d, 0x19, 0xf3, 0xfa, 0xff, 0x67, 0xe5, 0xb6, 0x47, 0xba, 0x6c, 0x6d, + 0xbb, 0x64, 0x4d, 0x0d, 0xfb, 0x49, 0xb9, 0x10, 0xb8, 0x0c, 0x0f, 0x4e, 0x49, 0xe2, 0x3c, 0xb7, 0x92, 0x88, 0x2c, + 0xf4, 0xba, 0x89, 0x6d, 0x12, 0x47, 0x04, 0x53, 0xfe, 0xf7, 0xc7, 0xfb, 0x77, 0xb8 +}; +static CONST UINT8 EncOutput064[] = { + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, + 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, + 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, + 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, + 0xff, 0x7f, 0xc8, 0x6d, 0xa8, 0xdd, 0x65, 0x22, 0x86, 0xd5, 0x02, 0x13, 0xd3, 0x28, 0xd6, 0x3e, 0x40, 0x06 +}; +static CONST UINT8 EncAssoc064[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; +static CONST UINT8 EncNonce064[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey064[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput065[] = { 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, 0x2a, 0x2c, 0xfb, + 0x7b, 0x1b, 0xbc, 0x01, 0x17, 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, + 0xc4, 0x84, 0xff, 0x1f, 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41 }; +static CONST UINT8 EncOutput065[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0xbe, 0xde, 0x90, 0x83, + 0xce, 0xb3, 0x6d, 0xdf, 0xe5, 0xfa, 0x81, 0x1f, 0x95, 0x47, 0x1c, 0x67 }; +static CONST UINT8 EncAssoc065[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce065[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey065[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput066[] = { 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, + 0xbc, 0x01, 0x17, 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, 0xff, 0x1f, + 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41, 0x9c, 0x43, 0x24, 0xa4, 0x61, 0x21, 0x03, + 0x01, 0xf4, 0x32, 0x5e, 0x49, 0xde, 0xa3, 0x73, 0xd4, 0x77, 0x96, 0x00, 0x2d, + 0x93, 0xa1, 0xd9, 0x1a, 0x2c, 0x48, 0x4d, 0xd8, 0x81, 0x78, 0x02, 0x42 }; +static CONST UINT8 EncOutput066[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x30, 0x08, 0x74, 0xbb, 0x06, 0x92, 0xb6, 0x89, + 0xde, 0xad, 0x9a, 0xe1, 0x5b, 0x06, 0x73, 0x90 }; +static CONST UINT8 EncAssoc066[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce066[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey066[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput067[] = { + 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, 0xa0, 0x66, 0xb8, + 0xfc, 0x7c, 0x04, 0xc4, 0x84, 0xff, 0x1f, 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41, 0x9c, 0x43, 0x24, 0xa4, 0x61, 0x21, + 0x03, 0x01, 0xf4, 0x32, 0x5e, 0x49, 0xde, 0xa3, 0x73, 0xd4, 0x77, 0x96, 0x00, 0x2d, 0x93, 0xa1, 0xd9, 0x1a, 0x2c, + 0x48, 0x4d, 0xd8, 0x81, 0x78, 0x02, 0x42, 0x05, 0x25, 0xbb, 0xbd, 0x3d, 0x96, 0x40, 0x05, 0x2a, 0xd8, 0x0d, 0x8f, + 0xd3, 0x09, 0x7a, 0xfd, 0xc8, 0xb3, 0xa5, 0x1d, 0x99, 0xf3, 0xfa, 0x7f, 0xe7, 0xe5, 0xb6, 0xc7, 0x3a, 0x6c, 0x6d, + 0x3b, 0xe4, 0x4d, 0x0d, 0x7b, 0xc9, 0xb9, 0x10, 0x38, 0x8c, 0x0f, 0x4e, 0xc9, 0x62, 0x3c, 0xb7, 0x12, 0x08, 0x2c, + 0xf4, 0x3a, 0x09, 0x6d, 0x12, 0xc7, 0x84, 0x53, 0xfe, 0x77, 0x47, 0xfb, 0x77, 0x38 +}; +static CONST UINT8 EncOutput067[] = { + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, + 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, + 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, + 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, + 0xff, 0xff, 0x99, 0xca, 0xd8, 0x5f, 0x45, 0xca, 0x40, 0x94, 0x2d, 0x0d, 0x4d, 0x5e, 0x95, 0x0a, 0xde, 0x22 +}; +static CONST UINT8 EncAssoc067[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce067[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey067[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput068[] = { 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, 0x55, 0xd3, 0x04, + 0x84, 0x9b, 0xbc, 0x01, 0x17, 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, + 0xc4, 0x84, 0x80, 0xe0, 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41 }; +static CONST UINT8 EncOutput068[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x8b, 0xbe, 0x14, 0x52, + 0x72, 0xe7, 0xc2, 0xd9, 0xa1, 0x89, 0x1a, 0x3a, 0xb0, 0x98, 0x3d, 0x9d }; +static CONST UINT8 EncAssoc068[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce068[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey068[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput069[] = { 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, 0x55, 0xd3, 0x04, 0x84, 0x9b, + 0xbc, 0x01, 0x17, 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, 0x80, 0xe0, + 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41, 0xe3, 0xbc, 0xdb, 0x5b, 0xe1, 0x21, 0x03, + 0x01, 0x8b, 0xcd, 0xa1, 0xb6, 0x5e, 0xa3, 0x73, 0xd4, 0x08, 0x69, 0xff, 0xd2, + 0x13, 0xa1, 0xd9, 0x1a, 0x53, 0xb7, 0xb2, 0x27, 0x01, 0x78, 0x02, 0x42 }; +static CONST UINT8 EncOutput069[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x3b, 0x41, 0x86, 0x19, 0x13, 0xa8, 0xf6, 0xde, + 0x7f, 0x61, 0xe2, 0x25, 0x63, 0x1b, 0xc3, 0x82 }; +static CONST UINT8 EncAssoc069[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce069[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey069[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput070[] = { + 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, 0xdf, 0x99, 0x47, + 0x03, 0xfc, 0x04, 0xc4, 0x84, 0x80, 0xe0, 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41, 0xe3, 0xbc, 0xdb, 0x5b, 0xe1, 0x21, + 0x03, 0x01, 0x8b, 0xcd, 0xa1, 0xb6, 0x5e, 0xa3, 0x73, 0xd4, 0x08, 0x69, 0xff, 0xd2, 0x13, 0xa1, 0xd9, 0x1a, 0x53, + 0xb7, 0xb2, 0x27, 0x01, 0x78, 0x02, 0x42, 0x7a, 0xda, 0x44, 0x42, 0xbd, 0x96, 0x40, 0x05, 0x55, 0x27, 0xf2, 0x70, + 0x53, 0x09, 0x7a, 0xfd, 0xb7, 0x4c, 0x5a, 0xe2, 0x19, 0xf3, 0xfa, 0x7f, 0x98, 0x1a, 0x49, 0x38, 0xba, 0x6c, 0x6d, + 0x3b, 0x9b, 0xb2, 0xf2, 0x84, 0x49, 0xb9, 0x10, 0x38, 0xf3, 0xf0, 0xb1, 0x36, 0xe2, 0x3c, 0xb7, 0x12, 0x77, 0xd3, + 0x0b, 0xc5, 0x89, 0x6d, 0x12, 0xc7, 0xfb, 0xac, 0x01, 0x88, 0xc7, 0xfb, 0x77, 0x38 +}; +static CONST UINT8 EncOutput070[] = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xff, 0xff, 0x84, 0x28, 0xbc, 0xf0, 0x23, 0xec, 0x6b, 0xf3, 0x1f, 0xd9, 0xef, 0xb2, 0x03, 0xff, 0x08, 0x71 +}; +static CONST UINT8 EncAssoc070[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce070[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey070[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput071[] = { 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, 0xaa, 0x2c, 0xfb, + 0x7b, 0x64, 0x43, 0xfe, 0xe8, 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, + 0x3b, 0x7b, 0x7f, 0x1f, 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe }; +static CONST UINT8 EncOutput071[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x13, 0x9f, 0xdf, 0x64, + 0x74, 0xea, 0x24, 0xf5, 0x49, 0xb0, 0x75, 0x82, 0x5f, 0x2c, 0x76, 0x20 }; +static CONST UINT8 EncAssoc071[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }; +static CONST UINT8 EncNonce071[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey071[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput072[] = { 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, 0xaa, 0x2c, 0xfb, 0x7b, 0x64, + 0x43, 0xfe, 0xe8, 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, 0x7f, 0x1f, + 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe, 0x1c, 0x43, 0x24, 0xa4, 0x1e, 0xde, 0xfc, + 0xfe, 0x74, 0x32, 0x5e, 0x49, 0xa1, 0x5c, 0x8c, 0x2b, 0xf7, 0x96, 0x00, 0x2d, + 0xec, 0x5e, 0x26, 0xe5, 0xac, 0x48, 0x4d, 0xd8, 0xfe, 0x87, 0xfd, 0xbd }; +static CONST UINT8 EncOutput072[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xbb, 0xad, 0x8d, 0x86, 0x3b, 0x83, 0x5a, 0x8e, + 0x86, 0x64, 0xfd, 0x1d, 0x45, 0x66, 0xb6, 0xb4 }; +static CONST UINT8 EncAssoc072[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }; +static CONST UINT8 EncNonce072[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey072[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - misc */ +static CONST UINT8 EncInput073[] = { + 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, 0x20, 0x66, 0xb8, + 0xfc, 0x03, 0xfb, 0x3b, 0x7b, 0x7f, 0x1f, 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe, 0x1c, 0x43, 0x24, 0xa4, 0x1e, 0xde, + 0xfc, 0xfe, 0x74, 0x32, 0x5e, 0x49, 0xa1, 0x5c, 0x8c, 0x2b, 0xf7, 0x96, 0x00, 0x2d, 0xec, 0x5e, 0x26, 0xe5, 0xac, + 0x48, 0x4d, 0xd8, 0xfe, 0x87, 0xfd, 0xbd, 0x85, 0x25, 0xbb, 0xbd, 0x42, 0x69, 0xbf, 0xfa, 0xaa, 0xd8, 0x0d, 0x8f, + 0xac, 0xf6, 0x85, 0x02, 0x48, 0xb3, 0xa5, 0x1d, 0xe6, 0x0c, 0x05, 0x80, 0x67, 0xe5, 0xb6, 0xc7, 0x45, 0x93, 0x92, + 0xc4, 0x64, 0x4d, 0x0d, 0x7b, 0xb6, 0x46, 0xef, 0xc7, 0x0c, 0x0f, 0x4e, 0xc9, 0x1d, 0xc3, 0x48, 0xed, 0x88, 0x2c, + 0xf4, 0x3a, 0x76, 0x92, 0xed, 0x38, 0x04, 0x53, 0xfe, 0x77, 0x38, 0x04, 0x88, 0xc7 +}; +static CONST UINT8 EncOutput073[] = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, + 0x00, 0x00, 0x42, 0xf2, 0x35, 0x42, 0x97, 0x84, 0x9a, 0x51, 0x1d, 0x53, 0xe5, 0x57, 0x17, 0x72, 0xf7, 0x1f +}; +static CONST UINT8 EncAssoc073[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }; +static CONST UINT8 EncNonce073[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; +static CONST UINT8 EncKey073[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - checking for int overflows */ +static CONST UINT8 EncInput074[] = { + 0xd4, 0x50, 0x0b, 0xf0, 0x09, 0x49, 0x35, 0x51, 0xc3, 0x80, 0xad, 0xf5, 0x2c, 0x57, 0x3a, 0x69, 0xdf, 0x7e, 0x8b, + 0x76, 0x24, 0x63, 0x33, 0x0f, 0xac, 0xc1, 0x6a, 0x57, 0x26, 0xbe, 0x71, 0x90, 0xc6, 0x3c, 0x5a, 0x1c, 0x92, 0x65, + 0x84, 0xa0, 0x96, 0x75, 0x68, 0x28, 0xdc, 0xdc, 0x64, 0xac, 0xdf, 0x96, 0x3d, 0x93, 0x1b, 0xf1, 0xda, 0xe2, 0x38, + 0xf3, 0xf1, 0x57, 0x22, 0x4a, 0xc4, 0xb5, 0x42, 0xd7, 0x85, 0xb0, 0xdd, 0x84, 0xdb, 0x6b, 0xe3, 0xbc, 0x5a, 0x36, + 0x63, 0xe8, 0x41, 0x49, 0xff, 0xbe, 0xd0, 0x9e, 0x54, 0xf7, 0x8f, 0x16, 0xa8, 0x22, 0x3b, 0x24, 0xcb, 0x01, 0x9f, + 0x58, 0xb2, 0x1b, 0x0e, 0x55, 0x1e, 0x7a, 0xa0, 0x73, 0x27, 0x62, 0x95, 0x51, 0x37, 0x6c, 0xcb, 0xc3, 0x93, 0x76, + 0x71, 0xa0, 0x62, 0x9b, 0xd9, 0x5c, 0x99, 0x15, 0xc7, 0x85, 0x55, 0x77, 0x1e, 0x7a +}; +static CONST UINT8 EncOutput074[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x0b, 0x30, 0x0d, 0x8d, 0xa5, 0x6c, 0x21, 0x85, 0x75, 0x52, 0x79, 0x55, 0x3c, 0x4c, 0x82, 0xca +}; +static CONST UINT8 EncAssoc074[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce074[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x00, 0x02, 0x50, 0x6e }; +static CONST UINT8 EncKey074[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 }; + +/* wycheproof - checking for int overflows */ +static CONST UINT8 EncInput075[] = { + 0x7d, 0xe8, 0x7f, 0x67, 0x29, 0x94, 0x52, 0x75, 0xd0, 0x65, 0x5d, 0xa4, 0xc7, 0xfd, 0xe4, 0x56, 0x9e, 0x16, 0xf1, + 0x11, 0xb5, 0xeb, 0x26, 0xc2, 0x2d, 0x85, 0x9e, 0x3f, 0xf8, 0x22, 0xec, 0xed, 0x3a, 0x6d, 0xd9, 0xa6, 0x0f, 0x22, + 0x95, 0x7f, 0x7b, 0x7c, 0x85, 0x7e, 0x88, 0x22, 0xeb, 0x9f, 0xe0, 0xb8, 0xd7, 0x02, 0x21, 0x41, 0xf2, 0xd0, 0xb4, + 0x8f, 0x4b, 0x56, 0x12, 0xd3, 0x22, 0xa8, 0x8d, 0xd0, 0xfe, 0x0b, 0x4d, 0x91, 0x79, 0x32, 0x4f, 0x7c, 0x6c, 0x9e, + 0x99, 0x0e, 0xfb, 0xd8, 0x0e, 0x5e, 0xd6, 0x77, 0x58, 0x26, 0x49, 0x8b, 0x1e, 0xfe, 0x0f, 0x71, 0xa0, 0xf3, 0xec, + 0x5b, 0x29, 0xcb, 0x28, 0xc2, 0x54, 0x0a, 0x7d, 0xcd, 0x51, 0xb7, 0xda, 0xae, 0xe0, 0xff, 0x4a, 0x7f, 0x3a, 0xc1, + 0xee, 0x54, 0xc2, 0x9e, 0xe4, 0xc1, 0x70, 0xde, 0x40, 0x8f, 0x66, 0x69, 0x21, 0x94 +}; +static CONST UINT8 EncOutput075[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xc5, 0x78, 0xe2, 0xaa, 0x44, 0xd3, 0x09, 0xb7, 0xb6, 0xa5, 0x19, 0x3b, 0xdc, 0x61, 0x18, 0xf5 +}; +static CONST UINT8 EncAssoc075[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce075[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x00, 0x03, 0x18, 0xa5 }; +static CONST UINT8 EncKey075[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 }; + +/* wycheproof - checking for int overflows */ +static CONST UINT8 EncInput076[] = { + 0x1b, 0x99, 0x6f, 0x9a, 0x3c, 0xcc, 0x67, 0x85, 0xde, 0x22, 0xff, 0x5b, 0x8a, 0xdd, 0x95, 0x02, 0xce, 0x03, 0xa0, + 0xfa, 0xf5, 0x99, 0x2a, 0x09, 0x52, 0x2c, 0xdd, 0x12, 0x06, 0xd2, 0x20, 0xb8, 0xf8, 0xbd, 0x07, 0xd1, 0xf1, 0xf5, + 0xa1, 0xbd, 0x9a, 0x71, 0xd1, 0x1c, 0x7f, 0x57, 0x9b, 0x85, 0x58, 0x18, 0xc0, 0x8d, 0x4d, 0xe0, 0x36, 0x39, 0x31, + 0x83, 0xb7, 0xf5, 0x90, 0xb3, 0x35, 0xae, 0xd8, 0xde, 0x5b, 0x57, 0xb1, 0x3c, 0x5f, 0xed, 0xe2, 0x44, 0x1c, 0x3e, + 0x18, 0x4a, 0xa9, 0xd4, 0x6e, 0x61, 0x59, 0x85, 0x06, 0xb3, 0xe1, 0x1c, 0x43, 0xc6, 0x2c, 0xbc, 0xac, 0xec, 0xed, + 0x33, 0x19, 0x08, 0x75, 0xb0, 0x12, 0x21, 0x8b, 0x19, 0x30, 0xfb, 0x7c, 0x38, 0xec, 0x45, 0xac, 0x11, 0xc3, 0x53, + 0xd0, 0xcf, 0x93, 0x8d, 0xcc, 0xb9, 0xef, 0xad, 0x8f, 0xed, 0xbe, 0x46, 0xda, 0xa5 +}; +static CONST UINT8 EncOutput076[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x4b, 0x0b, 0xda, 0x8a, 0xd0, 0x43, 0x83, 0x0d, 0x83, 0x19, 0xab, 0x82, 0xc5, 0x0c, 0x76, 0x63 +}; +static CONST UINT8 EncAssoc076[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce076[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xb4, 0xf0 }; +static CONST UINT8 EncKey076[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 }; + +/* wycheproof - checking for int overflows */ +static CONST UINT8 EncInput077[] = { + 0x86, 0xcb, 0xac, 0xae, 0x4d, 0x3f, 0x74, 0xae, 0x01, 0x21, 0x3e, 0x05, 0x51, 0xcc, 0x15, 0x16, 0x0e, 0xa1, 0xbe, + 0x84, 0x08, 0xe3, 0xd5, 0xd7, 0x4f, 0x01, 0x46, 0x49, 0x95, 0xa6, 0x9e, 0x61, 0x76, 0xcb, 0x9e, 0x02, 0xb2, 0x24, + 0x7e, 0xd2, 0x99, 0x89, 0x2f, 0x91, 0x82, 0xa4, 0x5c, 0xaf, 0x4c, 0x69, 0x40, 0x56, 0x11, 0x76, 0x6e, 0xdf, 0xaf, + 0xdc, 0x28, 0x55, 0x19, 0xea, 0x30, 0x48, 0x0c, 0x44, 0xf0, 0x5e, 0x78, 0x1e, 0xac, 0xf8, 0xfc, 0xec, 0xc7, 0x09, + 0x0a, 0xbb, 0x28, 0xfa, 0x5f, 0xd5, 0x85, 0xac, 0x8c, 0xda, 0x7e, 0x87, 0x72, 0xe5, 0x94, 0xe4, 0xce, 0x6c, 0x88, + 0x32, 0x81, 0x93, 0x2e, 0x0f, 0x89, 0xf8, 0x77, 0xa1, 0xf0, 0x4d, 0x9c, 0x32, 0xb0, 0x6c, 0xf9, 0x0b, 0x0e, 0x76, + 0x2b, 0x43, 0x0c, 0x4d, 0x51, 0x7c, 0x97, 0x10, 0x70, 0x68, 0xf4, 0x98, 0xef, 0x7f +}; +static CONST UINT8 EncOutput077[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x4b, 0xc9, 0x8f, 0x72, 0xc4, 0x94, 0xc2, 0xa4, 0x3c, 0x2b, 0x15, 0xa1, 0x04, 0x3f, 0x1c, 0xfa +}; +static CONST UINT8 EncAssoc077[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce077[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0xfb, 0x66 }; +static CONST UINT8 EncKey077[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 }; + +/* wycheproof - checking for int overflows */ +static CONST UINT8 EncInput078[] = { + 0xfa, 0xb1, 0xcd, 0xdf, 0x4f, 0xe1, 0x98, 0xef, 0x63, 0xad, 0xd8, 0x81, 0xd6, 0xea, 0xd6, 0xc5, 0x76, 0x37, 0xbb, + 0xe9, 0x20, 0x18, 0xca, 0x7c, 0x0b, 0x96, 0xfb, 0xa0, 0x87, 0x1e, 0x93, 0x2d, 0xb1, 0xfb, 0xf9, 0x07, 0x61, 0xbe, + 0x25, 0xdf, 0x8d, 0xfa, 0xf9, 0x31, 0xce, 0x57, 0x57, 0xe6, 0x17, 0xb3, 0xd7, 0xa9, 0xf0, 0xbf, 0x0f, 0xfe, 0x5d, + 0x59, 0x1a, 0x33, 0xc1, 0x43, 0xb8, 0xf5, 0x3f, 0xd0, 0xb5, 0xa1, 0x96, 0x09, 0xfd, 0x62, 0xe5, 0xc2, 0x51, 0xa4, + 0x28, 0x1a, 0x20, 0x0c, 0xfd, 0xc3, 0x4f, 0x28, 0x17, 0x10, 0x40, 0x6f, 0x4e, 0x37, 0x62, 0x54, 0x46, 0xff, 0x6e, + 0xf2, 0x24, 0x91, 0x3d, 0xeb, 0x0d, 0x89, 0xaf, 0x33, 0x71, 0x28, 0xe3, 0xd1, 0x55, 0xd1, 0x6d, 0x3e, 0xc3, 0x24, + 0x60, 0x41, 0x43, 0x21, 0x43, 0xe9, 0xab, 0x3a, 0x6d, 0x2c, 0xcc, 0x2f, 0x4d, 0x62 +}; +static CONST UINT8 EncOutput078[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xf7, 0xe9, 0xe1, 0x51, 0xb0, 0x25, 0x33, 0xc7, 0x46, 0x58, 0xbf, 0xc7, 0x73, 0x7c, 0x68, 0x0d +}; +static CONST UINT8 EncAssoc078[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce078[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0xbb, 0x90 }; +static CONST UINT8 EncKey078[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 }; + +/* wycheproof - checking for int overflows */ +static CONST UINT8 EncInput079[] = { + 0x22, 0x72, 0x02, 0xbe, 0x7f, 0x35, 0x15, 0xe9, 0xd1, 0xc0, 0x2e, 0xea, 0x2f, 0x19, 0x50, 0xb6, 0x48, 0x1b, 0x04, + 0x8a, 0x4c, 0x91, 0x50, 0x6c, 0xb4, 0x0d, 0x50, 0x4e, 0x6c, 0x94, 0x9f, 0x82, 0xd1, 0x97, 0xc2, 0x5a, 0xd1, 0x7d, + 0xc7, 0x21, 0x65, 0x11, 0x25, 0x78, 0x2a, 0xc7, 0xa7, 0x12, 0x47, 0xfe, 0xae, 0xf3, 0x2f, 0x1f, 0x25, 0x0c, 0xe4, + 0xbb, 0x8f, 0x79, 0xac, 0xaa, 0x17, 0x9d, 0x45, 0xa7, 0xb0, 0x54, 0x5f, 0x09, 0x24, 0x32, 0x5e, 0xfa, 0x87, 0xd5, + 0xe4, 0x41, 0xd2, 0x84, 0x78, 0xc6, 0x1f, 0x22, 0x23, 0xee, 0x67, 0xc3, 0xb4, 0x1f, 0x43, 0x94, 0x53, 0x5e, 0x2a, + 0x24, 0x36, 0x9a, 0x2e, 0x16, 0x61, 0x3c, 0x45, 0x94, 0x90, 0xc1, 0x4f, 0xb1, 0xd7, 0x55, 0xfe, 0x53, 0xfb, 0xe1, + 0xee, 0x45, 0xb1, 0xb2, 0x1f, 0x71, 0x62, 0xe2, 0xfc, 0xaa, 0x74, 0x2a, 0xbe, 0xfd +}; +static CONST UINT8 EncOutput079[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x79, 0x5b, 0xcf, 0xf6, 0x47, 0xc5, 0x53, 0xc2, 0xe4, 0xeb, 0x6e, 0x0e, 0xaf, 0xd9, 0xe0, 0x4e +}; +static CONST UINT8 EncAssoc079[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce079[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x48, 0x4a }; +static CONST UINT8 EncKey079[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 }; + +/* wycheproof - checking for int overflows */ +static CONST UINT8 EncInput080[] = { + 0xfa, 0xe5, 0x83, 0x45, 0xc1, 0x6c, 0xb0, 0xf5, 0xcc, 0x53, 0x7f, 0x2b, 0x1b, 0x34, 0x69, 0xc9, 0x69, 0x46, 0x3b, + 0x3e, 0xa7, 0x1b, 0xcf, 0x6b, 0x98, 0xd6, 0x69, 0xa8, 0xe6, 0x0e, 0x04, 0xfc, 0x08, 0xd5, 0xfd, 0x06, 0x9c, 0x36, + 0x26, 0x38, 0xe3, 0x40, 0x0e, 0xf4, 0xcb, 0x24, 0x2e, 0x27, 0xe2, 0x24, 0x5e, 0x68, 0xcb, 0x9e, 0xc5, 0x83, 0xda, + 0x53, 0x40, 0xb1, 0x2e, 0xdf, 0x42, 0x3b, 0x73, 0x26, 0xad, 0x20, 0xfe, 0xeb, 0x57, 0xda, 0xca, 0x2e, 0x04, 0x67, + 0xa3, 0x28, 0x99, 0xb4, 0x2d, 0xf8, 0xe5, 0x6d, 0x84, 0xe0, 0x06, 0xbc, 0x8a, 0x7a, 0xcc, 0x73, 0x1e, 0x7c, 0x1f, + 0x6b, 0xec, 0xb5, 0x71, 0x9f, 0x70, 0x77, 0xf0, 0xd4, 0xf4, 0xc6, 0x1a, 0xb1, 0x1e, 0xba, 0xc1, 0x00, 0x18, 0x01, + 0xce, 0x33, 0xc4, 0xe4, 0xa7, 0x7d, 0x83, 0x1d, 0x3c, 0xe3, 0x4e, 0x84, 0x10, 0xe1 +}; +static CONST UINT8 EncOutput080[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x19, 0x46, 0xd6, 0x53, 0x96, 0x0f, 0x94, 0x7a, 0x74, 0xd3, 0xe8, 0x09, 0x3c, 0xf4, 0x85, 0x02 +}; +static CONST UINT8 EncAssoc080[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce080[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x2f, 0x40 }; +static CONST UINT8 EncKey080[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 }; + +/* wycheproof - checking for int overflows */ +static CONST UINT8 EncInput081[] = { + 0xeb, 0xb2, 0x16, 0xdd, 0xd7, 0xca, 0x70, 0x92, 0x15, 0xf5, 0x03, 0xdf, 0x9c, 0xe6, 0x3c, 0x5c, 0xd2, 0x19, 0x4e, + 0x7d, 0x90, 0x99, 0xe8, 0xa9, 0x0b, 0x2a, 0xfa, 0xad, 0x5e, 0xba, 0x35, 0x06, 0x99, 0x25, 0xa6, 0x03, 0xfd, 0xbc, + 0x34, 0x1a, 0xae, 0xd4, 0x15, 0x05, 0xb1, 0x09, 0x41, 0xfa, 0x38, 0x56, 0xa7, 0xe2, 0x47, 0xb1, 0x04, 0x07, 0x09, + 0x74, 0x6c, 0xfc, 0x20, 0x96, 0xca, 0xa6, 0x31, 0xb2, 0xff, 0xf4, 0x1c, 0x25, 0x05, 0x06, 0xd8, 0x89, 0xc1, 0xc9, + 0x06, 0x71, 0xad, 0xe8, 0x53, 0xee, 0x63, 0x94, 0xc1, 0x91, 0x92, 0xa5, 0xcf, 0x37, 0x10, 0xd1, 0x07, 0x30, 0x99, + 0xe5, 0xbc, 0x94, 0x65, 0x82, 0xfc, 0x0f, 0xab, 0x9f, 0x54, 0x3c, 0x71, 0x6a, 0xe2, 0x48, 0x6a, 0x86, 0x83, 0xfd, + 0xca, 0x39, 0xd2, 0xe1, 0x4f, 0x23, 0xd0, 0x0a, 0x58, 0x26, 0x64, 0xf4, 0xec, 0xb1 +}; +static CONST UINT8 EncOutput081[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x36, 0xc3, 0x00, 0x29, 0x85, 0xdd, 0x21, 0xba, 0xf8, 0x95, 0xd6, 0x33, 0x57, 0x3f, 0x12, 0xc0 +}; +static CONST UINT8 EncAssoc081[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce081[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x93, 0x35 }; +static CONST UINT8 EncKey081[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 }; + +/* wycheproof - checking for int overflows */ +static CONST UINT8 EncInput082[] = { + 0x40, 0x8a, 0xe6, 0xef, 0x1c, 0x7e, 0xf0, 0xfb, 0x2c, 0x2d, 0x61, 0x08, 0x16, 0xfc, 0x78, 0x49, 0xef, 0xa5, 0x8f, + 0x78, 0x27, 0x3f, 0x5f, 0x16, 0x6e, 0xa6, 0x5f, 0x81, 0xb5, 0x75, 0x74, 0x7d, 0x03, 0x5b, 0x30, 0x40, 0xfe, 0xde, + 0x1e, 0xb9, 0x45, 0x97, 0x88, 0x66, 0x97, 0x88, 0x40, 0x8e, 0x00, 0x41, 0x3b, 0x3e, 0x37, 0x6d, 0x15, 0x2d, 0x20, + 0x4a, 0xa2, 0xb7, 0xa8, 0x35, 0x58, 0xfc, 0xd4, 0x8a, 0x0e, 0xf7, 0xa2, 0x6b, 0x1c, 0xd6, 0xd3, 0x5d, 0x23, 0xb3, + 0xf5, 0xdf, 0xe0, 0xca, 0x77, 0xa4, 0xce, 0x32, 0xb9, 0x4a, 0xbf, 0x83, 0xda, 0x2a, 0xef, 0xca, 0xf0, 0x68, 0x38, + 0x08, 0x79, 0xe8, 0x9f, 0xb0, 0xa3, 0x82, 0x95, 0x95, 0xcf, 0x44, 0xc3, 0x85, 0x2a, 0xe2, 0xcc, 0x66, 0x2b, 0x68, + 0x9f, 0x93, 0x55, 0xd9, 0xc1, 0x83, 0x80, 0x1f, 0x6a, 0xcc, 0x31, 0x3f, 0x89, 0x07 +}; +static CONST UINT8 EncOutput082[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x65, 0x14, 0x51, 0x8e, 0x0a, 0x26, 0x41, 0x42, 0xe0, 0xb7, 0x35, 0x1f, 0x96, 0x7f, 0xc2, 0xae +}; +static CONST UINT8 EncAssoc082[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce082[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0xf7, 0xd5 }; +static CONST UINT8 EncKey082[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - checking for int overflows */ +static CONST UINT8 EncInput083[] = { + 0x0a, 0x0a, 0x24, 0x49, 0x9b, 0xca, 0xde, 0x58, 0xcf, 0x15, 0x76, 0xc3, 0x12, 0xac, 0xa9, 0x84, 0x71, 0x8c, 0xb4, + 0xcc, 0x7e, 0x01, 0x53, 0xf5, 0xa9, 0x01, 0x58, 0x10, 0x85, 0x96, 0x44, 0xdf, 0xc0, 0x21, 0x17, 0x4e, 0x0b, 0x06, + 0x0a, 0x39, 0x74, 0x48, 0xde, 0x8b, 0x48, 0x4a, 0x86, 0x03, 0xbe, 0x68, 0x0a, 0x69, 0x34, 0xc0, 0x90, 0x6f, 0x30, + 0xdd, 0x17, 0xea, 0xe2, 0xd4, 0xc5, 0xfa, 0xa7, 0x77, 0xf8, 0xca, 0x53, 0x37, 0x0e, 0x08, 0x33, 0x1b, 0x88, 0xc3, + 0x42, 0xba, 0xc9, 0x59, 0x78, 0x7b, 0xbb, 0x33, 0x93, 0x0e, 0x3b, 0x56, 0xbe, 0x86, 0xda, 0x7f, 0x2a, 0x6e, 0xb1, + 0xf9, 0x40, 0x89, 0xd1, 0xd1, 0x81, 0x07, 0x4d, 0x43, 0x02, 0xf8, 0xe0, 0x55, 0x2d, 0x0d, 0xe1, 0xfa, 0xb3, 0x06, + 0xa2, 0x1b, 0x42, 0xd4, 0xc3, 0xba, 0x6e, 0x6f, 0x0c, 0xbc, 0xc8, 0x1e, 0x87, 0x7a +}; +static CONST UINT8 EncOutput083[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x4c, 0x19, 0x4d, 0xa6, 0xa9, 0x9f, 0xd6, 0x5b, 0x40, 0xe9, 0xca, 0xd7, 0x98, 0xf4, 0x4b, 0x19 +}; +static CONST UINT8 EncAssoc083[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce083[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0xfc, 0xe4 }; +static CONST UINT8 EncKey083[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - checking for int overflows */ +static CONST UINT8 EncInput084[] = { + 0x4a, 0x0a, 0xaf, 0xf8, 0x49, 0x47, 0x29, 0x18, 0x86, 0x91, 0x70, 0x13, 0x40, 0xf3, 0xce, 0x2b, 0x8a, 0x78, 0xee, + 0xd3, 0xa0, 0xf0, 0x65, 0x99, 0x4b, 0x72, 0x48, 0x4e, 0x79, 0x91, 0xd2, 0x5c, 0x29, 0xaa, 0x07, 0x5e, 0xb1, 0xfc, + 0x16, 0xde, 0x93, 0xfe, 0x06, 0x90, 0x58, 0x11, 0x2a, 0xb2, 0x84, 0xa3, 0xed, 0x18, 0x78, 0x03, 0x26, 0xd1, 0x25, + 0x8a, 0x47, 0x22, 0x2f, 0xa6, 0x33, 0xd8, 0xb2, 0x9f, 0x3b, 0xd9, 0x15, 0x0b, 0x23, 0x9b, 0x15, 0x46, 0xc2, 0xbb, + 0x9b, 0x9f, 0x41, 0x0f, 0xeb, 0xea, 0xd3, 0x96, 0x00, 0x0e, 0xe4, 0x77, 0x70, 0x15, 0x32, 0xc3, 0xd0, 0xf5, 0xfb, + 0xf8, 0x95, 0xd2, 0x80, 0x19, 0x6d, 0x2f, 0x73, 0x7c, 0x5e, 0x9f, 0xec, 0x50, 0xd9, 0x2b, 0xb0, 0xdf, 0x5d, 0x7e, + 0x51, 0x3b, 0xe5, 0xb8, 0xea, 0x97, 0x13, 0x10, 0xd5, 0xbf, 0x16, 0xba, 0x7a, 0xee +}; +static CONST UINT8 EncOutput084[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xc8, 0xae, 0x77, 0x88, 0xcd, 0x28, 0x74, 0xab, 0xc1, 0x38, 0x54, 0x1e, 0x11, 0xfd, 0x05, 0x87 +}; +static CONST UINT8 EncAssoc084[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce084[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0x84, 0x86, 0xa8 }; +static CONST UINT8 EncKey084[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - checking for int overflows */ +static CONST UINT8 EncInput085[] = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x78, 0x3d, 0x35, + 0xf6, 0x13, 0xe6, 0xd9, 0x09, 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, 0x9f, 0x7a, 0xed, 0x35, 0xcb, 0x5a, + 0x2f, 0xca, 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, 0x9c, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, 0x6e, + 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, 0x47, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, 0xb0, 0x23, 0xb8, 0xb2, + 0x88, 0x22, 0x27, 0x73, 0xd4, 0xd2, 0x06, 0x61, 0x6f, 0x92, 0x93, 0xf6, 0x5b, 0x45, 0xdb, 0xbc, 0x74, 0xe7, 0xc2, + 0xed, 0xfb, 0xcb, 0xbf, 0x1c, 0xfb, 0x67, 0x9b, 0xb7, 0x39, 0xa5, 0x86, 0x2d, 0xe2, 0xbc, 0xb9, 0x37, 0xf7, 0x4d, + 0x5b, 0xf8, 0x67, 0x1c, 0x5a, 0x8a, 0x50, 0x92, 0xf6, 0x1d, 0x54, 0xc9, 0xaa, 0x5b +}; +static CONST UINT8 EncOutput085[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x93, 0x3a, 0x51, 0x63, 0xc7, 0xf6, 0x23, 0x68, 0x32, 0x7b, 0x3f, 0xbc, 0x10, 0x36, 0xc9, 0x43 +}; +static CONST UINT8 EncAssoc085[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce085[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey085[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - special case tag */ +static CONST UINT8 EncInput086[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, + 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, 0xc0, 0xe2, + 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, + 0x19, 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, 0xe7, 0x4a, 0x71, 0x52, + 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; +static CONST UINT8 EncOutput086[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }; +static CONST UINT8 EncAssoc086[] = { 0x85, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xa6, 0x90, 0x2f, 0xcb, 0xc8, 0x83, + 0xbb, 0xc1, 0x80, 0xb2, 0x56, 0xae, 0x34, 0xad, 0x7f, 0x00 }; +static CONST UINT8 EncNonce086[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; +static CONST UINT8 EncKey086[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - special case tag */ +static CONST UINT8 EncInput087[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, + 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, 0xc0, 0xe2, + 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, + 0x19, 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, 0xe7, 0x4a, 0x71, 0x52, + 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; +static CONST UINT8 EncOutput087[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; +static CONST UINT8 EncAssoc087[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x24, 0x7e, 0x50, 0x64, 0x2a, 0x1c, + 0x0a, 0x2f, 0x8f, 0x77, 0x21, 0x96, 0x09, 0xdb, 0xa9, 0x58 }; +static CONST UINT8 EncNonce087[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; +static CONST UINT8 EncKey087[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - special case tag */ +static CONST UINT8 EncInput088[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, + 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, 0xc0, 0xe2, + 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, + 0x19, 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, 0xe7, 0x4a, 0x71, 0x52, + 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; +static CONST UINT8 EncOutput088[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncAssoc088[] = { 0x7c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xd9, 0xe7, 0x2c, 0x06, 0x4a, 0xc8, + 0x96, 0x1f, 0x3f, 0xa5, 0x85, 0xe0, 0xe2, 0xab, 0xd6, 0x00 }; +static CONST UINT8 EncNonce088[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; +static CONST UINT8 EncKey088[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - special case tag */ +static CONST UINT8 EncInput089[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, + 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, 0xc0, 0xe2, + 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, + 0x19, 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, 0xe7, 0x4a, 0x71, 0x52, + 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; +static CONST UINT8 EncOutput089[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; +static CONST UINT8 EncAssoc089[] = { 0x65, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x95, 0xaf, 0x0f, 0x4d, 0x0b, 0x68, + 0x6e, 0xae, 0xcc, 0xca, 0x43, 0x07, 0xd5, 0x96, 0xf5, 0x02 }; +static CONST UINT8 EncNonce089[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; +static CONST UINT8 EncKey089[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - special case tag */ +static CONST UINT8 EncInput090[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, + 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, 0xc0, 0xe2, + 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, + 0x19, 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, 0xe7, 0x4a, 0x71, 0x52, + 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; +static CONST UINT8 EncOutput090[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; +static CONST UINT8 EncAssoc090[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x85, 0x40, 0xb4, 0x64, 0x35, 0x77, + 0x07, 0xbe, 0x3a, 0x39, 0xd5, 0x5c, 0x34, 0xf8, 0xbc, 0xb3 }; +static CONST UINT8 EncNonce090[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; +static CONST UINT8 EncKey090[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - special case tag */ +static CONST UINT8 EncInput091[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, + 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, 0xc0, 0xe2, + 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, + 0x19, 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, 0xe7, 0x4a, 0x71, 0x52, + 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; +static CONST UINT8 EncOutput091[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00 }; +static CONST UINT8 EncAssoc091[] = { 0x4f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x66, 0x23, 0xd9, 0x90, 0xb8, 0x98, + 0xd8, 0x30, 0xd2, 0x12, 0xaf, 0x23, 0x83, 0x33, 0x07, 0x01 }; +static CONST UINT8 EncNonce091[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; +static CONST UINT8 EncKey091[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - special case tag */ +static CONST UINT8 EncInput092[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, + 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, 0xc0, 0xe2, + 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, + 0x19, 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, 0xe7, 0x4a, 0x71, 0x52, + 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; +static CONST UINT8 EncOutput092[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; +static CONST UINT8 EncAssoc092[] = { 0x83, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x5f, 0x16, 0xd0, 0x9f, 0x17, 0x78, + 0x72, 0x11, 0xb7, 0xd4, 0x84, 0xe0, 0x24, 0xf8, 0x97, 0x01 }; +static CONST UINT8 EncNonce092[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; +static CONST UINT8 EncKey092[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput093[] = { 0x00, 0x52, 0x35, 0xd2, 0xa9, 0x19, 0xf2, 0x8d, 0x3d, 0xb7, 0x66, 0x4a, 0x34, 0xae, + 0x6b, 0x44, 0x4d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, 0x3d, 0x38, 0xe9, 0x75, + 0xc3, 0x8f, 0xe3, 0xb8, 0x5b, 0x8b, 0x94, 0x50, 0x9e, 0x2b, 0x74, 0xa3, 0x6d, 0x34, + 0x6e, 0x33, 0xd5, 0x72, 0x65, 0x9b, 0xa9, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, 0x83, 0xdc, 0xe9, 0xf3, 0x07, 0x3e, + 0xfa, 0xdb, 0x7d, 0x23, 0xb8, 0x7a, 0xce, 0x35, 0x16, 0x8c }; +static CONST UINT8 EncOutput093[] = { 0x00, 0x39, 0xe2, 0xfd, 0x2f, 0xd3, 0x12, 0x14, 0x9e, 0x98, 0x98, 0x80, + 0x88, 0x48, 0x13, 0xe7, 0xca, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3b, 0x0e, 0x86, 0x9a, + 0xaa, 0x8e, 0xa4, 0x96, 0x32, 0xff, 0xff, 0x37, 0xb9, 0xe8, 0xce, 0x00, + 0xca, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x3b, 0x0e, 0x86, 0x9a, 0xaa, 0x8e, 0xa4, 0x96, + 0x32, 0xff, 0xff, 0x37, 0xb9, 0xe8, 0xce, 0x00, 0xa5, 0x19, 0xac, 0x1a, + 0x35, 0xb4, 0xa5, 0x77, 0x87, 0x51, 0x0a, 0xf7, 0x8d, 0x8d, 0x20, 0x0a }; +static CONST UINT8 EncAssoc093[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce093[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey093[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput094[] = { 0xd3, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, + 0x87, 0x5c, 0xe5, 0xda, 0x78, 0x76, 0x6f, 0xa1, 0x92, 0x90, 0xc0, 0x31, 0xf7, 0x52, + 0x08, 0x50, 0x67, 0x45, 0xae, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, 0xa0, 0x34, + 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, 0x49, 0x6d, 0xde, 0xb0, 0x55, 0x09, 0xc6, 0xef, + 0xff, 0xab, 0x75, 0xeb, 0x2d, 0xf4, 0xab, 0x09, 0x76, 0x2d, 0x90, 0x96, 0x52, 0x4f, + 0xa1, 0xb2, 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, 0x01, 0x49, 0xef, 0x50, + 0x4b, 0x71, 0xb1, 0x20, 0xca, 0x4f, 0xf3, 0x95, 0x19, 0xc2, 0xc2, 0x10 }; +static CONST UINT8 EncOutput094[] = { + 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x62, 0x18, 0xb2, + 0x7f, 0x83, 0xb8, 0xb4, 0x66, 0x02, 0xf6, 0xe1, 0xd8, 0x34, 0x20, 0x7b, 0x02, 0xce, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x2a, 0x64, 0x16, 0xce, 0xdb, 0x1c, 0xdd, 0x29, 0x6e, + 0xf5, 0xd7, 0xd6, 0x92, 0xda, 0xff, 0x02, 0xce, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x2a, 0x64, 0x16, 0xce, 0xdb, 0x1c, 0xdd, 0x29, 0x6e, 0xf5, 0xd7, 0xd6, 0x92, 0xda, 0xff, + 0x02, 0x30, 0x2f, 0xe8, 0x2a, 0xb0, 0xa0, 0x9a, 0xf6, 0x44, 0x00, 0xd0, 0x15, 0xae, 0x83, 0xd9, 0xcc +}; +static CONST UINT8 EncAssoc094[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce094[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey094[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput095[] = { 0xe9, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, + 0x87, 0x5c, 0x6d, 0xf1, 0x39, 0x4e, 0xdc, 0x53, 0x9b, 0x5b, 0x3a, 0x09, 0x57, 0xbe, + 0x0f, 0xb8, 0x59, 0x46, 0x80, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, 0xa0, 0x34, + 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, 0xd1, 0x76, 0x9f, 0xe8, 0x06, 0xbb, 0xfe, 0xb6, + 0xf5, 0x90, 0x95, 0x0f, 0x2e, 0xac, 0x9e, 0x0a, 0x58, 0x2d, 0x90, 0x96, 0x52, 0x4f, + 0xa1, 0xb2, 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, 0x99, 0x52, 0xae, 0x08, + 0x18, 0xc3, 0x89, 0x79, 0xc0, 0x74, 0x13, 0x71, 0x1a, 0x9a, 0xf7, 0x13 }; +static CONST UINT8 EncOutput095[] = { + 0xe9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xea, 0x33, 0xf3, + 0x47, 0x30, 0x4a, 0xbd, 0xad, 0xf8, 0xce, 0x41, 0x34, 0x33, 0xc8, 0x45, 0x01, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xb2, 0x7f, 0x57, 0x96, 0x88, 0xae, 0xe5, 0x70, 0x64, + 0xce, 0x37, 0x32, 0x91, 0x82, 0xca, 0x01, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xb2, 0x7f, 0x57, 0x96, 0x88, 0xae, 0xe5, 0x70, 0x64, 0xce, 0x37, 0x32, 0x91, 0x82, 0xca, + 0x01, 0x98, 0xa7, 0xe8, 0x36, 0xe0, 0xee, 0x4d, 0x02, 0x35, 0x00, 0xd0, 0x55, 0x7e, 0xc2, 0xcb, 0xe0 +}; +static CONST UINT8 EncAssoc095[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce095[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey095[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput096[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, + 0x19, 0x87, 0x5c, 0x64, 0xf9, 0x0f, 0x5b, 0x26, 0x92, 0xb8, 0x60, 0xd4, 0x59, + 0x6f, 0xf4, 0xb3, 0x40, 0x2c, 0x5c, 0x00, 0xb9, 0xbb, 0x53, 0x70, 0x7a, 0xa6, + 0x67, 0xd3, 0x56, 0xfe, 0x50, 0xc7, 0x19, 0x96, 0x94, 0x03, 0x35, 0x61, 0xe7, + 0xca, 0xca, 0x6d, 0x94, 0x1d, 0xc3, 0xcd, 0x69, 0x14, 0xad, 0x69, 0x04 }; +static CONST UINT8 EncOutput096[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xe3, 0x3b, 0xc5, 0x52, 0xca, 0x8b, 0x9e, 0x96, + 0x16, 0x9e, 0x79, 0x7e, 0x8f, 0x30, 0x30, 0x1b, 0x60, 0x3c, 0xa9, 0x99, + 0x44, 0xdf, 0x76, 0x52, 0x8c, 0x9d, 0x6f, 0x54, 0xab, 0x83, 0x3d, 0x0f, + 0x60, 0x3c, 0xa9, 0x99, 0x44, 0xdf, 0x76, 0x52, 0x8c, 0x9d, 0x6f, 0x54, + 0xab, 0x83, 0x3d, 0x0f, 0x6a, 0xb8, 0xdc, 0xe2, 0xc5, 0x9d, 0xa4, 0x73, + 0x71, 0x30, 0xb0, 0x25, 0x2f, 0x68, 0xa8, 0xd8 }; +static CONST UINT8 EncAssoc096[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce096[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey096[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput097[] = { 0x68, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, + 0x87, 0x5c, 0xb0, 0x8f, 0x25, 0x67, 0x5b, 0x9b, 0xcb, 0xf6, 0xe3, 0x84, 0x07, 0xde, + 0x2e, 0xc7, 0x5a, 0x47, 0x9f, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, 0xa0, 0x34, + 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, 0x2d, 0x2a, 0xf7, 0xcd, 0x6b, 0x08, 0x05, 0x01, + 0xd3, 0x1b, 0xa5, 0x4f, 0xb2, 0xeb, 0x75, 0x96, 0x47, 0x2d, 0x90, 0x96, 0x52, 0x4f, + 0xa1, 0xb2, 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, 0x65, 0x0e, 0xc6, 0x2d, + 0x75, 0x70, 0x72, 0xce, 0xe6, 0xff, 0x23, 0x31, 0x86, 0xdd, 0x1c, 0x8f }; +static CONST UINT8 EncOutput097[] = { + 0x68, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x37, 0x4d, 0xef, + 0x6e, 0xb7, 0x82, 0xed, 0x00, 0x21, 0x43, 0x11, 0x54, 0x12, 0xb7, 0x46, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x4e, 0x23, 0x3f, 0xb3, 0xe5, 0x1d, 0x1e, 0xc7, 0x42, + 0x45, 0x07, 0x72, 0x0d, 0xc5, 0x21, 0x9d, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x4e, 0x23, 0x3f, 0xb3, 0xe5, 0x1d, 0x1e, 0xc7, 0x42, 0x45, 0x07, 0x72, 0x0d, 0xc5, 0x21, + 0x9d, 0x04, 0x4d, 0xea, 0x60, 0x88, 0x80, 0x41, 0x2b, 0xfd, 0xff, 0xcf, 0x35, 0x57, 0x9e, 0x9b, 0x26 +}; +static CONST UINT8 EncAssoc097[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce097[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey097[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput098[] = { 0x6d, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, + 0x87, 0x5c, 0xa1, 0x61, 0xb5, 0xab, 0x04, 0x09, 0x00, 0x62, 0x9e, 0xfe, 0xff, 0x78, + 0xd7, 0xd8, 0x6b, 0x45, 0x9f, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, 0xa0, 0x34, + 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, 0xc6, 0xf8, 0x07, 0x8c, 0xc8, 0xef, 0x12, 0xa0, + 0xff, 0x65, 0x7d, 0x6d, 0x08, 0xdb, 0x10, 0xb8, 0x47, 0x2d, 0x90, 0x96, 0x52, 0x4f, + 0xa1, 0xb2, 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, 0x8e, 0xdc, 0x36, 0x6c, + 0xd6, 0x97, 0x65, 0x6f, 0xca, 0x81, 0xfb, 0x13, 0x3c, 0xed, 0x79, 0xa1 }; +static CONST UINT8 EncOutput098[] = { + 0x6d, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x26, 0xa3, 0x7f, + 0xa2, 0xe8, 0x10, 0x26, 0x94, 0x5c, 0x39, 0xe9, 0xf2, 0xeb, 0xa8, 0x77, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa5, 0xf1, 0xcf, 0xf2, 0x46, 0xfa, 0x09, 0x66, 0x6e, + 0x3b, 0xdf, 0x50, 0xb7, 0xf5, 0x44, 0xb3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xa5, 0xf1, 0xcf, 0xf2, 0x46, 0xfa, 0x09, 0x66, 0x6e, 0x3b, 0xdf, 0x50, 0xb7, 0xf5, 0x44, + 0xb3, 0x1e, 0x6b, 0xea, 0x63, 0x14, 0x54, 0x2e, 0x2e, 0xf9, 0xff, 0xcf, 0x45, 0x0b, 0x2e, 0x98, 0x2b +}; +static CONST UINT8 EncAssoc098[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce098[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey098[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput099[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, + 0x19, 0x87, 0x5c, 0xfc, 0x01, 0xb8, 0x91, 0xe5, 0xf0, 0xf9, 0x12, 0x8d, 0x7d, + 0x1c, 0x57, 0x91, 0x92, 0xb6, 0x98, 0x63, 0x41, 0x44, 0x15, 0xb6, 0x99, 0x68, + 0x95, 0x9a, 0x72, 0x91, 0xb7, 0xa5, 0xaf, 0x13, 0x48, 0x60, 0xcd, 0x9e, 0xa1, + 0x0c, 0x29, 0xa3, 0x66, 0x54, 0xe7, 0xa2, 0x8e, 0x76, 0x1b, 0xec, 0xd8 }; +static CONST UINT8 EncOutput099[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x7b, 0xc3, 0x72, 0x98, 0x09, 0xe9, 0xdf, 0xe4, + 0x4f, 0xba, 0x0a, 0xdd, 0xad, 0xe2, 0xaa, 0xdf, 0x03, 0xc4, 0x56, 0xdf, + 0x82, 0x3c, 0xb8, 0xa0, 0xc5, 0xb9, 0x00, 0xb3, 0xc9, 0x35, 0xb8, 0xd3, + 0x03, 0xc4, 0x56, 0xdf, 0x82, 0x3c, 0xb8, 0xa0, 0xc5, 0xb9, 0x00, 0xb3, + 0xc9, 0x35, 0xb8, 0xd3, 0xed, 0x20, 0x17, 0xc8, 0xdb, 0xa4, 0x77, 0x56, + 0x29, 0x04, 0x9d, 0x78, 0x6e, 0x3b, 0xce, 0xb1 }; +static CONST UINT8 EncAssoc099[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce099[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey099[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput100[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, + 0x19, 0x87, 0x5c, 0x6b, 0x6d, 0xc9, 0xd2, 0x1a, 0x81, 0x9e, 0x70, 0xb5, 0x77, + 0xf4, 0x41, 0x37, 0xd3, 0xd6, 0xbd, 0x13, 0x35, 0xf5, 0xeb, 0x44, 0x49, 0x40, + 0x77, 0xb2, 0x64, 0x49, 0xa5, 0x4b, 0x6c, 0x7c, 0x75, 0x10, 0xb9, 0x2f, 0x5f, + 0xfe, 0xf9, 0x8b, 0x84, 0x7c, 0xf1, 0x7a, 0x9c, 0x98, 0xd8, 0x83, 0xe5 }; +static CONST UINT8 EncOutput100[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xec, 0xaf, 0x03, 0xdb, 0xf6, 0x98, 0xb8, 0x86, + 0x77, 0xb0, 0xe2, 0xcb, 0x0b, 0xa3, 0xca, 0xfa, 0x73, 0xb0, 0xe7, 0x21, + 0x70, 0xec, 0x90, 0x42, 0xed, 0xaf, 0xd8, 0xa1, 0x27, 0xf6, 0xd7, 0xee, + 0x73, 0xb0, 0xe7, 0x21, 0x70, 0xec, 0x90, 0x42, 0xed, 0xaf, 0xd8, 0xa1, + 0x27, 0xf6, 0xd7, 0xee, 0x07, 0x3f, 0x17, 0xcb, 0x67, 0x78, 0x64, 0x59, + 0x25, 0x04, 0x9d, 0x88, 0x22, 0xcb, 0xca, 0xb6 }; +static CONST UINT8 EncAssoc100[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce100[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey100[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput101[] = { 0xff, 0xcb, 0x2b, 0x11, 0x06, 0xf8, 0x23, 0x4c, 0x5e, 0x99, 0xd4, 0xdb, 0x4c, 0x70, + 0x48, 0xde, 0x32, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, 0x3d, 0x38, 0xe9, 0x75, + 0xc3, 0x8f, 0xe3, 0xb8, 0x16, 0xe9, 0x88, 0x4a, 0x11, 0x4f, 0x0e, 0x92, 0x66, 0xce, + 0xa3, 0x88, 0x5f, 0xe3, 0x6b, 0x9f, 0xd6, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, 0xce, 0xbe, 0xf5, 0xe9, 0x88, 0x5a, + 0x80, 0xea, 0x76, 0xd9, 0x75, 0xc1, 0x44, 0xa4, 0x18, 0x88 }; +static CONST UINT8 EncOutput101[] = { 0xff, 0xa0, 0xfc, 0x3e, 0x80, 0x32, 0xc3, 0xd5, 0xfd, 0xb6, 0x2a, 0x11, + 0xf0, 0x96, 0x30, 0x7d, 0xb5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x76, 0x6c, 0x9a, 0x80, + 0x25, 0xea, 0xde, 0xa7, 0x39, 0x05, 0x32, 0x8c, 0x33, 0x79, 0xc0, 0x04, + 0xb5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x76, 0x6c, 0x9a, 0x80, 0x25, 0xea, 0xde, 0xa7, + 0x39, 0x05, 0x32, 0x8c, 0x33, 0x79, 0xc0, 0x04, 0x8b, 0x9b, 0xb4, 0xb4, + 0x86, 0x12, 0x89, 0x65, 0x8c, 0x69, 0x6a, 0x83, 0x40, 0x15, 0x04, 0x05 }; +static CONST UINT8 EncAssoc101[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce101[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey101[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput102[] = { 0x6f, 0x9e, 0x70, 0xed, 0x3b, 0x8b, 0xac, 0xa0, 0x26, 0xe4, 0x6a, 0x5a, 0x09, 0x43, + 0x15, 0x8d, 0x21, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, 0x3d, 0x38, 0xe9, 0x75, + 0xc3, 0x8f, 0xe3, 0xb8, 0x0c, 0x61, 0x2c, 0x5e, 0x8d, 0x89, 0xa8, 0x73, 0xdb, 0xca, + 0xad, 0x5b, 0x73, 0x46, 0x42, 0x9b, 0xc5, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, 0xd4, 0x36, 0x51, 0xfd, 0x14, 0x9c, + 0x26, 0x0b, 0xcb, 0xdd, 0x7b, 0x12, 0x68, 0x01, 0x31, 0x8c }; +static CONST UINT8 EncOutput102[] = { 0x6f, 0xf5, 0xa7, 0xc2, 0xbd, 0x41, 0x4c, 0x39, 0x85, 0xcb, 0x94, 0x90, + 0xb5, 0xa5, 0x6d, 0x2e, 0xa6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x6c, 0xe4, 0x3e, 0x94, + 0xb9, 0x2c, 0x78, 0x46, 0x84, 0x01, 0x3c, 0x5f, 0x1f, 0xdc, 0xe9, 0x00, + 0xa6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x6c, 0xe4, 0x3e, 0x94, 0xb9, 0x2c, 0x78, 0x46, + 0x84, 0x01, 0x3c, 0x5f, 0x1f, 0xdc, 0xe9, 0x00, 0x8b, 0x3b, 0xbd, 0x51, + 0x64, 0x44, 0x59, 0x56, 0x8d, 0x81, 0xca, 0x1f, 0xa7, 0x2c, 0xe4, 0x04 }; +static CONST UINT8 EncAssoc102[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce102[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey102[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput103[] = { 0x41, 0x2b, 0x08, 0x0a, 0x3e, 0x19, 0xc1, 0x0d, 0x44, 0xa1, 0xaf, 0x1e, 0xab, 0xde, + 0xb4, 0xce, 0x35, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, 0x3d, 0x38, 0xe9, 0x75, + 0xc3, 0x8f, 0xe3, 0xb8, 0x6b, 0x83, 0x94, 0x33, 0x09, 0x21, 0x48, 0x6c, 0xa1, 0x1d, + 0x29, 0x1c, 0x3e, 0x97, 0xee, 0x9a, 0xd1, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, 0xb3, 0xd4, 0xe9, 0x90, 0x90, 0x34, + 0xc6, 0x14, 0xb1, 0x0a, 0xff, 0x55, 0x25, 0xd0, 0x9d, 0x8d }; +static CONST UINT8 EncOutput103[] = { 0x41, 0x40, 0xdf, 0x25, 0xb8, 0xd3, 0x21, 0x94, 0xe7, 0x8e, 0x51, 0xd4, + 0x17, 0x38, 0xcc, 0x6d, 0xb2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0b, 0x06, 0x86, 0xf9, + 0x3d, 0x84, 0x98, 0x59, 0xfe, 0xd6, 0xb8, 0x18, 0x52, 0x0d, 0x45, 0x01, + 0xb2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x0b, 0x06, 0x86, 0xf9, 0x3d, 0x84, 0x98, 0x59, + 0xfe, 0xd6, 0xb8, 0x18, 0x52, 0x0d, 0x45, 0x01, 0x86, 0xfb, 0xab, 0x2b, + 0x4a, 0x94, 0xf4, 0x7a, 0xa5, 0x6f, 0x0a, 0xea, 0x65, 0xd1, 0x10, 0x08 }; +static CONST UINT8 EncAssoc103[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce103[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey103[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput104[] = { 0xb2, 0x47, 0xa7, 0x47, 0x23, 0x49, 0x1a, 0xac, 0xac, 0xaa, 0xd7, 0x09, 0xc9, 0x1e, + 0x93, 0x2b, 0x31, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, 0x3d, 0x38, 0xe9, 0x75, + 0xc3, 0x8f, 0xe3, 0xb8, 0x9a, 0xde, 0x04, 0xe7, 0x5b, 0xb7, 0x01, 0xd9, 0x66, 0x06, + 0x01, 0xb3, 0x47, 0x65, 0xde, 0x98, 0xd5, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, 0x42, 0x89, 0x79, 0x44, 0xc2, 0xa2, + 0x8f, 0xa1, 0x76, 0x11, 0xd7, 0xfa, 0x5c, 0x22, 0xad, 0x8f }; +static CONST UINT8 EncOutput104[] = { 0xb2, 0x2c, 0x70, 0x68, 0xa5, 0x83, 0xfa, 0x35, 0x0f, 0x85, 0x29, 0xc3, + 0x75, 0xf8, 0xeb, 0x88, 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfa, 0x5b, 0x16, 0x2d, + 0x6f, 0x12, 0xd1, 0xec, 0x39, 0xcd, 0x90, 0xb7, 0x2b, 0xff, 0x75, 0x03, + 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xfa, 0x5b, 0x16, 0x2d, 0x6f, 0x12, 0xd1, 0xec, + 0x39, 0xcd, 0x90, 0xb7, 0x2b, 0xff, 0x75, 0x03, 0xa0, 0x19, 0xac, 0x2e, + 0xd6, 0x67, 0xe1, 0x7d, 0xa1, 0x6f, 0x0a, 0xfa, 0x19, 0x61, 0x0d, 0x0d }; +static CONST UINT8 EncAssoc104[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce104[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey104[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput105[] = { 0x74, 0x0f, 0x9e, 0x49, 0xf6, 0x10, 0xef, 0xa5, 0x85, 0xb6, 0x59, 0xca, 0x6e, 0xd8, + 0xb4, 0x99, 0x2d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, 0x3d, 0x38, 0xe9, 0x75, + 0xc3, 0x8f, 0xe3, 0xb8, 0x41, 0x2d, 0x96, 0xaf, 0xbe, 0x80, 0xec, 0x3e, 0x79, 0xd4, + 0x51, 0xb0, 0x0a, 0x2d, 0xb2, 0x9a, 0xc9, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, 0x99, 0x7a, 0xeb, 0x0c, 0x27, 0x95, + 0x62, 0x46, 0x69, 0xc3, 0x87, 0xf9, 0x11, 0x6a, 0xc1, 0x8d }; +static CONST UINT8 EncOutput105[] = { 0x74, 0x64, 0x49, 0x66, 0x70, 0xda, 0x0f, 0x3c, 0x26, 0x99, 0xa7, 0x00, + 0xd2, 0x3e, 0xcc, 0x3a, 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x21, 0xa8, 0x84, 0x65, + 0x8a, 0x25, 0x3c, 0x0b, 0x26, 0x1f, 0xc0, 0xb4, 0x66, 0xb7, 0x19, 0x01, + 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x21, 0xa8, 0x84, 0x65, 0x8a, 0x25, 0x3c, 0x0b, + 0x26, 0x1f, 0xc0, 0xb4, 0x66, 0xb7, 0x19, 0x01, 0x73, 0x6e, 0x18, 0x18, + 0x16, 0x96, 0xa5, 0x88, 0x9c, 0x31, 0x59, 0xfa, 0xab, 0xab, 0x20, 0xfd }; +static CONST UINT8 EncAssoc105[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce105[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey105[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput106[] = { 0xad, 0xba, 0x5d, 0x10, 0x5b, 0xc8, 0xaa, 0x06, 0x2c, 0x23, 0x36, 0xcb, 0x88, 0x9d, + 0xdb, 0xd5, 0x37, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, 0x3d, 0x38, 0xe9, 0x75, + 0xc3, 0x8f, 0xe3, 0xb8, 0x17, 0x7c, 0x5f, 0xfe, 0x28, 0x75, 0xf4, 0x68, 0xf6, 0xc2, + 0x96, 0x57, 0x48, 0xf3, 0x59, 0x9a, 0xd3, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, 0xcf, 0x2b, 0x22, 0x5d, 0xb1, 0x60, + 0x7a, 0x10, 0xe6, 0xd5, 0x40, 0x1e, 0x53, 0xb4, 0x2a, 0x8d }; +static CONST UINT8 EncOutput106[] = { 0xad, 0xd1, 0x8a, 0x3f, 0xdd, 0x02, 0x4a, 0x9f, 0x8f, 0x0c, 0xc8, 0x01, + 0x34, 0x7b, 0xa3, 0x76, 0xb0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x77, 0xf9, 0x4d, 0x34, + 0x1c, 0xd0, 0x24, 0x5d, 0xa9, 0x09, 0x07, 0x53, 0x24, 0x69, 0xf2, 0x01, + 0xb0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x77, 0xf9, 0x4d, 0x34, 0x1c, 0xd0, 0x24, 0x5d, + 0xa9, 0x09, 0x07, 0x53, 0x24, 0x69, 0xf2, 0x01, 0xba, 0xd5, 0x8f, 0x10, + 0xa9, 0x1e, 0x6a, 0x88, 0x9a, 0xba, 0x32, 0xfd, 0x17, 0xd8, 0x33, 0x1a }; +static CONST UINT8 EncAssoc106[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce106[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey106[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput107[] = { 0xfe, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, + 0x87, 0x5c, 0xc0, 0x01, 0xed, 0xc5, 0xda, 0x44, 0x2e, 0x71, 0x9b, 0xce, 0x9a, 0xbe, + 0x27, 0x3a, 0xf1, 0x44, 0xb4, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, 0xa0, 0x34, + 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, 0x48, 0x02, 0x5f, 0x41, 0xfa, 0x4e, 0x33, 0x6c, + 0x78, 0x69, 0x57, 0xa2, 0xa7, 0xc4, 0x93, 0x0a, 0x6c, 0x2d, 0x90, 0x96, 0x52, 0x4f, + 0xa1, 0xb2, 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, 0x00, 0x26, 0x6e, 0xa1, + 0xe4, 0x36, 0x44, 0xa3, 0x4d, 0x8d, 0xd1, 0xdc, 0x93, 0xf2, 0xfa, 0x13 }; +static CONST UINT8 EncOutput107[] = { + 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x47, 0xc3, 0x27, + 0xcc, 0x36, 0x5d, 0x08, 0x87, 0x59, 0x09, 0x8c, 0x34, 0x1b, 0x4a, 0xed, 0x03, 0xd4, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x2b, 0x0b, 0x97, 0x3f, 0x74, 0x5b, 0x28, 0xaa, 0xe9, + 0x37, 0xf5, 0x9f, 0x18, 0xea, 0xc7, 0x01, 0xd4, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x2b, 0x0b, 0x97, 0x3f, 0x74, 0x5b, 0x28, 0xaa, 0xe9, 0x37, 0xf5, 0x9f, 0x18, 0xea, 0xc7, + 0x01, 0xd6, 0x8c, 0xe1, 0x74, 0x07, 0x9a, 0xdd, 0x02, 0x8d, 0xd0, 0x5c, 0xf8, 0x14, 0x63, 0x04, 0x88 +}; +static CONST UINT8 EncAssoc107[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce107[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey107[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput108[] = { 0xb5, 0x13, 0xb0, 0x6a, 0xb9, 0xac, 0x14, 0x43, 0x5a, 0xcb, 0x8a, 0xa3, 0xa3, 0x7a, + 0xfd, 0xb6, 0x54, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, 0x3d, 0x38, 0xe9, 0x75, + 0xc3, 0x8f, 0xe3, 0xb8, 0x61, 0x95, 0x01, 0x93, 0xb1, 0xbf, 0x03, 0x11, 0xff, 0x11, + 0x79, 0x89, 0xae, 0xd9, 0xa9, 0x99, 0xb0, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, 0xb9, 0xc2, 0x7c, 0x30, 0x28, 0xaa, + 0x8d, 0x69, 0xef, 0x06, 0xaf, 0xc0, 0xb5, 0x9e, 0xda, 0x8e }; +static CONST UINT8 EncOutput108[] = { 0xb5, 0x78, 0x67, 0x45, 0x3f, 0x66, 0xf4, 0xda, 0xf9, 0xe4, 0x74, 0x69, + 0x1f, 0x9c, 0x85, 0x15, 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0x10, 0x13, 0x59, + 0x85, 0x1a, 0xd3, 0x24, 0xa0, 0xda, 0xe8, 0x8d, 0xc2, 0x43, 0x02, 0x02, + 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x01, 0x10, 0x13, 0x59, 0x85, 0x1a, 0xd3, 0x24, + 0xa0, 0xda, 0xe8, 0x8d, 0xc2, 0x43, 0x02, 0x02, 0xaa, 0x48, 0xa3, 0x88, + 0x7d, 0x4b, 0x05, 0x96, 0x99, 0xc2, 0xfd, 0xf9, 0xc6, 0x78, 0x7e, 0x0a }; +static CONST UINT8 EncAssoc108[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce108[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey108[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput109[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, + 0x87, 0x5c, 0xd4, 0xf1, 0x09, 0xe8, 0x14, 0xce, 0xa8, 0x5a, 0x08, 0xc0, 0x11, 0xd8, + 0x50, 0xdd, 0x1d, 0xcb, 0xcf, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, 0xa0, 0x34, + 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, 0x53, 0x40, 0xb8, 0x5a, 0x9a, 0xa0, 0x82, 0x96, + 0xb7, 0x7a, 0x5f, 0xc3, 0x96, 0x1f, 0x66, 0x0f, 0x17, 0x2d, 0x90, 0x96, 0x52, 0x4f, + 0xa1, 0xb2, 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, 0x1b, 0x64, 0x89, 0xba, + 0x84, 0xd8, 0xf5, 0x59, 0x82, 0x9e, 0xd9, 0xbd, 0xa2, 0x29, 0x0f, 0x16 }; +static CONST UINT8 EncOutput109[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x53, 0x33, 0xc3, + 0xe1, 0xf8, 0xd7, 0x8e, 0xac, 0xca, 0x07, 0x07, 0x52, 0x6c, 0xad, 0x01, 0x8c, 0xaf, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x30, 0x49, 0x70, 0x24, 0x14, 0xb5, 0x99, 0x50, 0x26, + 0x24, 0xfd, 0xfe, 0x29, 0x31, 0x32, 0x04, 0xaf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x30, 0x49, 0x70, 0x24, 0x14, 0xb5, 0x99, 0x50, 0x26, 0x24, 0xfd, 0xfe, 0x29, 0x31, 0x32, + 0x04, 0xb9, 0x36, 0xa8, 0x17, 0xf2, 0x21, 0x1a, 0xf1, 0x29, 0xe2, 0xcf, 0x16, 0x0f, 0xd4, 0x2b, 0xcb +}; +static CONST UINT8 EncAssoc109[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce109[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey109[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput110[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, + 0x87, 0x5c, 0xdf, 0x4c, 0x62, 0x03, 0x2d, 0x41, 0x19, 0xb5, 0x88, 0x47, 0x7e, 0x99, + 0x92, 0x5a, 0x56, 0xd9, 0xd6, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, 0xa0, 0x34, + 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, 0xfa, 0x84, 0xf0, 0x64, 0x55, 0x36, 0x42, 0x1b, + 0x2b, 0xb9, 0x24, 0x6e, 0xc2, 0x19, 0xed, 0x0b, 0x0e, 0x2d, 0x90, 0x96, 0x52, 0x4f, + 0xa1, 0xb2, 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, 0xb2, 0xa0, 0xc1, 0x84, + 0x4b, 0x4e, 0x35, 0xd4, 0x1e, 0x5d, 0xa2, 0x10, 0xf6, 0x2f, 0x84, 0x12 }; +static CONST UINT8 EncOutput110[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x58, 0x8e, 0xa8, + 0x0a, 0xc1, 0x58, 0x3f, 0x43, 0x4a, 0x80, 0x68, 0x13, 0xae, 0x2a, 0x4a, 0x9e, 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x99, 0x8d, 0x38, 0x1a, 0xdb, 0x23, 0x59, 0xdd, 0xba, + 0xe7, 0x86, 0x53, 0x7d, 0x37, 0xb9, 0x00, 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x99, 0x8d, 0x38, 0x1a, 0xdb, 0x23, 0x59, 0xdd, 0xba, 0xe7, 0x86, 0x53, 0x7d, 0x37, 0xb9, + 0x00, 0x9f, 0x7a, 0xc4, 0x35, 0x1f, 0x6b, 0x91, 0xe6, 0x30, 0x97, 0xa7, 0x13, 0x11, 0x5d, 0x05, 0xbe +}; +static CONST UINT8 EncAssoc110[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce110[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey110[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput111[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, + 0x87, 0x5c, 0x13, 0xf8, 0x0a, 0x00, 0x6d, 0xc1, 0xbb, 0xda, 0xd6, 0x39, 0xa9, 0x2f, + 0xc7, 0xec, 0xa6, 0x55, 0xf7, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, 0xa0, 0x34, + 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, 0x63, 0x48, 0xb8, 0xfd, 0x29, 0xbf, 0x96, 0xd5, + 0x63, 0xa5, 0x17, 0xe2, 0x7d, 0x7b, 0xfc, 0x0f, 0x2f, 0x2d, 0x90, 0x96, 0x52, 0x4f, + 0xa1, 0xb2, 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, 0x2b, 0x6c, 0x89, 0x1d, + 0x37, 0xc7, 0xe1, 0x1a, 0x56, 0x41, 0x91, 0x9c, 0x49, 0x4d, 0x95, 0x16 }; +static CONST UINT8 EncOutput111[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x94, 0x3a, 0xc0, + 0x09, 0x81, 0xd8, 0x9d, 0x2c, 0x14, 0xfe, 0xbf, 0xa5, 0xfb, 0x9c, 0xba, 0x12, 0x97, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x41, 0x70, 0x83, 0xa7, 0xaa, 0x8d, 0x13, 0xf2, + 0xfb, 0xb5, 0xdf, 0xc2, 0x55, 0xa8, 0x04, 0x97, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x41, 0x70, 0x83, 0xa7, 0xaa, 0x8d, 0x13, 0xf2, 0xfb, 0xb5, 0xdf, 0xc2, 0x55, 0xa8, + 0x04, 0x9a, 0x18, 0xa8, 0x28, 0x07, 0x02, 0x69, 0xf4, 0x47, 0x00, 0xd0, 0x09, 0xe7, 0x17, 0x1c, 0xc9 +}; +static CONST UINT8 EncAssoc111[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce111[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey111[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput112[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, + 0x87, 0x5c, 0x82, 0xe5, 0x9b, 0x45, 0x82, 0x91, 0x50, 0x38, 0xf9, 0x33, 0x81, 0x1e, + 0x65, 0x2d, 0xc6, 0x6a, 0xfc, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, 0xa0, 0x34, + 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, 0xb6, 0x71, 0xc8, 0xca, 0xc2, 0x70, 0xc2, 0x65, + 0xa0, 0xac, 0x2f, 0x53, 0x57, 0x99, 0x88, 0x0a, 0x24, 0x2d, 0x90, 0x96, 0x52, 0x4f, + 0xa1, 0xb2, 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, 0xfe, 0x55, 0xf9, 0x2a, + 0xdc, 0x08, 0xb5, 0xaa, 0x95, 0x48, 0xa9, 0x2d, 0x63, 0xaf, 0xe1, 0x13 }; +static CONST UINT8 EncOutput112[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x05, 0x27, 0x51, + 0x4c, 0x6e, 0x88, 0x76, 0xce, 0x3b, 0xf4, 0x97, 0x94, 0x59, 0x5d, 0xda, 0x2d, 0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xd5, 0x78, 0x00, 0xb4, 0x4c, 0x65, 0xd9, 0xa3, 0x31, + 0xf2, 0x8d, 0x6e, 0xe8, 0xb7, 0xdc, 0x01, 0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xd5, 0x78, 0x00, 0xb4, 0x4c, 0x65, 0xd9, 0xa3, 0x31, 0xf2, 0x8d, 0x6e, 0xe8, 0xb7, 0xdc, + 0x01, 0xb4, 0x36, 0xa8, 0x2b, 0x93, 0xd5, 0x55, 0xf7, 0x43, 0x00, 0xd0, 0x19, 0x9b, 0xa7, 0x18, 0xce +}; +static CONST UINT8 EncAssoc112[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce112[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey112[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput113[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, + 0x87, 0x5c, 0xf1, 0xd1, 0x28, 0x87, 0xb7, 0x21, 0x69, 0x86, 0xa1, 0x2d, 0x79, 0x09, + 0x8b, 0x6d, 0xe6, 0x0f, 0xc0, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, 0xa0, 0x34, + 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, 0xa7, 0xc7, 0x58, 0x99, 0xf3, 0xe6, 0x0a, 0xf1, + 0xfc, 0xb6, 0xc7, 0x30, 0x7d, 0x87, 0x59, 0x0f, 0x18, 0x2d, 0x90, 0x96, 0x52, 0x4f, + 0xa1, 0xb2, 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, 0xef, 0xe3, 0x69, 0x79, + 0xed, 0x9e, 0x7d, 0x3e, 0xc9, 0x52, 0x41, 0x4e, 0x49, 0xb1, 0x30, 0x16 }; +static CONST UINT8 EncOutput113[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x76, 0x13, 0xe2, + 0x8e, 0x5b, 0x38, 0x4f, 0x70, 0x63, 0xea, 0x6f, 0x83, 0xb7, 0x1d, 0xfa, 0x48, 0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc4, 0xce, 0x90, 0xe7, 0x7d, 0xf3, 0x11, 0x37, 0x6d, + 0xe8, 0x65, 0x0d, 0xc2, 0xa9, 0x0d, 0x04, 0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xc4, 0xce, 0x90, 0xe7, 0x7d, 0xf3, 0x11, 0x37, 0x6d, 0xe8, 0x65, 0x0d, 0xc2, 0xa9, 0x0d, + 0x04, 0xce, 0x54, 0xa8, 0x2e, 0x1f, 0xa9, 0x42, 0xfa, 0x3f, 0x00, 0xd0, 0x29, 0x4f, 0x37, 0x15, 0xd3 +}; +static CONST UINT8 EncAssoc113[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce113[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey113[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput114[] = { 0xcb, 0xf1, 0xda, 0x9e, 0x0b, 0xa9, 0x37, 0x73, 0x74, 0xe6, 0x9e, 0x1c, 0x0e, 0x60, + 0x0c, 0xfc, 0x34, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, 0x3d, 0x38, 0xe9, 0x75, + 0xc3, 0x8f, 0xe3, 0xb8, 0xbe, 0x3f, 0xa6, 0x6b, 0x6c, 0xe7, 0x80, 0x8a, 0xa3, 0xe4, + 0x59, 0x49, 0xf9, 0x44, 0x64, 0x9f, 0xd0, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, 0x66, 0x68, 0xdb, 0xc8, 0xf5, 0xf2, + 0x0e, 0xf2, 0xb3, 0xf3, 0x8f, 0x00, 0xe2, 0x03, 0x17, 0x88 }; +static CONST UINT8 EncOutput114[] = { 0xcb, 0x9a, 0x0d, 0xb1, 0x8d, 0x63, 0xd7, 0xea, 0xd7, 0xc9, 0x60, 0xd6, + 0xb2, 0x86, 0x74, 0x5f, 0xb3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xde, 0xba, 0xb4, 0xa1, + 0x58, 0x42, 0x50, 0xbf, 0xfc, 0x2f, 0xc8, 0x4d, 0x95, 0xde, 0xcf, 0x04, + 0xb3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xde, 0xba, 0xb4, 0xa1, 0x58, 0x42, 0x50, 0xbf, + 0xfc, 0x2f, 0xc8, 0x4d, 0x95, 0xde, 0xcf, 0x04, 0x23, 0x83, 0xab, 0x0b, + 0x79, 0x92, 0x05, 0x69, 0x9b, 0x51, 0x0a, 0xa7, 0x09, 0xbf, 0x31, 0xf1 }; +static CONST UINT8 EncAssoc114[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce114[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey114[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput115[] = { 0x8f, 0x27, 0x86, 0x94, 0xc4, 0xe9, 0xda, 0xeb, 0xd5, 0x8d, 0x3e, 0x5b, 0x96, 0x6e, + 0x8b, 0x68, 0x42, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, 0x3d, 0x38, 0xe9, 0x75, + 0xc3, 0x8f, 0xe3, 0xb8, 0x06, 0x53, 0xe7, 0xa3, 0x31, 0x71, 0x88, 0x33, 0xac, 0xc3, + 0xb9, 0xad, 0xff, 0x1c, 0x31, 0x98, 0xa6, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, 0xde, 0x04, 0x9a, 0x00, 0xa8, 0x64, + 0x06, 0x4b, 0xbc, 0xd4, 0x6f, 0xe4, 0xe4, 0x5b, 0x42, 0x8f }; +static CONST UINT8 EncOutput115[] = { 0x8f, 0x4c, 0x51, 0xbb, 0x42, 0x23, 0x3a, 0x72, 0x76, 0xa2, 0xc0, 0x91, + 0x2a, 0x88, 0xf3, 0xcb, 0xc5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x66, 0xd6, 0xf5, 0x69, + 0x05, 0xd4, 0x58, 0x06, 0xf3, 0x08, 0x28, 0xa9, 0x93, 0x86, 0x9a, 0x03, + 0xc5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x66, 0xd6, 0xf5, 0x69, 0x05, 0xd4, 0x58, 0x06, + 0xf3, 0x08, 0x28, 0xa9, 0x93, 0x86, 0x9a, 0x03, 0x8b, 0xfb, 0xab, 0x17, + 0xa9, 0xe0, 0xb8, 0x74, 0x8b, 0x51, 0x0a, 0xe7, 0xd9, 0xfd, 0x23, 0x05 }; +static CONST UINT8 EncAssoc115[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce115[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey115[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput116[] = { 0xd5, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, + 0x87, 0x5c, 0x9a, 0x22, 0xd7, 0x0a, 0x48, 0xe2, 0x4f, 0xdd, 0xcd, 0xd4, 0x41, 0x9d, + 0xe6, 0x4c, 0x8f, 0x44, 0xfc, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, 0xa0, 0x34, + 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, 0x77, 0xb5, 0xc9, 0x07, 0xd9, 0xc9, 0xe1, 0xea, + 0x51, 0x85, 0x1a, 0x20, 0x4a, 0xad, 0x9f, 0x0a, 0x24, 0x2d, 0x90, 0x96, 0x52, 0x4f, + 0xa1, 0xb2, 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, 0x3f, 0x91, 0xf8, 0xe7, + 0xc7, 0xb1, 0x96, 0x25, 0x64, 0x61, 0x9c, 0x5e, 0x7e, 0x9b, 0xf6, 0x13 }; +static CONST UINT8 EncOutput116[] = { + 0xd5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x1d, 0xe0, 0x1d, + 0x03, 0xa4, 0xfb, 0x69, 0x2b, 0x0f, 0x13, 0x57, 0x17, 0xda, 0x3c, 0x93, 0x03, 0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x14, 0xbc, 0x01, 0x79, 0x57, 0xdc, 0xfa, 0x2c, 0xc0, + 0xdb, 0xb8, 0x1d, 0xf5, 0x83, 0xcb, 0x01, 0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x14, 0xbc, 0x01, 0x79, 0x57, 0xdc, 0xfa, 0x2c, 0xc0, 0xdb, 0xb8, 0x1d, 0xf5, 0x83, 0xcb, + 0x01, 0x49, 0xbc, 0x6e, 0x9f, 0xc5, 0x1c, 0x4d, 0x50, 0x30, 0x36, 0x64, 0x4d, 0x84, 0x27, 0x73, 0xd2 +}; +static CONST UINT8 EncAssoc116[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce116[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey116[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput117[] = { 0xdb, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, + 0x87, 0x5c, 0x75, 0xd5, 0x64, 0x3a, 0xa5, 0xaf, 0x93, 0x4d, 0x8c, 0xce, 0x39, 0x2c, + 0xc3, 0xee, 0xdb, 0x47, 0xc0, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, 0xa0, 0x34, + 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, 0x60, 0x1b, 0x5a, 0xd2, 0x06, 0x7f, 0x28, 0x06, + 0x6a, 0x8f, 0x32, 0x81, 0x71, 0x5b, 0xa8, 0x08, 0x18, 0x2d, 0x90, 0x96, 0x52, 0x4f, + 0xa1, 0xb2, 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, 0x28, 0x3f, 0x6b, 0x32, + 0x18, 0x07, 0x5f, 0xc9, 0x5f, 0x6b, 0xb4, 0xff, 0x45, 0x6d, 0xc1, 0x11 }; +static CONST UINT8 EncOutput117[] = { + 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf2, 0x17, 0xae, + 0x33, 0x49, 0xb6, 0xb5, 0xbb, 0x4e, 0x09, 0x2f, 0xa6, 0xff, 0x9e, 0xc7, 0x00, 0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03, 0x12, 0x92, 0xac, 0x88, 0x6a, 0x33, 0xc0, 0xfb, + 0xd1, 0x90, 0xbc, 0xce, 0x75, 0xfc, 0x03, 0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x03, 0x12, 0x92, 0xac, 0x88, 0x6a, 0x33, 0xc0, 0xfb, 0xd1, 0x90, 0xbc, 0xce, 0x75, 0xfc, + 0x03, 0x63, 0xda, 0x6e, 0xa2, 0x51, 0xf0, 0x39, 0x53, 0x2c, 0x36, 0x64, 0x5d, 0x38, 0xb7, 0x6f, 0xd7 +}; +static CONST UINT8 EncAssoc117[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce117[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey117[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +/* wycheproof - edge case intermediate sums in Poly1305 */ +static CONST UINT8 EncInput118[] = { 0x93, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, + 0x87, 0x5c, 0x62, 0x48, 0x39, 0x60, 0x42, 0x16, 0xe4, 0x03, 0xeb, 0xcc, 0x6a, 0xf5, + 0x59, 0xec, 0x8b, 0x43, 0x97, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, 0xa0, 0x34, + 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, 0xd8, 0xc8, 0xc3, 0xfa, 0x1a, 0x9e, 0x47, 0x4a, + 0xbe, 0x52, 0xd0, 0x2c, 0x81, 0x87, 0xe9, 0x0f, 0x4f, 0x2d, 0x90, 0x96, 0x52, 0x4f, + 0xa1, 0xb2, 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, 0x90, 0xec, 0xf2, 0x1a, + 0x04, 0xe6, 0x30, 0x85, 0x8b, 0xb6, 0x56, 0x52, 0xb5, 0xb1, 0x80, 0x16 }; +static CONST UINT8 EncOutput118[] = { + 0x93, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe5, 0x8a, 0xf3, + 0x69, 0xae, 0x0f, 0xc2, 0xf5, 0x29, 0x0b, 0x7c, 0x7f, 0x65, 0x9c, 0x97, 0x04, 0xf7, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xbb, 0xc1, 0x0b, 0x84, 0x94, 0x8b, 0x5c, 0x8c, 0x2f, + 0x0c, 0x72, 0x11, 0x3e, 0xa9, 0xbd, 0x04, 0xf7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xbb, 0xc1, 0x0b, 0x84, 0x94, 0x8b, 0x5c, 0x8c, 0x2f, 0x0c, 0x72, 0x11, 0x3e, 0xa9, 0xbd, + 0x04, 0x73, 0xeb, 0x27, 0x24, 0xb5, 0xc4, 0x05, 0xf0, 0x4d, 0x00, 0xd0, 0xf1, 0x58, 0x40, 0xa1, 0xc1 +}; +static CONST UINT8 EncAssoc118[] = { 0xff, 0xff, 0xff, 0xff }; +static CONST UINT8 EncNonce118[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; +static CONST UINT8 EncKey118[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; + +static CONST struct ChaCha20Poly1305TestVector ChaCha20Poly1305EncVectors[] = { + { EncInput001, + EncOutput001, + EncAssoc001, + EncNonce001, + EncKey001, + sizeof(EncInput001), + sizeof(EncAssoc001), + sizeof(EncNonce001) }, + { NULL, EncOutput002, NULL, EncNonce002, EncKey002, 0, 0, sizeof(EncNonce002) }, + { NULL, EncOutput003, EncAssoc003, EncNonce003, EncKey003, 0, sizeof(EncAssoc003), sizeof(EncNonce003) }, + { EncInput004, + EncOutput004, + EncAssoc004, + EncNonce004, + EncKey004, + sizeof(EncInput004), + sizeof(EncAssoc004), + sizeof(EncNonce004) }, + { EncInput005, EncOutput005, NULL, EncNonce005, EncKey005, sizeof(EncInput005), 0, sizeof(EncNonce005) }, + { EncInput006, + EncOutput006, + EncAssoc006, + EncNonce006, + EncKey006, + sizeof(EncInput006), + sizeof(EncAssoc006), + sizeof(EncNonce006) }, + { EncInput007, EncOutput007, NULL, EncNonce007, EncKey007, sizeof(EncInput007), 0, sizeof(EncNonce007) }, + { EncInput008, EncOutput008, NULL, EncNonce008, EncKey008, sizeof(EncInput008), 0, sizeof(EncNonce008) }, + { EncInput009, + EncOutput009, + EncAssoc009, + EncNonce009, + EncKey009, + sizeof(EncInput009), + sizeof(EncAssoc009), + sizeof(EncNonce009) }, + { EncInput010, + EncOutput010, + EncAssoc010, + EncNonce010, + EncKey010, + sizeof(EncInput010), + sizeof(EncAssoc010), + sizeof(EncNonce010) }, + { EncInput011, + EncOutput011, + EncAssoc011, + EncNonce011, + EncKey011, + sizeof(EncInput011), + sizeof(EncAssoc011), + sizeof(EncNonce011) }, + { EncInput012, + EncOutput012, + EncAssoc012, + EncNonce012, + EncKey012, + sizeof(EncInput012), + sizeof(EncAssoc012), + sizeof(EncNonce012) }, + { EncInput013, + EncOutput013, + EncAssoc013, + EncNonce013, + EncKey013, + sizeof(EncInput013), + sizeof(EncAssoc013), + sizeof(EncNonce013) }, + { NULL, EncOutput014, NULL, EncNonce014, EncKey014, 0, 0, sizeof(EncNonce014) }, + { NULL, EncOutput015, EncAssoc015, EncNonce015, EncKey015, 0, sizeof(EncAssoc015), sizeof(EncNonce015) }, + { EncInput016, EncOutput016, NULL, EncNonce016, EncKey016, sizeof(EncInput016), 0, sizeof(EncNonce016) }, + { EncInput017, + EncOutput017, + EncAssoc017, + EncNonce017, + EncKey017, + sizeof(EncInput017), + sizeof(EncAssoc017), + sizeof(EncNonce017) }, + { EncInput018, EncOutput018, NULL, EncNonce018, EncKey018, sizeof(EncInput018), 0, sizeof(EncNonce018) }, + { EncInput019, + EncOutput019, + EncAssoc019, + EncNonce019, + EncKey019, + sizeof(EncInput019), + sizeof(EncAssoc019), + sizeof(EncNonce019) }, + { EncInput020, EncOutput020, NULL, EncNonce020, EncKey020, sizeof(EncInput020), 0, sizeof(EncNonce020) }, + { EncInput021, + EncOutput021, + EncAssoc021, + EncNonce021, + EncKey021, + sizeof(EncInput021), + sizeof(EncAssoc021), + sizeof(EncNonce021) }, + { EncInput022, EncOutput022, NULL, EncNonce022, EncKey022, sizeof(EncInput022), 0, sizeof(EncNonce022) }, + { EncInput023, + EncOutput023, + EncAssoc023, + EncNonce023, + EncKey023, + sizeof(EncInput023), + sizeof(EncAssoc023), + sizeof(EncNonce023) }, + { EncInput024, EncOutput024, NULL, EncNonce024, EncKey024, sizeof(EncInput024), 0, sizeof(EncNonce024) }, + { EncInput025, + EncOutput025, + EncAssoc025, + EncNonce025, + EncKey025, + sizeof(EncInput025), + sizeof(EncAssoc025), + sizeof(EncNonce025) }, + { EncInput026, EncOutput026, NULL, EncNonce026, EncKey026, sizeof(EncInput026), 0, sizeof(EncNonce026) }, + { EncInput027, + EncOutput027, + EncAssoc027, + EncNonce027, + EncKey027, + sizeof(EncInput027), + sizeof(EncAssoc027), + sizeof(EncNonce027) }, + { EncInput028, EncOutput028, NULL, EncNonce028, EncKey028, sizeof(EncInput028), 0, sizeof(EncNonce028) }, + { EncInput029, + EncOutput029, + EncAssoc029, + EncNonce029, + EncKey029, + sizeof(EncInput029), + sizeof(EncAssoc029), + sizeof(EncNonce029) }, + { EncInput030, EncOutput030, NULL, EncNonce030, EncKey030, sizeof(EncInput030), 0, sizeof(EncNonce030) }, + { EncInput031, + EncOutput031, + EncAssoc031, + EncNonce031, + EncKey031, + sizeof(EncInput031), + sizeof(EncAssoc031), + sizeof(EncNonce031) }, + { EncInput032, EncOutput032, NULL, EncNonce032, EncKey032, sizeof(EncInput032), 0, sizeof(EncNonce032) }, + { EncInput033, + EncOutput033, + EncAssoc033, + EncNonce033, + EncKey033, + sizeof(EncInput033), + sizeof(EncAssoc033), + sizeof(EncNonce033) }, + { EncInput034, EncOutput034, NULL, EncNonce034, EncKey034, sizeof(EncInput034), 0, sizeof(EncNonce034) }, + { EncInput035, + EncOutput035, + EncAssoc035, + EncNonce035, + EncKey035, + sizeof(EncInput035), + sizeof(EncAssoc035), + sizeof(EncNonce035) }, + { EncInput036, EncOutput036, NULL, EncNonce036, EncKey036, sizeof(EncInput036), 0, sizeof(EncNonce036) }, + { EncInput037, + EncOutput037, + EncAssoc037, + EncNonce037, + EncKey037, + sizeof(EncInput037), + sizeof(EncAssoc037), + sizeof(EncNonce037) }, + { EncInput038, EncOutput038, NULL, EncNonce038, EncKey038, sizeof(EncInput038), 0, sizeof(EncNonce038) }, + { EncInput039, + EncOutput039, + EncAssoc039, + EncNonce039, + EncKey039, + sizeof(EncInput039), + sizeof(EncAssoc039), + sizeof(EncNonce039) }, + { EncInput040, EncOutput040, NULL, EncNonce040, EncKey040, sizeof(EncInput040), 0, sizeof(EncNonce040) }, + { EncInput041, + EncOutput041, + EncAssoc041, + EncNonce041, + EncKey041, + sizeof(EncInput041), + sizeof(EncAssoc041), + sizeof(EncNonce041) }, + { EncInput042, EncOutput042, NULL, EncNonce042, EncKey042, sizeof(EncInput042), 0, sizeof(EncNonce042) }, + { EncInput043, + EncOutput043, + EncAssoc043, + EncNonce043, + EncKey043, + sizeof(EncInput043), + sizeof(EncAssoc043), + sizeof(EncNonce043) }, + { EncInput044, + EncOutput044, + EncAssoc044, + EncNonce044, + EncKey044, + sizeof(EncInput044), + sizeof(EncAssoc044), + sizeof(EncNonce044) }, + { EncInput045, + EncOutput045, + EncAssoc045, + EncNonce045, + EncKey045, + sizeof(EncInput045), + sizeof(EncAssoc045), + sizeof(EncNonce045) }, + { EncInput046, + EncOutput046, + EncAssoc046, + EncNonce046, + EncKey046, + sizeof(EncInput046), + sizeof(EncAssoc046), + sizeof(EncNonce046) }, + { EncInput047, + EncOutput047, + EncAssoc047, + EncNonce047, + EncKey047, + sizeof(EncInput047), + sizeof(EncAssoc047), + sizeof(EncNonce047) }, + { EncInput048, + EncOutput048, + EncAssoc048, + EncNonce048, + EncKey048, + sizeof(EncInput048), + sizeof(EncAssoc048), + sizeof(EncNonce048) }, + { EncInput049, + EncOutput049, + EncAssoc049, + EncNonce049, + EncKey049, + sizeof(EncInput049), + sizeof(EncAssoc049), + sizeof(EncNonce049) }, + { EncInput050, + EncOutput050, + EncAssoc050, + EncNonce050, + EncKey050, + sizeof(EncInput050), + sizeof(EncAssoc050), + sizeof(EncNonce050) }, + { EncInput051, + EncOutput051, + EncAssoc051, + EncNonce051, + EncKey051, + sizeof(EncInput051), + sizeof(EncAssoc051), + sizeof(EncNonce051) }, + { EncInput052, + EncOutput052, + EncAssoc052, + EncNonce052, + EncKey052, + sizeof(EncInput052), + sizeof(EncAssoc052), + sizeof(EncNonce052) }, + { EncInput053, + EncOutput053, + EncAssoc053, + EncNonce053, + EncKey053, + sizeof(EncInput053), + sizeof(EncAssoc053), + sizeof(EncNonce053) }, + { EncInput054, + EncOutput054, + EncAssoc054, + EncNonce054, + EncKey054, + sizeof(EncInput054), + sizeof(EncAssoc054), + sizeof(EncNonce054) }, + { EncInput055, + EncOutput055, + EncAssoc055, + EncNonce055, + EncKey055, + sizeof(EncInput055), + sizeof(EncAssoc055), + sizeof(EncNonce055) }, + { EncInput056, + EncOutput056, + EncAssoc056, + EncNonce056, + EncKey056, + sizeof(EncInput056), + sizeof(EncAssoc056), + sizeof(EncNonce056) }, + { EncInput057, + EncOutput057, + EncAssoc057, + EncNonce057, + EncKey057, + sizeof(EncInput057), + sizeof(EncAssoc057), + sizeof(EncNonce057) }, + { EncInput058, + EncOutput058, + EncAssoc058, + EncNonce058, + EncKey058, + sizeof(EncInput058), + sizeof(EncAssoc058), + sizeof(EncNonce058) }, + { EncInput059, + EncOutput059, + EncAssoc059, + EncNonce059, + EncKey059, + sizeof(EncInput059), + sizeof(EncAssoc059), + sizeof(EncNonce059) }, + { EncInput060, + EncOutput060, + EncAssoc060, + EncNonce060, + EncKey060, + sizeof(EncInput060), + sizeof(EncAssoc060), + sizeof(EncNonce060) }, + { EncInput061, + EncOutput061, + EncAssoc061, + EncNonce061, + EncKey061, + sizeof(EncInput061), + sizeof(EncAssoc061), + sizeof(EncNonce061) }, + { EncInput062, + EncOutput062, + EncAssoc062, + EncNonce062, + EncKey062, + sizeof(EncInput062), + sizeof(EncAssoc062), + sizeof(EncNonce062) }, + { EncInput063, + EncOutput063, + EncAssoc063, + EncNonce063, + EncKey063, + sizeof(EncInput063), + sizeof(EncAssoc063), + sizeof(EncNonce063) }, + { EncInput064, + EncOutput064, + EncAssoc064, + EncNonce064, + EncKey064, + sizeof(EncInput064), + sizeof(EncAssoc064), + sizeof(EncNonce064) }, + { EncInput065, + EncOutput065, + EncAssoc065, + EncNonce065, + EncKey065, + sizeof(EncInput065), + sizeof(EncAssoc065), + sizeof(EncNonce065) }, + { EncInput066, + EncOutput066, + EncAssoc066, + EncNonce066, + EncKey066, + sizeof(EncInput066), + sizeof(EncAssoc066), + sizeof(EncNonce066) }, + { EncInput067, + EncOutput067, + EncAssoc067, + EncNonce067, + EncKey067, + sizeof(EncInput067), + sizeof(EncAssoc067), + sizeof(EncNonce067) }, + { EncInput068, + EncOutput068, + EncAssoc068, + EncNonce068, + EncKey068, + sizeof(EncInput068), + sizeof(EncAssoc068), + sizeof(EncNonce068) }, + { EncInput069, + EncOutput069, + EncAssoc069, + EncNonce069, + EncKey069, + sizeof(EncInput069), + sizeof(EncAssoc069), + sizeof(EncNonce069) }, + { EncInput070, + EncOutput070, + EncAssoc070, + EncNonce070, + EncKey070, + sizeof(EncInput070), + sizeof(EncAssoc070), + sizeof(EncNonce070) }, + { EncInput071, + EncOutput071, + EncAssoc071, + EncNonce071, + EncKey071, + sizeof(EncInput071), + sizeof(EncAssoc071), + sizeof(EncNonce071) }, + { EncInput072, + EncOutput072, + EncAssoc072, + EncNonce072, + EncKey072, + sizeof(EncInput072), + sizeof(EncAssoc072), + sizeof(EncNonce072) }, + { EncInput073, + EncOutput073, + EncAssoc073, + EncNonce073, + EncKey073, + sizeof(EncInput073), + sizeof(EncAssoc073), + sizeof(EncNonce073) }, + { EncInput074, + EncOutput074, + EncAssoc074, + EncNonce074, + EncKey074, + sizeof(EncInput074), + sizeof(EncAssoc074), + sizeof(EncNonce074) }, + { EncInput075, + EncOutput075, + EncAssoc075, + EncNonce075, + EncKey075, + sizeof(EncInput075), + sizeof(EncAssoc075), + sizeof(EncNonce075) }, + { EncInput076, + EncOutput076, + EncAssoc076, + EncNonce076, + EncKey076, + sizeof(EncInput076), + sizeof(EncAssoc076), + sizeof(EncNonce076) }, + { EncInput077, + EncOutput077, + EncAssoc077, + EncNonce077, + EncKey077, + sizeof(EncInput077), + sizeof(EncAssoc077), + sizeof(EncNonce077) }, + { EncInput078, + EncOutput078, + EncAssoc078, + EncNonce078, + EncKey078, + sizeof(EncInput078), + sizeof(EncAssoc078), + sizeof(EncNonce078) }, + { EncInput079, + EncOutput079, + EncAssoc079, + EncNonce079, + EncKey079, + sizeof(EncInput079), + sizeof(EncAssoc079), + sizeof(EncNonce079) }, + { EncInput080, + EncOutput080, + EncAssoc080, + EncNonce080, + EncKey080, + sizeof(EncInput080), + sizeof(EncAssoc080), + sizeof(EncNonce080) }, + { EncInput081, + EncOutput081, + EncAssoc081, + EncNonce081, + EncKey081, + sizeof(EncInput081), + sizeof(EncAssoc081), + sizeof(EncNonce081) }, + { EncInput082, + EncOutput082, + EncAssoc082, + EncNonce082, + EncKey082, + sizeof(EncInput082), + sizeof(EncAssoc082), + sizeof(EncNonce082) }, + { EncInput083, + EncOutput083, + EncAssoc083, + EncNonce083, + EncKey083, + sizeof(EncInput083), + sizeof(EncAssoc083), + sizeof(EncNonce083) }, + { EncInput084, + EncOutput084, + EncAssoc084, + EncNonce084, + EncKey084, + sizeof(EncInput084), + sizeof(EncAssoc084), + sizeof(EncNonce084) }, + { EncInput085, + EncOutput085, + EncAssoc085, + EncNonce085, + EncKey085, + sizeof(EncInput085), + sizeof(EncAssoc085), + sizeof(EncNonce085) }, + { EncInput086, + EncOutput086, + EncAssoc086, + EncNonce086, + EncKey086, + sizeof(EncInput086), + sizeof(EncAssoc086), + sizeof(EncNonce086) }, + { EncInput087, + EncOutput087, + EncAssoc087, + EncNonce087, + EncKey087, + sizeof(EncInput087), + sizeof(EncAssoc087), + sizeof(EncNonce087) }, + { EncInput088, + EncOutput088, + EncAssoc088, + EncNonce088, + EncKey088, + sizeof(EncInput088), + sizeof(EncAssoc088), + sizeof(EncNonce088) }, + { EncInput089, + EncOutput089, + EncAssoc089, + EncNonce089, + EncKey089, + sizeof(EncInput089), + sizeof(EncAssoc089), + sizeof(EncNonce089) }, + { EncInput090, + EncOutput090, + EncAssoc090, + EncNonce090, + EncKey090, + sizeof(EncInput090), + sizeof(EncAssoc090), + sizeof(EncNonce090) }, + { EncInput091, + EncOutput091, + EncAssoc091, + EncNonce091, + EncKey091, + sizeof(EncInput091), + sizeof(EncAssoc091), + sizeof(EncNonce091) }, + { EncInput092, + EncOutput092, + EncAssoc092, + EncNonce092, + EncKey092, + sizeof(EncInput092), + sizeof(EncAssoc092), + sizeof(EncNonce092) }, + { EncInput093, + EncOutput093, + EncAssoc093, + EncNonce093, + EncKey093, + sizeof(EncInput093), + sizeof(EncAssoc093), + sizeof(EncNonce093) }, + { EncInput094, + EncOutput094, + EncAssoc094, + EncNonce094, + EncKey094, + sizeof(EncInput094), + sizeof(EncAssoc094), + sizeof(EncNonce094) }, + { EncInput095, + EncOutput095, + EncAssoc095, + EncNonce095, + EncKey095, + sizeof(EncInput095), + sizeof(EncAssoc095), + sizeof(EncNonce095) }, + { EncInput096, + EncOutput096, + EncAssoc096, + EncNonce096, + EncKey096, + sizeof(EncInput096), + sizeof(EncAssoc096), + sizeof(EncNonce096) }, + { EncInput097, + EncOutput097, + EncAssoc097, + EncNonce097, + EncKey097, + sizeof(EncInput097), + sizeof(EncAssoc097), + sizeof(EncNonce097) }, + { EncInput098, + EncOutput098, + EncAssoc098, + EncNonce098, + EncKey098, + sizeof(EncInput098), + sizeof(EncAssoc098), + sizeof(EncNonce098) }, + { EncInput099, + EncOutput099, + EncAssoc099, + EncNonce099, + EncKey099, + sizeof(EncInput099), + sizeof(EncAssoc099), + sizeof(EncNonce099) }, + { EncInput100, + EncOutput100, + EncAssoc100, + EncNonce100, + EncKey100, + sizeof(EncInput100), + sizeof(EncAssoc100), + sizeof(EncNonce100) }, + { EncInput101, + EncOutput101, + EncAssoc101, + EncNonce101, + EncKey101, + sizeof(EncInput101), + sizeof(EncAssoc101), + sizeof(EncNonce101) }, + { EncInput102, + EncOutput102, + EncAssoc102, + EncNonce102, + EncKey102, + sizeof(EncInput102), + sizeof(EncAssoc102), + sizeof(EncNonce102) }, + { EncInput103, + EncOutput103, + EncAssoc103, + EncNonce103, + EncKey103, + sizeof(EncInput103), + sizeof(EncAssoc103), + sizeof(EncNonce103) }, + { EncInput104, + EncOutput104, + EncAssoc104, + EncNonce104, + EncKey104, + sizeof(EncInput104), + sizeof(EncAssoc104), + sizeof(EncNonce104) }, + { EncInput105, + EncOutput105, + EncAssoc105, + EncNonce105, + EncKey105, + sizeof(EncInput105), + sizeof(EncAssoc105), + sizeof(EncNonce105) }, + { EncInput106, + EncOutput106, + EncAssoc106, + EncNonce106, + EncKey106, + sizeof(EncInput106), + sizeof(EncAssoc106), + sizeof(EncNonce106) }, + { EncInput107, + EncOutput107, + EncAssoc107, + EncNonce107, + EncKey107, + sizeof(EncInput107), + sizeof(EncAssoc107), + sizeof(EncNonce107) }, + { EncInput108, + EncOutput108, + EncAssoc108, + EncNonce108, + EncKey108, + sizeof(EncInput108), + sizeof(EncAssoc108), + sizeof(EncNonce108) }, + { EncInput109, + EncOutput109, + EncAssoc109, + EncNonce109, + EncKey109, + sizeof(EncInput109), + sizeof(EncAssoc109), + sizeof(EncNonce109) }, + { EncInput110, + EncOutput110, + EncAssoc110, + EncNonce110, + EncKey110, + sizeof(EncInput110), + sizeof(EncAssoc110), + sizeof(EncNonce110) }, + { EncInput111, + EncOutput111, + EncAssoc111, + EncNonce111, + EncKey111, + sizeof(EncInput111), + sizeof(EncAssoc111), + sizeof(EncNonce111) }, + { EncInput112, + EncOutput112, + EncAssoc112, + EncNonce112, + EncKey112, + sizeof(EncInput112), + sizeof(EncAssoc112), + sizeof(EncNonce112) }, + { EncInput113, + EncOutput113, + EncAssoc113, + EncNonce113, + EncKey113, + sizeof(EncInput113), + sizeof(EncAssoc113), + sizeof(EncNonce113) }, + { EncInput114, + EncOutput114, + EncAssoc114, + EncNonce114, + EncKey114, + sizeof(EncInput114), + sizeof(EncAssoc114), + sizeof(EncNonce114) }, + { EncInput115, + EncOutput115, + EncAssoc115, + EncNonce115, + EncKey115, + sizeof(EncInput115), + sizeof(EncAssoc115), + sizeof(EncNonce115) }, + { EncInput116, + EncOutput116, + EncAssoc116, + EncNonce116, + EncKey116, + sizeof(EncInput116), + sizeof(EncAssoc116), + sizeof(EncNonce116) }, + { EncInput117, + EncOutput117, + EncAssoc117, + EncNonce117, + EncKey117, + sizeof(EncInput117), + sizeof(EncAssoc117), + sizeof(EncNonce117) }, + { EncInput118, + EncOutput118, + EncAssoc118, + EncNonce118, + EncKey118, + sizeof(EncInput118), + sizeof(EncAssoc118), + sizeof(EncNonce118) } +}; + +static CONST UINT8 DecInput001[] = { + 0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4, 0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd, 0x5e, 0x80, 0x5c, + 0xfd, 0x34, 0x5c, 0xf3, 0x89, 0xf1, 0x08, 0x67, 0x0a, 0xc7, 0x6c, 0x8c, 0xb2, 0x4c, 0x6c, 0xfc, 0x18, 0x75, 0x5d, + 0x43, 0xee, 0xa0, 0x9e, 0xe9, 0x4e, 0x38, 0x2d, 0x26, 0xb0, 0xbd, 0xb7, 0xb7, 0x3c, 0x32, 0x1b, 0x01, 0x00, 0xd4, + 0xf0, 0x3b, 0x7f, 0x35, 0x58, 0x94, 0xcf, 0x33, 0x2f, 0x83, 0x0e, 0x71, 0x0b, 0x97, 0xce, 0x98, 0xc8, 0xa8, 0x4a, + 0xbd, 0x0b, 0x94, 0x81, 0x14, 0xad, 0x17, 0x6e, 0x00, 0x8d, 0x33, 0xbd, 0x60, 0xf9, 0x82, 0xb1, 0xff, 0x37, 0xc8, + 0x55, 0x97, 0x97, 0xa0, 0x6e, 0xf4, 0xf0, 0xef, 0x61, 0xc1, 0x86, 0x32, 0x4e, 0x2b, 0x35, 0x06, 0x38, 0x36, 0x06, + 0x90, 0x7b, 0x6a, 0x7c, 0x02, 0xb0, 0xf9, 0xf6, 0x15, 0x7b, 0x53, 0xc8, 0x67, 0xe4, 0xb9, 0x16, 0x6c, 0x76, 0x7b, + 0x80, 0x4d, 0x46, 0xa5, 0x9b, 0x52, 0x16, 0xcd, 0xe7, 0xa4, 0xe9, 0x90, 0x40, 0xc5, 0xa4, 0x04, 0x33, 0x22, 0x5e, + 0xe2, 0x82, 0xa1, 0xb0, 0xa0, 0x6c, 0x52, 0x3e, 0xaf, 0x45, 0x34, 0xd7, 0xf8, 0x3f, 0xa1, 0x15, 0x5b, 0x00, 0x47, + 0x71, 0x8c, 0xbc, 0x54, 0x6a, 0x0d, 0x07, 0x2b, 0x04, 0xb3, 0x56, 0x4e, 0xea, 0x1b, 0x42, 0x22, 0x73, 0xf5, 0x48, + 0x27, 0x1a, 0x0b, 0xb2, 0x31, 0x60, 0x53, 0xfa, 0x76, 0x99, 0x19, 0x55, 0xeb, 0xd6, 0x31, 0x59, 0x43, 0x4e, 0xce, + 0xbb, 0x4e, 0x46, 0x6d, 0xae, 0x5a, 0x10, 0x73, 0xa6, 0x72, 0x76, 0x27, 0x09, 0x7a, 0x10, 0x49, 0xe6, 0x17, 0xd9, + 0x1d, 0x36, 0x10, 0x94, 0xfa, 0x68, 0xf0, 0xff, 0x77, 0x98, 0x71, 0x30, 0x30, 0x5b, 0xea, 0xba, 0x2e, 0xda, 0x04, + 0xdf, 0x99, 0x7b, 0x71, 0x4d, 0x6c, 0x6f, 0x2c, 0x29, 0xa6, 0xad, 0x5c, 0xb4, 0x02, 0x2b, 0x02, 0x70, 0x9b, 0xee, + 0xad, 0x9d, 0x67, 0x89, 0x0c, 0xbb, 0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f, 0x38 +}; +static CONST UINT8 DecOutput001[] = { + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, + 0x20, 0x64, 0x72, 0x61, 0x66, 0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, + 0x6c, 0x69, 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20, 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20, 0x6f, + 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d, 0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x6d, 0x61, + 0x79, 0x20, 0x62, 0x65, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, + 0x63, 0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f, 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x20, 0x62, + 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x20, 0x61, + 0x74, 0x20, 0x61, 0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69, + 0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72, 0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20, 0x75, 0x73, 0x65, + 0x20, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x73, + 0x20, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, 0x6c, + 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, 0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65, 0x6d, 0x20, 0x6f, 0x74, + 0x68, 0x65, 0x72, 0x20, 0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20, 0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, + 0x6b, 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d +}; +static CONST UINT8 DecAssoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; +static CONST UINT8 DecNonce001[] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 }; +static CONST UINT8 DecKey001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, + 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, + 0x80, 0x09, 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 }; + +static CONST UINT8 DecInput002[] = { 0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1, + 0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92 }; +// static CONST UINT8 DecOutput002[] = { }; +// static CONST UINT8 DecAssoc002[] = { }; +static CONST UINT8 DecNonce002[] = { 0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e }; +static CONST UINT8 DecKey002[] = { 0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f, 0x2d, 0x29, 0x25, + 0x76, 0xd5, 0x75, 0x27, 0x86, 0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, + 0xc5, 0xef, 0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68 }; + +static CONST UINT8 DecInput003[] = { 0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6, + 0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77 }; +// static CONST UINT8 DecOutput003[] = { }; +static CONST UINT8 DecAssoc003[] = { 0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b }; +static CONST UINT8 DecNonce003[] = { 0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d }; +static CONST UINT8 DecKey003[] = { 0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88, 0x34, 0xd1, 0x13, + 0xaf, 0x57, 0xa1, 0xeb, 0x3a, 0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, + 0xbc, 0x08, 0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d }; + +static CONST UINT8 DecInput004[] = { 0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2, 0x6d, + 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac, 0x89 }; +static CONST UINT8 DecOutput004[] = { 0xa4 }; +static CONST UINT8 DecAssoc004[] = { 0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40 }; +static CONST UINT8 DecNonce004[] = { 0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4 }; +static CONST UINT8 DecKey004[] = { 0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8, 0x31, 0x80, 0x82, + 0xd7, 0xd8, 0xe8, 0xb5, 0xa1, 0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, + 0xa3, 0x3d, 0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e }; + +static CONST UINT8 DecInput005[] = { 0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e, 0x6c, + 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c, 0xac }; +static CONST UINT8 DecOutput005[] = { 0x2d }; +// static CONST UINT8 DecAssoc005[] = { }; +static CONST UINT8 DecNonce005[] = { 0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30 }; +static CONST UINT8 DecKey005[] = { 0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31, 0x0e, 0x92, 0x89, + 0x8b, 0xf4, 0x93, 0xc7, 0x87, 0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, + 0xa7, 0x01, 0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87 }; + +static CONST UINT8 DecInput006[] = { + 0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1, 0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15, 0x81, 0x2c, 0xb5, + 0xf0, 0xc6, 0x2b, 0xc7, 0x8c, 0x56, 0xd1, 0xbf, 0x69, 0x6c, 0x07, 0xa0, 0xda, 0x65, 0x27, 0xc9, 0x90, 0x3d, 0xef, + 0x4b, 0x11, 0x0f, 0x19, 0x07, 0xfd, 0x29, 0x92, 0xd9, 0xc8, 0xf7, 0x99, 0x2e, 0x4a, 0xd0, 0xb8, 0x2c, 0xdc, 0x93, + 0xf5, 0x9e, 0x33, 0x78, 0xd1, 0x37, 0xc3, 0x66, 0xd7, 0x5e, 0xbc, 0x44, 0xbf, 0x53, 0xa5, 0xbc, 0xc4, 0xcb, 0x7b, + 0x3a, 0x8e, 0x7f, 0x02, 0xbd, 0xbb, 0xe7, 0xca, 0xa6, 0x6c, 0x6b, 0x93, 0x21, 0x93, 0x10, 0x61, 0xe7, 0x69, 0xd0, + 0x78, 0xf3, 0x07, 0x5a, 0x1a, 0x8f, 0x73, 0xaa, 0xb1, 0x4e, 0xd3, 0xda, 0x4f, 0xf3, 0x32, 0xe1, 0x66, 0x3e, 0x6c, + 0xc6, 0x13, 0xba, 0x06, 0x5b, 0xfc, 0x6a, 0xe5, 0x6f, 0x60, 0xfb, 0x07, 0x40, 0xb0, 0x8c, 0x9d, 0x84, 0x43, 0x6b, + 0xc1, 0xf7, 0x8d, 0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a, 0xeb +}; +static CONST UINT8 DecOutput006[] = { + 0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a, 0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92, 0x3c, 0xd9, 0x24, + 0x11, 0xa9, 0x71, 0xf9, 0x37, 0x14, 0x99, 0xfa, 0xbe, 0xe6, 0x80, 0xde, 0x50, 0xc9, 0x96, 0xd4, 0xb0, 0xec, 0x9e, + 0x17, 0xec, 0xd2, 0x5e, 0x72, 0x99, 0xfc, 0x0a, 0xe1, 0xcb, 0x48, 0xd2, 0x85, 0xdd, 0x2f, 0x90, 0xe0, 0x66, 0x3b, + 0xe6, 0x20, 0x74, 0xbe, 0x23, 0x8f, 0xcb, 0xb4, 0xe4, 0xda, 0x48, 0x40, 0xa6, 0xd1, 0x1b, 0xc7, 0x42, 0xce, 0x2f, + 0x0c, 0xa6, 0x85, 0x6e, 0x87, 0x37, 0x03, 0xb1, 0x7c, 0x25, 0x96, 0xa3, 0x05, 0xd8, 0xb0, 0xf4, 0xed, 0xea, 0xc2, + 0xf0, 0x31, 0x98, 0x6c, 0xd1, 0x14, 0x25, 0xc0, 0xcb, 0x01, 0x74, 0xd0, 0x82, 0xf4, 0x36, 0xf5, 0x41, 0xd5, 0xdc, + 0xca, 0xc5, 0xbb, 0x98, 0xfe, 0xfc, 0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde, 0x8f +}; +static CONST UINT8 DecAssoc006[] = { 0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b }; +static CONST UINT8 DecNonce006[] = { 0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c }; +static CONST UINT8 DecKey006[] = { 0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae, 0xdf, 0x72, 0x7f, + 0x53, 0x72, 0x25, 0x1e, 0x78, 0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, + 0x93, 0xf9, 0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01 }; + +static CONST UINT8 DecInput007[] = { + 0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c, 0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8, 0xc9, 0x50, 0xc3, + 0xc6, 0xa5, 0xe3, 0xa4, 0x7c, 0xc3, 0x23, 0x49, 0x5e, 0xa9, 0xb9, 0x32, 0xeb, 0x8a, 0x7c, 0xca, 0xe5, 0xec, 0xfb, + 0x7c, 0xc0, 0xcb, 0x7d, 0xdc, 0x2c, 0x9d, 0x92, 0x55, 0x21, 0x0a, 0xc8, 0x43, 0x63, 0x59, 0x0a, 0x31, 0x70, 0x82, + 0x67, 0x41, 0x03, 0xf8, 0xdf, 0xf2, 0xac, 0xa7, 0x02, 0xd4, 0xd5, 0x8a, 0x2d, 0xc8, 0x99, 0x19, 0x66, 0xd0, 0xf6, + 0x88, 0x2c, 0x77, 0xd9, 0xd4, 0x0d, 0x6c, 0xbd, 0x98, 0xde, 0xe7, 0x7f, 0xad, 0x7e, 0x8a, 0xfb, 0xe9, 0x4b, 0xe5, + 0xf7, 0xe5, 0x50, 0xa0, 0x90, 0x3f, 0xd6, 0x22, 0x53, 0xe3, 0xfe, 0x1b, 0xcc, 0x79, 0x3b, 0xec, 0x12, 0x47, 0x52, + 0xa7, 0xd6, 0x04, 0xe3, 0x52, 0xe6, 0x93, 0x90, 0x91, 0x32, 0x73, 0x79, 0xb8, 0xd0, 0x31, 0xde, 0x1f, 0x9f, 0x2f, + 0x05, 0x38, 0x54, 0x2f, 0x35, 0x04, 0x39, 0xe0, 0xa7, 0xba, 0xc6, 0x52, 0xf6, 0x37, 0x65, 0x4c, 0x07, 0xa9, 0x7e, + 0xb3, 0x21, 0x6f, 0x74, 0x8c, 0xc9, 0xde, 0xdb, 0x65, 0x1b, 0x9b, 0xaa, 0x60, 0xb1, 0x03, 0x30, 0x6b, 0xb2, 0x03, + 0xc4, 0x1c, 0x04, 0xf8, 0x0f, 0x64, 0xaf, 0x46, 0xe4, 0x65, 0x99, 0x49, 0xe2, 0xea, 0xce, 0x78, 0x00, 0xd8, 0x8b, + 0xd5, 0x2e, 0xcf, 0xfc, 0x40, 0x49, 0xe8, 0x58, 0xdc, 0x34, 0x9c, 0x8c, 0x61, 0xbf, 0x0a, 0x8e, 0xec, 0x39, 0xa9, + 0x30, 0x05, 0x5a, 0xd2, 0x56, 0x01, 0xc7, 0xda, 0x8f, 0x4e, 0xbb, 0x43, 0xa3, 0x3a, 0xf9, 0x15, 0x2a, 0xd0, 0xa0, + 0x7a, 0x87, 0x34, 0x82, 0xfe, 0x8a, 0xd1, 0x2d, 0x5e, 0xc7, 0xbf, 0x04, 0x53, 0x5f, 0x3b, 0x36, 0xd4, 0x25, 0x5c, + 0x34, 0x7a, 0x8d, 0xd5, 0x05, 0xce, 0x72, 0xca, 0xef, 0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42, 0x3a, 0x00, + 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53 +}; +static CONST UINT8 DecOutput007[] = { + 0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5, 0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a, 0x67, 0x30, 0x12, + 0xe2, 0x34, 0x77, 0x4b, 0xc1, 0xd9, 0xc6, 0x1f, 0xab, 0xc6, 0x18, 0x50, 0x17, 0xa7, 0x9d, 0x3c, 0xa6, 0xc5, 0x35, + 0x8c, 0x1c, 0xc0, 0xa1, 0x7c, 0x9f, 0x03, 0x89, 0xca, 0xe1, 0xe6, 0xe9, 0xd4, 0xd3, 0x88, 0xdb, 0xb4, 0x51, 0x9d, + 0xec, 0xb4, 0xfc, 0x52, 0xee, 0x6d, 0xf1, 0x75, 0x42, 0xc6, 0xfd, 0xbd, 0x7a, 0x8e, 0x86, 0xfc, 0x44, 0xb3, 0x4f, + 0xf3, 0xea, 0x67, 0x5a, 0x41, 0x13, 0xba, 0xb0, 0xdc, 0xe1, 0xd3, 0x2a, 0x7c, 0x22, 0xb3, 0xca, 0xac, 0x6a, 0x37, + 0x98, 0x3e, 0x1d, 0x40, 0x97, 0xf7, 0x9b, 0x1d, 0x36, 0x6b, 0xb3, 0x28, 0xbd, 0x60, 0x82, 0x47, 0x34, 0xaa, 0x2f, + 0x7d, 0xe9, 0xa8, 0x70, 0x81, 0x57, 0xd4, 0xb9, 0x77, 0x0a, 0x9d, 0x29, 0xa7, 0x84, 0x52, 0x4f, 0xc2, 0x4a, 0x40, + 0x3b, 0x3c, 0xd4, 0xc9, 0x2a, 0xdb, 0x4a, 0x53, 0xc4, 0xbe, 0x80, 0xe9, 0x51, 0x7f, 0x8f, 0xc7, 0xa2, 0xce, 0x82, + 0x5c, 0x91, 0x1e, 0x74, 0xd9, 0xd0, 0xbd, 0xd5, 0xf3, 0xfd, 0xda, 0x4d, 0x25, 0xb4, 0xbb, 0x2d, 0xac, 0x2f, 0x3d, + 0x71, 0x85, 0x7b, 0xcf, 0x3c, 0x7b, 0x3e, 0x0e, 0x22, 0x78, 0x0c, 0x29, 0xbf, 0xe4, 0xf4, 0x57, 0xb3, 0xcb, 0x49, + 0xa0, 0xfc, 0x1e, 0x05, 0x4e, 0x16, 0xbc, 0xd5, 0xa8, 0xa3, 0xee, 0x05, 0x35, 0xc6, 0x7c, 0xab, 0x60, 0x14, 0x55, + 0x1a, 0x8e, 0xc5, 0x88, 0x5d, 0xd5, 0x81, 0xc2, 0x81, 0xa5, 0xc4, 0x60, 0xdb, 0xaf, 0x77, 0x91, 0xe1, 0xce, 0xa2, + 0x7e, 0x7f, 0x42, 0xe3, 0xb0, 0x13, 0x1c, 0x1f, 0x25, 0x60, 0x21, 0xe2, 0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, + 0x2b, 0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3 +}; +// static CONST UINT8 DecAssoc007[] = { }; +static CONST UINT8 DecNonce007[] = { 0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0 }; +static CONST UINT8 DecKey007[] = { 0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd, 0xf9, 0x3f, 0xcd, + 0xd9, 0xa0, 0x1e, 0x42, 0x4c, 0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, + 0x05, 0x80, 0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01 }; + +static CONST UINT8 DecInput008[] = { + 0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd, 0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1, 0x1e, 0x6b, 0xd2, + 0xbc, 0x11, 0xf4, 0x28, 0x93, 0x63, 0x54, 0xef, 0xbb, 0x5e, 0x1d, 0x3a, 0x1d, 0x37, 0x3c, 0x0a, 0x6c, 0x1e, 0xc2, + 0xd1, 0x2c, 0xb5, 0xa3, 0xb5, 0x7b, 0xb8, 0x8f, 0x25, 0xa6, 0x1b, 0x61, 0x1c, 0xec, 0x28, 0x58, 0x26, 0xa4, 0xa8, + 0x33, 0x28, 0x25, 0x5c, 0x45, 0x05, 0xe5, 0x6c, 0x99, 0xe5, 0x45, 0xc4, 0xa2, 0x03, 0x84, 0x03, 0x73, 0x1e, 0x8c, + 0x49, 0xac, 0x20, 0xdd, 0x8d, 0xb3, 0xc4, 0xf5, 0xe7, 0x4f, 0xf1, 0xed, 0xa1, 0x98, 0xde, 0xa4, 0x96, 0xdd, 0x2f, + 0xab, 0xab, 0x97, 0xcf, 0x3e, 0xd2, 0x9e, 0xb8, 0x13, 0x07, 0x28, 0x29, 0x19, 0xaf, 0xfd, 0xf2, 0x49, 0x43, 0xea, + 0x49, 0x26, 0x91, 0xc1, 0x07, 0xd6, 0xbb, 0x81, 0x75, 0x35, 0x0d, 0x24, 0x7f, 0xc8, 0xda, 0xd4, 0xb7, 0xeb, 0xe8, + 0x5c, 0x09, 0xa2, 0x2f, 0xdc, 0x28, 0x7d, 0x3a, 0x03, 0xfa, 0x94, 0xb5, 0x1d, 0x17, 0x99, 0x36, 0xc3, 0x1c, 0x18, + 0x34, 0xe3, 0x9f, 0xf5, 0x55, 0x7c, 0xb0, 0x60, 0x9d, 0xff, 0xac, 0xd4, 0x61, 0xf2, 0xad, 0xf8, 0xce, 0xc7, 0xbe, + 0x5c, 0xd2, 0x95, 0xa8, 0x4b, 0x77, 0x13, 0x19, 0x59, 0x26, 0xc9, 0xb7, 0x8f, 0x6a, 0xcb, 0x2d, 0x37, 0x91, 0xea, + 0x92, 0x9c, 0x94, 0x5b, 0xda, 0x0b, 0xce, 0xfe, 0x30, 0x20, 0xf8, 0x51, 0xad, 0xf2, 0xbe, 0xe7, 0xc7, 0xff, 0xb3, + 0x33, 0x91, 0x6a, 0xc9, 0x1a, 0x41, 0xc9, 0x0f, 0xf3, 0x10, 0x0e, 0xfd, 0x53, 0xff, 0x6c, 0x16, 0x52, 0xd9, 0xf3, + 0xf7, 0x98, 0x2e, 0xc9, 0x07, 0x31, 0x2c, 0x0c, 0x72, 0xd7, 0xc5, 0xc6, 0x08, 0x2a, 0x7b, 0xda, 0xbd, 0x7e, 0x02, + 0xea, 0x1a, 0xbb, 0xf2, 0x04, 0x27, 0x61, 0x28, 0x8e, 0xf5, 0x04, 0x03, 0x1f, 0x4c, 0x07, 0x55, 0x82, 0xec, 0x1e, + 0xd7, 0x8b, 0x2f, 0x65, 0x56, 0xd1, 0xd9, 0x1e, 0x3c, 0xe9, 0x1f, 0x5e, 0x98, 0x70, 0x38, 0x4a, 0x8c, 0x49, 0xc5, + 0x43, 0xa0, 0xa1, 0x8b, 0x74, 0x9d, 0x4c, 0x62, 0x0d, 0x10, 0x0c, 0xf4, 0x6c, 0x8f, 0xe0, 0xaa, 0x9a, 0x8d, 0xb7, + 0xe0, 0xbe, 0x4c, 0x87, 0xf1, 0x98, 0x2f, 0xcc, 0xed, 0xc0, 0x52, 0x29, 0xdc, 0x83, 0xf8, 0xfc, 0x2c, 0x0e, 0xa8, + 0x51, 0x4d, 0x80, 0x0d, 0xa3, 0xfe, 0xd8, 0x37, 0xe7, 0x41, 0x24, 0xfc, 0xfb, 0x75, 0xe3, 0x71, 0x7b, 0x57, 0x45, + 0xf5, 0x97, 0x73, 0x65, 0x63, 0x14, 0x74, 0xb8, 0x82, 0x9f, 0xf8, 0x60, 0x2f, 0x8a, 0xf2, 0x4e, 0xf1, 0x39, 0xda, + 0x33, 0x91, 0xf8, 0x36, 0xe0, 0x8d, 0x3f, 0x1f, 0x3b, 0x56, 0xdc, 0xa0, 0x8f, 0x3c, 0x9d, 0x71, 0x52, 0xa7, 0xb8, + 0xc0, 0xa5, 0xc6, 0xa2, 0x73, 0xda, 0xf4, 0x4b, 0x74, 0x5b, 0x00, 0x3d, 0x99, 0xd7, 0x96, 0xba, 0xe6, 0xe1, 0xa6, + 0x96, 0x38, 0xad, 0xb3, 0xc0, 0xd2, 0xba, 0x91, 0x6b, 0xf9, 0x19, 0xdd, 0x3b, 0xbe, 0xbe, 0x9c, 0x20, 0x50, 0xba, + 0xa1, 0xd0, 0xce, 0x11, 0xbd, 0x95, 0xd8, 0xd1, 0xdd, 0x33, 0x85, 0x74, 0xdc, 0xdb, 0x66, 0x76, 0x44, 0xdc, 0x03, + 0x74, 0x48, 0x35, 0x98, 0xb1, 0x18, 0x47, 0x94, 0x7d, 0xff, 0x62, 0xe4, 0x58, 0x78, 0xab, 0xed, 0x95, 0x36, 0xd9, + 0x84, 0x91, 0x82, 0x64, 0x41, 0xbb, 0x58, 0xe6, 0x1c, 0x20, 0x6d, 0x15, 0x6b, 0x13, 0x96, 0xe8, 0x35, 0x7f, 0xdc, + 0x40, 0x2c, 0xe9, 0xbc, 0x8a, 0x4f, 0x92, 0xec, 0x06, 0x2d, 0x50, 0xdf, 0x93, 0x5d, 0x65, 0x5a, 0xa8, 0xfc, 0x20, + 0x50, 0x14, 0xa9, 0x8a, 0x7e, 0x1d, 0x08, 0x1f, 0xe2, 0x99, 0xd0, 0xbe, 0xfb, 0x3a, 0x21, 0x9d, 0xad, 0x86, 0x54, + 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f, 0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54 +}; +static CONST UINT8 DecOutput008[] = { + 0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10, 0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2, 0x0f, 0xc2, 0x8b, + 0x28, 0xdc, 0xba, 0xb4, 0x3c, 0xe4, 0x21, 0x58, 0x61, 0xcd, 0x8b, 0xcd, 0xfb, 0xac, 0x94, 0xa1, 0x45, 0xf5, 0x1c, + 0xe1, 0x12, 0xe0, 0x3b, 0x67, 0x21, 0x54, 0x5e, 0x8c, 0xaa, 0xcf, 0xdb, 0xb4, 0x51, 0xd4, 0x13, 0xda, 0xe6, 0x83, + 0x89, 0xb6, 0x92, 0xe9, 0x21, 0x76, 0xa4, 0x93, 0x7d, 0x0e, 0xfd, 0x96, 0x36, 0x03, 0x91, 0x43, 0x5c, 0x92, 0x49, + 0x62, 0x61, 0x7b, 0xeb, 0x43, 0x89, 0xb8, 0x12, 0x20, 0x43, 0xd4, 0x47, 0x06, 0x84, 0xee, 0x47, 0xe9, 0x8a, 0x73, + 0x15, 0x0f, 0x72, 0xcf, 0xed, 0xce, 0x96, 0xb2, 0x7f, 0x21, 0x45, 0x76, 0xeb, 0x26, 0x28, 0x83, 0x6a, 0xad, 0xaa, + 0xa6, 0x81, 0xd8, 0x55, 0xb1, 0xa3, 0x85, 0xb3, 0x0c, 0xdf, 0xf1, 0x69, 0x2d, 0x97, 0x05, 0x2a, 0xbc, 0x7c, 0x7b, + 0x25, 0xf8, 0x80, 0x9d, 0x39, 0x25, 0xf3, 0x62, 0xf0, 0x66, 0x5e, 0xf4, 0xa0, 0xcf, 0xd8, 0xfd, 0x4f, 0xb1, 0x1f, + 0x60, 0x3a, 0x08, 0x47, 0xaf, 0xe1, 0xf6, 0x10, 0x77, 0x09, 0xa7, 0x27, 0x8f, 0x9a, 0x97, 0x5a, 0x26, 0xfa, 0xfe, + 0x41, 0x32, 0x83, 0x10, 0xe0, 0x1d, 0xbf, 0x64, 0x0d, 0xf4, 0x1c, 0x32, 0x35, 0xe5, 0x1b, 0x36, 0xef, 0xd4, 0x4a, + 0x93, 0x4d, 0x00, 0x7c, 0xec, 0x02, 0x07, 0x8b, 0x5d, 0x7d, 0x1b, 0x0e, 0xd1, 0xa6, 0xa5, 0x5d, 0x7d, 0x57, 0x88, + 0xa8, 0xcc, 0x81, 0xb4, 0x86, 0x4e, 0xb4, 0x40, 0xe9, 0x1d, 0xc3, 0xb1, 0x24, 0x3e, 0x7f, 0xcc, 0x8a, 0x24, 0x9b, + 0xdf, 0x6d, 0xf0, 0x39, 0x69, 0x3e, 0x4c, 0xc0, 0x96, 0xe4, 0x13, 0xda, 0x90, 0xda, 0xf4, 0x95, 0x66, 0x8b, 0x17, + 0x17, 0xfe, 0x39, 0x43, 0x25, 0xaa, 0xda, 0xa0, 0x43, 0x3c, 0xb1, 0x41, 0x02, 0xa3, 0xf0, 0xa7, 0x19, 0x59, 0xbc, + 0x1d, 0x7d, 0x6c, 0x6d, 0x91, 0x09, 0x5c, 0xb7, 0x5b, 0x01, 0xd1, 0x6f, 0x17, 0x21, 0x97, 0xbf, 0x89, 0x71, 0xa5, + 0xb0, 0x6e, 0x07, 0x45, 0xfd, 0x9d, 0xea, 0x07, 0xf6, 0x7a, 0x9f, 0x10, 0x18, 0x22, 0x30, 0x73, 0xac, 0xd4, 0x6b, + 0x72, 0x44, 0xed, 0xd9, 0x19, 0x9b, 0x2d, 0x4a, 0x41, 0xdd, 0xd1, 0x85, 0x5e, 0x37, 0x19, 0xed, 0xd2, 0x15, 0x8f, + 0x5e, 0x91, 0xdb, 0x33, 0xf2, 0xe4, 0xdb, 0xff, 0x98, 0xfb, 0xa3, 0xb5, 0xca, 0x21, 0x69, 0x08, 0xe7, 0x8a, 0xdf, + 0x90, 0xff, 0x3e, 0xe9, 0x20, 0x86, 0x3c, 0xe9, 0xfc, 0x0b, 0xfe, 0x5c, 0x61, 0xaa, 0x13, 0x92, 0x7f, 0x7b, 0xec, + 0xe0, 0x6d, 0xa8, 0x23, 0x22, 0xf6, 0x6b, 0x77, 0xc4, 0xfe, 0x40, 0x07, 0x3b, 0xb6, 0xf6, 0x8e, 0x5f, 0xd4, 0xb9, + 0xb7, 0x0f, 0x21, 0x04, 0xef, 0x83, 0x63, 0x91, 0x69, 0x40, 0xa3, 0x48, 0x5c, 0xd2, 0x60, 0xf9, 0x4f, 0x6c, 0x47, + 0x8b, 0x3b, 0xb1, 0x9f, 0x8e, 0xee, 0x16, 0x8a, 0x13, 0xfc, 0x46, 0x17, 0xc3, 0xc3, 0x32, 0x56, 0xf8, 0x3c, 0x85, + 0x3a, 0xb6, 0x3e, 0xaa, 0x89, 0x4f, 0xb3, 0xdf, 0x38, 0xfd, 0xf1, 0xe4, 0x3a, 0xc0, 0xe6, 0x58, 0xb5, 0x8f, 0xc5, + 0x29, 0xa2, 0x92, 0x4a, 0xb6, 0xa0, 0x34, 0x7f, 0xab, 0xb5, 0x8a, 0x90, 0xa1, 0xdb, 0x4d, 0xca, 0xb6, 0x2c, 0x41, + 0x3c, 0xf7, 0x2b, 0x21, 0xc3, 0xfd, 0xf4, 0x17, 0x5c, 0xb5, 0x33, 0x17, 0x68, 0x2b, 0x08, 0x30, 0xf3, 0xf7, 0x30, + 0x3c, 0x96, 0xe6, 0x6a, 0x20, 0x97, 0xe7, 0x4d, 0x10, 0x5f, 0x47, 0x5f, 0x49, 0x96, 0x09, 0xf0, 0x27, 0x91, 0xc8, + 0xf8, 0x5a, 0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b, 0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3 +}; +// static CONST UINT8 DecAssoc008[] = { }; +static CONST UINT8 DecNonce008[] = { 0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02 }; +static CONST UINT8 DecKey008[] = { 0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53, 0xc1, 0x44, 0xe9, + 0x81, 0x18, 0xdc, 0xf5, 0xf0, 0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, + 0x44, 0x86, 0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba }; + +static CONST UINT8 DecInput009[] = { + 0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf, 0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66, 0x4b, 0x2e, 0x0c, + 0x27, 0x9c, 0x96, 0x4c, 0x72, 0x02, 0xa3, 0x65, 0xc3, 0xb3, 0x6f, 0x2e, 0xbd, 0x63, 0x8a, 0x4a, 0x5d, 0x29, 0xa2, + 0xd0, 0x28, 0x48, 0xc5, 0x3d, 0x98, 0xa3, 0xbc, 0xe0, 0xbe, 0x3b, 0x3f, 0xe6, 0x8a, 0xa4, 0x7f, 0x53, 0x06, 0xfa, + 0x7f, 0x27, 0x76, 0x72, 0x31, 0xa1, 0xf5, 0xd6, 0x0c, 0x52, 0x47, 0xba, 0xcd, 0x4f, 0xd7, 0xeb, 0x05, 0x48, 0x0d, + 0x7c, 0x35, 0x4a, 0x09, 0xc9, 0x76, 0x71, 0x02, 0xa3, 0xfb, 0xb7, 0x1a, 0x65, 0xb7, 0xed, 0x98, 0xc6, 0x30, 0x8a, + 0x00, 0xae, 0xa1, 0x31, 0xe5, 0xb5, 0x9e, 0x6d, 0x62, 0xda, 0xda, 0x07, 0x0f, 0x38, 0x38, 0xd3, 0xcb, 0xc1, 0xb0, + 0xad, 0xec, 0x72, 0xec, 0xb1, 0xa2, 0x7b, 0x59, 0xf3, 0x3d, 0x2b, 0xef, 0xcd, 0x28, 0x5b, 0x83, 0xcc, 0x18, 0x91, + 0x88, 0xb0, 0x2e, 0xf9, 0x29, 0x31, 0x18, 0xf9, 0x4e, 0xe9, 0x0a, 0x91, 0x92, 0x9f, 0xae, 0x2d, 0xad, 0xf4, 0xe6, + 0x1a, 0xe2, 0xa4, 0xee, 0x47, 0x15, 0xbf, 0x83, 0x6e, 0xd7, 0x72, 0x12, 0x3b, 0x2d, 0x24, 0xe9, 0xb2, 0x55, 0xcb, + 0x3c, 0x10, 0xf0, 0x24, 0x8a, 0x4a, 0x02, 0xea, 0x90, 0x25, 0xf0, 0xb4, 0x79, 0x3a, 0xef, 0x6e, 0xf5, 0x52, 0xdf, + 0xb0, 0x0a, 0xcd, 0x24, 0x1c, 0xd3, 0x2e, 0x22, 0x74, 0xea, 0x21, 0x6f, 0xe9, 0xbd, 0xc8, 0x3e, 0x36, 0x5b, 0x19, + 0xf1, 0xca, 0x99, 0x0a, 0xb4, 0xa7, 0x52, 0x1a, 0x4e, 0xf2, 0xad, 0x8d, 0x56, 0x85, 0xbb, 0x64, 0x89, 0xba, 0x26, + 0xf9, 0xc7, 0xe1, 0x89, 0x19, 0x22, 0x77, 0xc3, 0xa8, 0xfc, 0xff, 0xad, 0xfe, 0xb9, 0x48, 0xae, 0x12, 0x30, 0x9f, + 0x19, 0xfb, 0x1b, 0xef, 0x14, 0x87, 0x8a, 0x78, 0x71, 0xf3, 0xf4, 0xb7, 0x00, 0x9c, 0x1d, 0xb5, 0x3d, 0x49, 0x00, + 0x0c, 0x06, 0xd4, 0x50, 0xf9, 0x54, 0x45, 0xb2, 0x5b, 0x43, 0xdb, 0x6d, 0xcf, 0x1a, 0xe9, 0x7a, 0x7a, 0xcf, 0xfc, + 0x8a, 0x4e, 0x4d, 0x0b, 0x07, 0x63, 0x28, 0xd8, 0xe7, 0x08, 0x95, 0xdf, 0xa6, 0x72, 0x93, 0x2e, 0xbb, 0xa0, 0x42, + 0x89, 0x16, 0xf1, 0xd9, 0x0c, 0xf9, 0xa1, 0x16, 0xfd, 0xd9, 0x03, 0xb4, 0x3b, 0x8a, 0xf5, 0xf6, 0xe7, 0x6b, 0x2e, + 0x8e, 0x4c, 0x3d, 0xe2, 0xaf, 0x08, 0x45, 0x03, 0xff, 0x09, 0xb6, 0xeb, 0x2d, 0xc6, 0x1b, 0x88, 0x94, 0xac, 0x3e, + 0xf1, 0x9f, 0x0e, 0x0e, 0x2b, 0xd5, 0x00, 0x4d, 0x3f, 0x3b, 0x53, 0xae, 0xaf, 0x1c, 0x33, 0x5f, 0x55, 0x6e, 0x8d, + 0xaf, 0x05, 0x7a, 0x10, 0x34, 0xc9, 0xf4, 0x66, 0xcb, 0x62, 0x12, 0xa6, 0xee, 0xe8, 0x1c, 0x5d, 0x12, 0x86, 0xdb, + 0x6f, 0x1c, 0x33, 0xc4, 0x1c, 0xda, 0x82, 0x2d, 0x3b, 0x59, 0xfe, 0xb1, 0xa4, 0x59, 0x41, 0x86, 0xd0, 0xef, 0xae, + 0xfb, 0xda, 0x6d, 0x11, 0xb8, 0xca, 0xe9, 0x6e, 0xff, 0xf7, 0xa9, 0xd9, 0x70, 0x30, 0xfc, 0x53, 0xe2, 0xd7, 0xa2, + 0x4e, 0xc7, 0x91, 0xd9, 0x07, 0x06, 0xaa, 0xdd, 0xb0, 0x59, 0x28, 0x1d, 0x00, 0x66, 0xc5, 0x54, 0xc2, 0xfc, 0x06, + 0xda, 0x05, 0x90, 0x52, 0x1d, 0x37, 0x66, 0xee, 0xf0, 0xb2, 0x55, 0x8a, 0x5d, 0xd2, 0x38, 0x86, 0x94, 0x9b, 0xfc, + 0x10, 0x4c, 0xa1, 0xb9, 0x64, 0x3e, 0x44, 0xb8, 0x5f, 0xb0, 0x0c, 0xec, 0xe0, 0xc9, 0xe5, 0x62, 0x75, 0x3f, 0x09, + 0xd5, 0xf5, 0xd9, 0x26, 0xba, 0x9e, 0xd2, 0xf4, 0xb9, 0x48, 0x0a, 0xbc, 0xa2, 0xd6, 0x7c, 0x36, 0x11, 0x7d, 0x26, + 0x81, 0x89, 0xcf, 0xa4, 0xad, 0x73, 0x0e, 0xee, 0xcc, 0x06, 0xa9, 0xdb, 0xb1, 0xfd, 0xfb, 0x09, 0x7f, 0x90, 0x42, + 0x37, 0x2f, 0xe1, 0x9c, 0x0f, 0x6f, 0xcf, 0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8, 0xae +}; +static CONST UINT8 DecOutput009[] = { + 0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b, 0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8, 0x3a, 0x6b, 0xd7, + 0x81, 0x96, 0x35, 0x97, 0xca, 0xd7, 0x68, 0xe8, 0xef, 0xce, 0xab, 0xda, 0x09, 0x6e, 0xd6, 0x8e, 0xcb, 0x55, 0xb5, + 0xe1, 0xe5, 0x57, 0xfd, 0xc4, 0xe3, 0xe0, 0x18, 0x4f, 0x85, 0xf5, 0x3f, 0x7e, 0x4b, 0x88, 0xc9, 0x52, 0x44, 0x0f, + 0xea, 0xaf, 0x1f, 0x71, 0x48, 0x9f, 0x97, 0x6d, 0xb9, 0x6f, 0x00, 0xa6, 0xde, 0x2b, 0x77, 0x8b, 0x15, 0xad, 0x10, + 0xa0, 0x2b, 0x7b, 0x41, 0x90, 0x03, 0x2d, 0x69, 0xae, 0xcc, 0x77, 0x7c, 0xa5, 0x9d, 0x29, 0x22, 0xc2, 0xea, 0xb4, + 0x00, 0x1a, 0xd2, 0x7a, 0x98, 0x8a, 0xf9, 0xf7, 0x82, 0xb0, 0xab, 0xd8, 0xa6, 0x94, 0x8d, 0x58, 0x2f, 0x01, 0x9e, + 0x00, 0x20, 0xfc, 0x49, 0xdc, 0x0e, 0x03, 0xe8, 0x45, 0x10, 0xd6, 0xa8, 0xda, 0x55, 0x10, 0x9a, 0xdf, 0x67, 0x22, + 0x8b, 0x43, 0xab, 0x00, 0xbb, 0x02, 0xc8, 0xdd, 0x7b, 0x97, 0x17, 0xd7, 0x1d, 0x9e, 0x02, 0x5e, 0x48, 0xde, 0x8e, + 0xcf, 0x99, 0x07, 0x95, 0x92, 0x3c, 0x5f, 0x9f, 0xc5, 0x8a, 0xc0, 0x23, 0xaa, 0xd5, 0x8c, 0x82, 0x6e, 0x16, 0x92, + 0xb1, 0x12, 0x17, 0x07, 0xc3, 0xfb, 0x36, 0xf5, 0x6c, 0x35, 0xd6, 0x06, 0x1f, 0x9f, 0xa7, 0x94, 0xa2, 0x38, 0x63, + 0x9c, 0xb0, 0x71, 0xb3, 0xa5, 0xd2, 0xd8, 0xba, 0x9f, 0x08, 0x01, 0xb3, 0xff, 0x04, 0x97, 0x73, 0x45, 0x1b, 0xd5, + 0xa9, 0x9c, 0x80, 0xaf, 0x04, 0x9a, 0x85, 0xdb, 0x32, 0x5b, 0x5d, 0x1a, 0xc1, 0x36, 0x28, 0x10, 0x79, 0xf1, 0x3c, + 0xbf, 0x1a, 0x41, 0x5c, 0x4e, 0xdf, 0xb2, 0x7c, 0x79, 0x3b, 0x7a, 0x62, 0x3d, 0x4b, 0xc9, 0x9b, 0x2a, 0x2e, 0x7c, + 0xa2, 0xb1, 0x11, 0x98, 0xa7, 0x34, 0x1a, 0x00, 0xf3, 0xd1, 0xbc, 0x18, 0x22, 0xba, 0x02, 0x56, 0x62, 0x31, 0x10, + 0x11, 0x6d, 0xe0, 0x54, 0x9d, 0x40, 0x1f, 0x26, 0x80, 0x41, 0xca, 0x3f, 0x68, 0x0f, 0x32, 0x1d, 0x0a, 0x8e, 0x79, + 0xd8, 0xa4, 0x1b, 0x29, 0x1c, 0x90, 0x8e, 0xc5, 0xe3, 0xb4, 0x91, 0x37, 0x9a, 0x97, 0x86, 0x99, 0xd5, 0x09, 0xc5, + 0xbb, 0xa3, 0x3f, 0x21, 0x29, 0x82, 0x14, 0x5c, 0xab, 0x25, 0xfb, 0xf2, 0x4f, 0x58, 0x26, 0xd4, 0x83, 0xaa, 0x66, + 0x89, 0x67, 0x7e, 0xc0, 0x49, 0xe1, 0x11, 0x10, 0x7f, 0x7a, 0xda, 0x29, 0x04, 0xff, 0xf0, 0xcb, 0x09, 0x7c, 0x9d, + 0xfa, 0x03, 0x6f, 0x81, 0x09, 0x31, 0x60, 0xfb, 0x08, 0xfa, 0x74, 0xd3, 0x64, 0x44, 0x7c, 0x55, 0x85, 0xec, 0x9c, + 0x6e, 0x25, 0xb7, 0x6c, 0xc5, 0x37, 0xb6, 0x83, 0x87, 0x72, 0x95, 0x8b, 0x9d, 0xe1, 0x69, 0x5c, 0x31, 0x95, 0x42, + 0xa6, 0x2c, 0xd1, 0x36, 0x47, 0x1f, 0xec, 0x54, 0xab, 0xa2, 0x1c, 0xd8, 0x00, 0xcc, 0xbc, 0x0d, 0x65, 0xe2, 0x67, + 0xbf, 0xbc, 0xea, 0xee, 0x9e, 0xe4, 0x36, 0x95, 0xbe, 0x73, 0xd9, 0xa6, 0xd9, 0x0f, 0xa0, 0xcc, 0x82, 0x76, 0x26, + 0xad, 0x5b, 0x58, 0x6c, 0x4e, 0xab, 0x29, 0x64, 0xd3, 0xd9, 0xa9, 0x08, 0x8c, 0x1d, 0xa1, 0x4f, 0x80, 0xd8, 0x3f, + 0x94, 0xfb, 0xd3, 0x7b, 0xfc, 0xd1, 0x2b, 0xc3, 0x21, 0xeb, 0xe5, 0x1c, 0x84, 0x23, 0x7f, 0x4b, 0xfa, 0xdb, 0x34, + 0x18, 0xa2, 0xc2, 0xe5, 0x13, 0xfe, 0x6c, 0x49, 0x81, 0xd2, 0x73, 0xe7, 0xe2, 0xd7, 0xe4, 0x4f, 0x4b, 0x08, 0x6e, + 0xb1, 0x12, 0x22, 0x10, 0x9d, 0xac, 0x51, 0x1e, 0x17, 0xd9, 0x8a, 0x0b, 0x42, 0x88, 0x16, 0x81, 0x37, 0x7c, 0x6a, + 0xf7, 0xef, 0x2d, 0xe3, 0xd9, 0xf8, 0x5f, 0xe0, 0x53, 0x27, 0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52, 0x65 +}; +static CONST UINT8 DecAssoc009[] = { 0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e, 0xef }; +static CONST UINT8 DecNonce009[] = { 0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78 }; +static CONST UINT8 DecKey009[] = { 0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5, 0xcc, 0x1a, 0xd7, + 0xc1, 0x57, 0x72, 0xea, 0x86, 0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, + 0x9b, 0xb2, 0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b }; + +static CONST UINT8 DecInput010[] = { + 0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b, 0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74, 0xa6, 0xdd, 0xbd, + 0x95, 0xeb, 0xf9, 0xa4, 0xf1, 0x59, 0x93, 0x91, 0x70, 0xd9, 0xfe, 0x9a, 0xcd, 0x53, 0x1f, 0x3a, 0xab, 0xa6, 0x7c, + 0x9f, 0xa6, 0x9e, 0xbd, 0x99, 0xd9, 0xb5, 0x97, 0x44, 0xd5, 0x14, 0x48, 0x4d, 0x9d, 0xc0, 0xd0, 0x05, 0x96, 0xeb, + 0x4c, 0x78, 0x55, 0x09, 0x08, 0x01, 0x02, 0x30, 0x90, 0x7b, 0x96, 0x7a, 0x7b, 0x5f, 0x30, 0x41, 0x24, 0xce, 0x68, + 0x61, 0x49, 0x86, 0x57, 0x82, 0xdd, 0x53, 0x1c, 0x51, 0x28, 0x2b, 0x53, 0x6e, 0x2d, 0xc2, 0x20, 0x4c, 0xdd, 0x8f, + 0x65, 0x10, 0x20, 0x50, 0xdd, 0x9d, 0x50, 0xe5, 0x71, 0x40, 0x53, 0x69, 0xfc, 0x77, 0x48, 0x11, 0xb9, 0xde, 0xa4, + 0x8d, 0x58, 0xe4, 0xa6, 0x1a, 0x18, 0x47, 0x81, 0x7e, 0xfc, 0xdd, 0xf6, 0xef, 0xce, 0x2f, 0x43, 0x68, 0xd6, 0x06, + 0xe2, 0x74, 0x6a, 0xad, 0x90, 0xf5, 0x37, 0xf3, 0x3d, 0x82, 0x69, 0x40, 0xe9, 0x6b, 0xa7, 0x3d, 0xa8, 0x1e, 0xd2, + 0x02, 0x7c, 0xb7, 0x9b, 0xe4, 0xda, 0x8f, 0x95, 0x06, 0xc5, 0xdf, 0x73, 0xa3, 0x20, 0x9a, 0x49, 0xde, 0x9c, 0xbc, + 0xee, 0x14, 0x3f, 0x81, 0x5e, 0xf8, 0x3b, 0x59, 0x3c, 0xe1, 0x68, 0x12, 0x5a, 0x3a, 0x76, 0x3a, 0x3f, 0xf7, 0x87, + 0x33, 0x0a, 0x01, 0xb8, 0xd4, 0xed, 0xb6, 0xbe, 0x94, 0x5e, 0x70, 0x40, 0x56, 0x67, 0x1f, 0x50, 0x44, 0x19, 0xce, + 0x82, 0x70, 0x10, 0x87, 0x13, 0x20, 0x0b, 0x4c, 0x5a, 0xb6, 0xf6, 0xa7, 0xae, 0x81, 0x75, 0x01, 0x81, 0xe6, 0x4b, + 0x57, 0x7c, 0xdd, 0x6d, 0xf8, 0x1c, 0x29, 0x32, 0xf7, 0xda, 0x3c, 0x2d, 0xf8, 0x9b, 0x25, 0x6e, 0x00, 0xb4, 0xf7, + 0x2f, 0xf7, 0x04, 0xf7, 0xa1, 0x56, 0xac, 0x4f, 0x1a, 0x64, 0xb8, 0x47, 0x55, 0x18, 0x7b, 0x07, 0x4d, 0xbd, 0x47, + 0x24, 0x80, 0x5d, 0xa2, 0x70, 0xc5, 0xdd, 0x8e, 0x82, 0xd4, 0xeb, 0xec, 0xb2, 0x0c, 0x39, 0xd2, 0x97, 0xc1, 0xcb, + 0xeb, 0xf4, 0x77, 0x59, 0xb4, 0x87, 0xef, 0xcb, 0x43, 0x2d, 0x46, 0x54, 0xd1, 0xa7, 0xd7, 0x15, 0x99, 0x0a, 0x43, + 0xa1, 0xe0, 0x99, 0x33, 0x71, 0xc1, 0xed, 0xfe, 0x72, 0x46, 0x33, 0x8e, 0x91, 0x08, 0x9f, 0xc8, 0x2e, 0xca, 0xfa, + 0xdc, 0x59, 0xd5, 0xc3, 0x76, 0x84, 0x9f, 0xa3, 0x37, 0x68, 0xc3, 0xf0, 0x47, 0x2c, 0x68, 0xdb, 0x5e, 0xc3, 0x49, + 0x4c, 0xe8, 0x92, 0x85, 0xe2, 0x23, 0xd3, 0x3f, 0xad, 0x32, 0xe5, 0x2b, 0x82, 0xd7, 0x8f, 0x99, 0x0a, 0x59, 0x5c, + 0x45, 0xd9, 0xb4, 0x51, 0x52, 0xc2, 0xae, 0xbf, 0x80, 0xcf, 0xc9, 0xc9, 0x51, 0x24, 0x2a, 0x3b, 0x3a, 0x4d, 0xae, + 0xeb, 0xbd, 0x22, 0xc3, 0x0e, 0x0f, 0x59, 0x25, 0x92, 0x17, 0xe9, 0x74, 0xc7, 0x8b, 0x70, 0x70, 0x36, 0x55, 0x95, + 0x75, 0x4b, 0xad, 0x61, 0x2b, 0x09, 0xbc, 0x82, 0xf2, 0x6e, 0x94, 0x43, 0xae, 0xc3, 0xd5, 0xcd, 0x8e, 0xfe, 0x5b, + 0x9a, 0x88, 0x43, 0x01, 0x75, 0xb2, 0x23, 0x09, 0xf7, 0x89, 0x83, 0xe7, 0xfa, 0xf9, 0xb4, 0x9b, 0xf8, 0xef, 0xbd, + 0x1c, 0x92, 0xc1, 0xda, 0x7e, 0xfe, 0x05, 0xba, 0x5a, 0xcd, 0x07, 0x6a, 0x78, 0x9e, 0x5d, 0xfb, 0x11, 0x2f, 0x79, + 0x38, 0xb6, 0xc2, 0x5b, 0x6b, 0x51, 0xb4, 0x71, 0xdd, 0xf7, 0x2a, 0xe4, 0xf4, 0x72, 0x76, 0xad, 0xc2, 0xdd, 0x64, + 0x5d, 0x79, 0xb6, 0xf5, 0x7a, 0x77, 0x20, 0x05, 0x3d, 0x30, 0x06, 0xd4, 0x4c, 0x0a, 0x2c, 0x98, 0x5a, 0xb9, 0xd4, + 0x98, 0xa9, 0x3f, 0xc6, 0x12, 0xea, 0x3b, 0x4b, 0xc5, 0x79, 0x64, 0x63, 0x6b, 0x09, 0x54, 0x3b, 0x14, 0x27, 0xba, + 0x99, 0x80, 0xc8, 0x72, 0xa8, 0x12, 0x90, 0x29, 0xba, 0x40, 0x54, 0x97, 0x2b, 0x7b, 0xfe, 0xeb, 0xcd, 0x01, 0x05, + 0x44, 0x72, 0xdb, 0x99, 0xe4, 0x61, 0xc9, 0x69, 0xd6, 0xb9, 0x28, 0xd1, 0x05, 0x3e, 0xf9, 0x0b, 0x49, 0x0a, 0x49, + 0xe9, 0x8d, 0x0e, 0xa7, 0x4a, 0x0f, 0xaf, 0x32, 0xd0, 0xe0, 0xb2, 0x3a, 0x55, 0x58, 0xfe, 0x5c, 0x28, 0x70, 0x51, + 0x23, 0xb0, 0x7b, 0x6a, 0x5f, 0x1e, 0xb8, 0x17, 0xd7, 0x94, 0x15, 0x8f, 0xee, 0x20, 0xc7, 0x42, 0x25, 0x3e, 0x9a, + 0x14, 0xd7, 0x60, 0x72, 0x39, 0x47, 0x48, 0xa9, 0xfe, 0xdd, 0x47, 0x0a, 0xb1, 0xe6, 0x60, 0x28, 0x8c, 0x11, 0x68, + 0xe1, 0xff, 0xd7, 0xce, 0xc8, 0xbe, 0xb3, 0xfe, 0x27, 0x30, 0x09, 0x70, 0xd7, 0xfa, 0x02, 0x33, 0x3a, 0x61, 0x2e, + 0xc7, 0xff, 0xa4, 0x2a, 0xa8, 0x6e, 0xb4, 0x79, 0x35, 0x6d, 0x4c, 0x1e, 0x38, 0xf8, 0xee, 0xd4, 0x84, 0x4e, 0x6e, + 0x28, 0xa7, 0xce, 0xc8, 0xc1, 0xcf, 0x80, 0x05, 0xf3, 0x04, 0xef, 0xc8, 0x18, 0x28, 0x2e, 0x8d, 0x5e, 0x0c, 0xdf, + 0xb8, 0x5f, 0x96, 0xe8, 0xc6, 0x9c, 0x2f, 0xe5, 0xa6, 0x44, 0xd7, 0xe7, 0x99, 0x44, 0x0c, 0xec, 0xd7, 0x05, 0x60, + 0x97, 0xbb, 0x74, 0x77, 0x58, 0xd5, 0xbb, 0x48, 0xde, 0x5a, 0xb2, 0x54, 0x7f, 0x0e, 0x46, 0x70, 0x6a, 0x6f, 0x78, + 0xa5, 0x08, 0x89, 0x05, 0x4e, 0x7e, 0xa0, 0x69, 0xb4, 0x40, 0x60, 0x55, 0x77, 0x75, 0x9b, 0x19, 0xf2, 0xd5, 0x13, + 0x80, 0x77, 0xf9, 0x4b, 0x3f, 0x1e, 0xee, 0xe6, 0x76, 0x84, 0x7b, 0x8c, 0xe5, 0x27, 0xa8, 0x0a, 0x91, 0x01, 0x68, + 0x71, 0x8a, 0x3f, 0x06, 0xab, 0xf6, 0xa9, 0xa5, 0xe6, 0x72, 0x92, 0xe4, 0x67, 0xe2, 0xa2, 0x46, 0x35, 0x84, 0x55, + 0x7d, 0xca, 0xa8, 0x85, 0xd0, 0xf1, 0x3f, 0xbe, 0xd7, 0x34, 0x64, 0xfc, 0xae, 0xe3, 0xe4, 0x04, 0x9f, 0x66, 0x02, + 0xb9, 0x88, 0x10, 0xd9, 0xc4, 0x4c, 0x31, 0x43, 0x7a, 0x93, 0xe2, 0x9b, 0x56, 0x43, 0x84, 0xdc, 0xdc, 0xde, 0x1d, + 0xa4, 0x02, 0x0e, 0xc2, 0xef, 0xc3, 0xf8, 0x78, 0xd1, 0xb2, 0x6b, 0x63, 0x18, 0xc9, 0xa9, 0xe5, 0x72, 0xd8, 0xf3, + 0xb9, 0xd1, 0x8a, 0xc7, 0x1a, 0x02, 0x27, 0x20, 0x77, 0x10, 0xe5, 0xc8, 0xd4, 0x4a, 0x47, 0xe5, 0xdf, 0x5f, 0x01, + 0xaa, 0xb0, 0xd4, 0x10, 0xbb, 0x69, 0xe3, 0x36, 0xc8, 0xe1, 0x3d, 0x43, 0xfb, 0x86, 0xcd, 0xcc, 0xbf, 0xf4, 0x88, + 0xe0, 0x20, 0xca, 0xb7, 0x1b, 0xf1, 0x2f, 0x5c, 0xee, 0xd4, 0xd3, 0xa3, 0xcc, 0xa4, 0x1e, 0x1c, 0x47, 0xfb, 0xbf, + 0xfc, 0xa2, 0x41, 0x55, 0x9d, 0xf6, 0x5a, 0x5e, 0x65, 0x32, 0x34, 0x7b, 0x52, 0x8d, 0xd5, 0xd0, 0x20, 0x60, 0x03, + 0xab, 0x3f, 0x8c, 0xd4, 0x21, 0xea, 0x2a, 0xd9, 0xc4, 0xd0, 0xd3, 0x65, 0xd8, 0x7a, 0x13, 0x28, 0x62, 0x32, 0x4b, + 0x2c, 0x87, 0x93, 0xa8, 0xb4, 0x52, 0x45, 0x09, 0x44, 0xec, 0xec, 0xc3, 0x17, 0xdb, 0x9a, 0x4d, 0x5c, 0xa9, 0x11, + 0xd4, 0x7d, 0xaf, 0x9e, 0xf1, 0x2d, 0xb2, 0x66, 0xc5, 0x1d, 0xed, 0xb7, 0xcd, 0x0b, 0x25, 0x5e, 0x30, 0x47, 0x3f, + 0x40, 0xf4, 0xa1, 0xa0, 0x00, 0x94, 0x10, 0xc5, 0x6a, 0x63, 0x1a, 0xd5, 0x88, 0x92, 0x8e, 0x82, 0x39, 0x87, 0x3c, + 0x78, 0x65, 0x58, 0x42, 0x75, 0x5b, 0xdd, 0x77, 0x3e, 0x09, 0x4e, 0x76, 0x5b, 0xe6, 0x0e, 0x4d, 0x38, 0xb2, 0xc0, + 0xb8, 0x95, 0x01, 0x7a, 0x10, 0xe0, 0xfb, 0x07, 0xf2, 0xab, 0x2d, 0x8c, 0x32, 0xed, 0x2b, 0xc0, 0x46, 0xc2, 0xf5, + 0x38, 0x83, 0xf0, 0x17, 0xec, 0xc1, 0x20, 0x6a, 0x9a, 0x0b, 0x00, 0xa0, 0x98, 0x22, 0x50, 0x23, 0xd5, 0x80, 0x6b, + 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9, 0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0 +}; +static CONST UINT8 DecOutput010[] = { + 0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf, 0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c, 0xef, 0x0a, 0xa9, + 0x48, 0x5f, 0x5f, 0x37, 0x22, 0x3a, 0xde, 0xe3, 0xae, 0xbe, 0xad, 0x07, 0xcc, 0xb1, 0xf6, 0xf5, 0xf9, 0x56, 0xdd, + 0xe7, 0x16, 0x1e, 0x7f, 0xdf, 0x7a, 0x9e, 0x75, 0xb7, 0xc7, 0xbe, 0xbe, 0x8a, 0x36, 0x04, 0xc0, 0x10, 0xf4, 0x95, + 0x20, 0x03, 0xec, 0xdc, 0x05, 0xa1, 0x7d, 0xc4, 0xa9, 0x2c, 0x82, 0xd0, 0xbc, 0x8b, 0xc5, 0xc7, 0x45, 0x50, 0xf6, + 0xa2, 0x1a, 0xb5, 0x46, 0x3b, 0x73, 0x02, 0xa6, 0x83, 0x4b, 0x73, 0x82, 0x58, 0x5e, 0x3b, 0x65, 0x2f, 0x0e, 0xfd, + 0x2b, 0x59, 0x16, 0xce, 0xa1, 0x60, 0x9c, 0xe8, 0x3a, 0x99, 0xed, 0x8d, 0x5a, 0xcf, 0xf6, 0x83, 0xaf, 0xba, 0xd7, + 0x73, 0x73, 0x40, 0x97, 0x3d, 0xca, 0xef, 0x07, 0x57, 0xe6, 0xd9, 0x70, 0x0e, 0x95, 0xae, 0xa6, 0x8d, 0x04, 0xcc, + 0xee, 0xf7, 0x09, 0x31, 0x77, 0x12, 0xa3, 0x23, 0x97, 0x62, 0xb3, 0x7b, 0x32, 0xfb, 0x80, 0x14, 0x48, 0x81, 0xc3, + 0xe5, 0xea, 0x91, 0x39, 0x52, 0x81, 0xa2, 0x4f, 0xe4, 0xb3, 0x09, 0xff, 0xde, 0x5e, 0xe9, 0x58, 0x84, 0x6e, 0xf9, + 0x3d, 0xdf, 0x25, 0xea, 0xad, 0xae, 0xe6, 0x9a, 0xd1, 0x89, 0x55, 0xd3, 0xde, 0x6c, 0x52, 0xdb, 0x70, 0xfe, 0x37, + 0xce, 0x44, 0x0a, 0xa8, 0x25, 0x5f, 0x92, 0xc1, 0x33, 0x4a, 0x4f, 0x9b, 0x62, 0x35, 0xff, 0xce, 0xc0, 0xa9, 0x60, + 0xce, 0x52, 0x00, 0x97, 0x51, 0x35, 0x26, 0x2e, 0xb9, 0x36, 0xa9, 0x87, 0x6e, 0x1e, 0xcc, 0x91, 0x78, 0x53, 0x98, + 0x86, 0x5b, 0x9c, 0x74, 0x7d, 0x88, 0x33, 0xe1, 0xdf, 0x37, 0x69, 0x2b, 0xbb, 0xf1, 0x4d, 0xf4, 0xd1, 0xf1, 0x39, + 0x93, 0x17, 0x51, 0x19, 0xe3, 0x19, 0x1e, 0x76, 0x37, 0x25, 0xfb, 0x09, 0x27, 0x6a, 0xab, 0x67, 0x6f, 0x14, 0x12, + 0x64, 0xe7, 0xc4, 0x07, 0xdf, 0x4d, 0x17, 0xbb, 0x6d, 0xe0, 0xe9, 0xb9, 0xab, 0xca, 0x10, 0x68, 0xaf, 0x7e, 0xb7, + 0x33, 0x54, 0x73, 0x07, 0x6e, 0xf7, 0x81, 0x97, 0x9c, 0x05, 0x6f, 0x84, 0x5f, 0xd2, 0x42, 0xfb, 0x38, 0xcf, 0xd1, + 0x2f, 0x14, 0x30, 0x88, 0x98, 0x4d, 0x5a, 0xa9, 0x76, 0xd5, 0x4f, 0x3e, 0x70, 0x6c, 0x85, 0x76, 0xd7, 0x01, 0xa0, + 0x1a, 0xc8, 0x4e, 0xaa, 0xac, 0x78, 0xfe, 0x46, 0xde, 0x6a, 0x05, 0x46, 0xa7, 0x43, 0x0c, 0xb9, 0xde, 0xb9, 0x68, + 0xfb, 0xce, 0x42, 0x99, 0x07, 0x4d, 0x0b, 0x3b, 0x5a, 0x30, 0x35, 0xa8, 0xf9, 0x3a, 0x73, 0xef, 0x0f, 0xdb, 0x1e, + 0x16, 0x42, 0xc4, 0xba, 0xae, 0x58, 0xaa, 0xf8, 0xe5, 0x75, 0x2f, 0x1b, 0x15, 0x5c, 0xfd, 0x0a, 0x97, 0xd0, 0xe4, + 0x37, 0x83, 0x61, 0x5f, 0x43, 0xa6, 0xc7, 0x3f, 0x38, 0x59, 0xe6, 0xeb, 0xa3, 0x90, 0xc3, 0xaa, 0xaa, 0x5a, 0xd3, + 0x34, 0xd4, 0x17, 0xc8, 0x65, 0x3e, 0x57, 0xbc, 0x5e, 0xdd, 0x9e, 0xb7, 0xf0, 0x2e, 0x5b, 0xb2, 0x1f, 0x8a, 0x08, + 0x0d, 0x45, 0x91, 0x0b, 0x29, 0x53, 0x4f, 0x4c, 0x5a, 0x73, 0x56, 0xfe, 0xaf, 0x41, 0x01, 0x39, 0x0a, 0x24, 0x3c, + 0x7e, 0xbe, 0x4e, 0x53, 0xf3, 0xeb, 0x06, 0x66, 0x51, 0x28, 0x1d, 0xbd, 0x41, 0x0a, 0x01, 0xab, 0x16, 0x47, 0x27, + 0x47, 0x47, 0xf7, 0xcb, 0x46, 0x0a, 0x70, 0x9e, 0x01, 0x9c, 0x09, 0xe1, 0x2a, 0x00, 0x1a, 0xd8, 0xd4, 0x79, 0x9d, + 0x80, 0x15, 0x8e, 0x53, 0x2a, 0x65, 0x83, 0x78, 0x3e, 0x03, 0x00, 0x07, 0x12, 0x1f, 0x33, 0x3e, 0x7b, 0x13, 0x37, + 0xf1, 0xc3, 0xef, 0xb7, 0xc1, 0x20, 0x3c, 0x3e, 0x67, 0x66, 0x5d, 0x88, 0xa7, 0x7d, 0x33, 0x50, 0x77, 0xb0, 0x28, + 0x8e, 0xe7, 0x2c, 0x2e, 0x7a, 0xf4, 0x3c, 0x8d, 0x74, 0x83, 0xaf, 0x8e, 0x87, 0x0f, 0xe4, 0x50, 0xff, 0x84, 0x5c, + 0x47, 0x0c, 0x6a, 0x49, 0xbf, 0x42, 0x86, 0x77, 0x15, 0x48, 0xa5, 0x90, 0x5d, 0x93, 0xd6, 0x2a, 0x11, 0xd5, 0xd5, + 0x11, 0xaa, 0xce, 0xe7, 0x6f, 0xa5, 0xb0, 0x09, 0x2c, 0x8d, 0xd3, 0x92, 0xf0, 0x5a, 0x2a, 0xda, 0x5b, 0x1e, 0xd5, + 0x9a, 0xc4, 0xc4, 0xf3, 0x49, 0x74, 0x41, 0xca, 0xe8, 0xc1, 0xf8, 0x44, 0xd6, 0x3c, 0xae, 0x6c, 0x1d, 0x9a, 0x30, + 0x04, 0x4d, 0x27, 0x0e, 0xb1, 0x5f, 0x59, 0xa2, 0x24, 0xe8, 0xe1, 0x98, 0xc5, 0x6a, 0x4c, 0xfe, 0x41, 0xd2, 0x27, + 0x42, 0x52, 0xe1, 0xe9, 0x7d, 0x62, 0xe4, 0x88, 0x0f, 0xad, 0xb2, 0x70, 0xcb, 0x9d, 0x4c, 0x27, 0x2e, 0x76, 0x1e, + 0x1a, 0x63, 0x65, 0xf5, 0x3b, 0xf8, 0x57, 0x69, 0xeb, 0x5b, 0x38, 0x26, 0x39, 0x33, 0x25, 0x45, 0x3e, 0x91, 0xb8, + 0xd8, 0xc7, 0xd5, 0x42, 0xc0, 0x22, 0x31, 0x74, 0xf4, 0xbc, 0x0c, 0x23, 0xf1, 0xca, 0xc1, 0x8d, 0xd7, 0xbe, 0xc9, + 0x62, 0xe4, 0x08, 0x1a, 0xcf, 0x36, 0xd5, 0xfe, 0x55, 0x21, 0x59, 0x91, 0x87, 0x87, 0xdf, 0x06, 0xdb, 0xdf, 0x96, + 0x45, 0x58, 0xda, 0x05, 0xcd, 0x50, 0x4d, 0xd2, 0x7d, 0x05, 0x18, 0x73, 0x6a, 0x8d, 0x11, 0x85, 0xa6, 0x88, 0xe8, + 0xda, 0xe6, 0x30, 0x33, 0xa4, 0x89, 0x31, 0x75, 0xbe, 0x69, 0x43, 0x84, 0x43, 0x50, 0x87, 0xdd, 0x71, 0x36, 0x83, + 0xc3, 0x78, 0x74, 0x24, 0x0a, 0xed, 0x7b, 0xdb, 0xa4, 0x24, 0x0b, 0xb9, 0x7e, 0x5d, 0xff, 0xde, 0xb1, 0xef, 0x61, + 0x5a, 0x45, 0x33, 0xf6, 0x17, 0x07, 0x08, 0x98, 0x83, 0x92, 0x0f, 0x23, 0x6d, 0xe6, 0xaa, 0x17, 0x54, 0xad, 0x6a, + 0xc8, 0xdb, 0x26, 0xbe, 0xb8, 0xb6, 0x08, 0xfa, 0x68, 0xf1, 0xd7, 0x79, 0x6f, 0x18, 0xb4, 0x9e, 0x2d, 0x3f, 0x1b, + 0x64, 0xaf, 0x8d, 0x06, 0x0e, 0x49, 0x28, 0xe0, 0x5d, 0x45, 0x68, 0x13, 0x87, 0xfa, 0xde, 0x40, 0x7b, 0xd2, 0xc3, + 0x94, 0xd5, 0xe1, 0xd9, 0xc2, 0xaf, 0x55, 0x89, 0xeb, 0xb4, 0x12, 0x59, 0xa8, 0xd4, 0xc5, 0x29, 0x66, 0x38, 0xe6, + 0xac, 0x22, 0x22, 0xd9, 0x64, 0x9b, 0x34, 0x0a, 0x32, 0x9f, 0xc2, 0xbf, 0x17, 0x6c, 0x3f, 0x71, 0x7a, 0x38, 0x6b, + 0x98, 0xfb, 0x49, 0x36, 0x89, 0xc9, 0xe2, 0xd6, 0xc7, 0x5d, 0xd0, 0x69, 0x5f, 0x23, 0x35, 0xc9, 0x30, 0xe2, 0xfd, + 0x44, 0x58, 0x39, 0xd7, 0x97, 0xfb, 0x5c, 0x00, 0xd5, 0x4f, 0x7a, 0x1a, 0x95, 0x8b, 0x62, 0x4b, 0xce, 0xe5, 0x91, + 0x21, 0x7b, 0x30, 0x00, 0xd6, 0xdd, 0x6d, 0x02, 0x86, 0x49, 0x0f, 0x3c, 0x1a, 0x27, 0x3c, 0xd3, 0x0e, 0x71, 0xf2, + 0xff, 0xf5, 0x2f, 0x87, 0xac, 0x67, 0x59, 0x81, 0xa3, 0xf7, 0xf8, 0xd6, 0x11, 0x0c, 0x84, 0xa9, 0x03, 0xee, 0x2a, + 0xc4, 0xf3, 0x22, 0xab, 0x7c, 0xe2, 0x25, 0xf5, 0x67, 0xa3, 0xe4, 0x11, 0xe0, 0x59, 0xb3, 0xca, 0x87, 0xa0, 0xae, + 0xc9, 0xa6, 0x62, 0x1b, 0x6e, 0x4d, 0x02, 0x6b, 0x07, 0x9d, 0xfd, 0xd0, 0x92, 0x06, 0xe1, 0xb2, 0x9a, 0x4a, 0x1f, + 0x1f, 0x13, 0x49, 0x99, 0x97, 0x08, 0xde, 0x7f, 0x98, 0xaf, 0x51, 0x98, 0xee, 0x2c, 0xcb, 0xf0, 0x0b, 0xc6, 0xb6, + 0xb7, 0x2d, 0x9a, 0xb1, 0xac, 0xa6, 0xe3, 0x15, 0x77, 0x9d, 0x6b, 0x1a, 0xe4, 0xfc, 0x8b, 0xf2, 0x17, 0x59, 0x08, + 0x04, 0x58, 0x81, 0x9d, 0x1b, 0x1b, 0x69, 0x55, 0xc2, 0xb4, 0x3c, 0x1f, 0x50, 0xf1, 0x7f, 0x77, 0x90, 0x4c, 0x66, + 0x40, 0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c, 0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f +}; +static CONST UINT8 DecAssoc010[] = { 0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27, + 0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2 }; +static CONST UINT8 DecNonce010[] = { 0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30 }; +static CONST UINT8 DecKey010[] = { 0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44, 0x34, 0xda, 0x7f, + 0x57, 0x03, 0x39, 0x0c, 0xaf, 0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, + 0x8e, 0x74, 0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7 }; + +static CONST UINT8 DecInput011[] = { + 0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8, 0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc, 0x3c, 0xba, 0x6a, + 0x77, 0x47, 0xdb, 0xe3, 0x74, 0x7a, 0x4d, 0x5f, 0x8d, 0x37, 0x55, 0x80, 0x73, 0x90, 0x66, 0x5d, 0x3a, 0x7d, 0x5d, + 0x86, 0x5e, 0x8d, 0xfd, 0x83, 0xff, 0x4e, 0x74, 0x6f, 0xf9, 0xe6, 0x70, 0x17, 0x70, 0x3e, 0x96, 0xa7, 0x7e, 0xcb, + 0xab, 0x8f, 0x58, 0x24, 0x9b, 0x01, 0xfd, 0xcb, 0xe6, 0x4d, 0x9b, 0xf0, 0x88, 0x94, 0x57, 0x66, 0xef, 0x72, 0x4c, + 0x42, 0x6e, 0x16, 0x19, 0x15, 0xea, 0x70, 0x5b, 0xac, 0x13, 0xdb, 0x9f, 0x18, 0xe2, 0x3c, 0x26, 0x97, 0xbc, 0xdc, + 0x45, 0x8c, 0x6c, 0x24, 0x69, 0x9c, 0xf7, 0x65, 0x1e, 0x18, 0x59, 0x31, 0x7c, 0xe4, 0x73, 0xbc, 0x39, 0x62, 0xc6, + 0x5c, 0x9f, 0xbf, 0xfa, 0x90, 0x03, 0xc9, 0x72, 0x26, 0xb6, 0x1b, 0xc2, 0xb7, 0x3f, 0xf2, 0x13, 0x77, 0xf2, 0x8d, + 0xb9, 0x47, 0xd0, 0x53, 0xdd, 0xc8, 0x91, 0x83, 0x8b, 0xb1, 0xce, 0xa3, 0xfe, 0xcd, 0xd9, 0xdd, 0x92, 0x7b, 0xdb, + 0xb8, 0xfb, 0xc9, 0x2d, 0x01, 0x59, 0x39, 0x52, 0xad, 0x1b, 0xec, 0xcf, 0xd7, 0x70, 0x13, 0x21, 0xf5, 0x47, 0xaa, + 0x18, 0x21, 0x5c, 0xc9, 0x9a, 0xd2, 0x6b, 0x05, 0x9c, 0x01, 0xa1, 0xda, 0x35, 0x5d, 0xb3, 0x70, 0xe6, 0xa9, 0x80, + 0x8b, 0x91, 0xb7, 0xb3, 0x5f, 0x24, 0x9a, 0xb7, 0xd1, 0x6b, 0xa1, 0x1c, 0x50, 0xba, 0x49, 0xe0, 0xee, 0x2e, 0x75, + 0xac, 0x69, 0xc0, 0xeb, 0x03, 0xdd, 0x19, 0xe5, 0xf6, 0x06, 0xdd, 0xc3, 0xd7, 0x2b, 0x07, 0x07, 0x30, 0xa7, 0x19, + 0x0c, 0xbf, 0xe6, 0x18, 0xcc, 0xb1, 0x01, 0x11, 0x85, 0x77, 0x1d, 0x96, 0xa7, 0xa3, 0x00, 0x84, 0x02, 0xa2, 0x83, + 0x68, 0xda, 0x17, 0x27, 0xc8, 0x7f, 0x23, 0xb7, 0xf4, 0x13, 0x85, 0xcf, 0xdd, 0x7a, 0x7d, 0x24, 0x57, 0xfe, 0x05, + 0x93, 0xf5, 0x74, 0xce, 0xed, 0x0c, 0x20, 0x98, 0x8d, 0x92, 0x30, 0xa1, 0x29, 0x23, 0x1a, 0xa0, 0x4f, 0x69, 0x56, + 0x4c, 0xe1, 0xc8, 0xce, 0xf6, 0x9a, 0x0c, 0xa4, 0xfa, 0x04, 0xf6, 0x62, 0x95, 0xf2, 0xfa, 0xc7, 0x40, 0x68, 0x40, + 0x8f, 0x41, 0xda, 0xb4, 0x26, 0x6f, 0x70, 0xab, 0x40, 0x61, 0xa4, 0x0e, 0x75, 0xfb, 0x86, 0xeb, 0x9d, 0x9a, 0x1f, + 0xec, 0x76, 0x99, 0xe7, 0xea, 0xaa, 0x1e, 0x2d, 0xb5, 0xd4, 0xa6, 0x1a, 0xb8, 0x61, 0x0a, 0x1d, 0x16, 0x5b, 0x98, + 0xc2, 0x31, 0x40, 0xe7, 0x23, 0x1d, 0x66, 0x99, 0xc8, 0xc0, 0xd7, 0xce, 0xf3, 0x57, 0x40, 0x04, 0x3f, 0xfc, 0xea, + 0xb3, 0xfc, 0xd2, 0xd3, 0x99, 0xa4, 0x94, 0x69, 0xa0, 0xef, 0xd1, 0x85, 0xb3, 0xa6, 0xb1, 0x28, 0xbf, 0x94, 0x67, + 0x22, 0xc3, 0x36, 0x46, 0xf8, 0xd2, 0x0f, 0x5f, 0xf4, 0x59, 0x80, 0xe6, 0x2d, 0x43, 0x08, 0x7d, 0x19, 0x09, 0x97, + 0xa7, 0x4c, 0x3d, 0x8d, 0xba, 0x65, 0x62, 0xa3, 0x71, 0x33, 0x29, 0x62, 0xdb, 0xc1, 0x33, 0x34, 0x1a, 0x63, 0x33, + 0x16, 0xb6, 0x64, 0x7e, 0xab, 0x33, 0xf0, 0xe6, 0x26, 0x68, 0xba, 0x1d, 0x2e, 0x38, 0x08, 0xe6, 0x02, 0xd3, 0x25, + 0x2c, 0x47, 0x23, 0x58, 0x34, 0x0f, 0x9d, 0x63, 0x4f, 0x63, 0xbb, 0x7f, 0x3b, 0x34, 0x38, 0xa7, 0xb5, 0x8d, 0x65, + 0xd9, 0x9f, 0x79, 0x55, 0x3e, 0x4d, 0xe7, 0x73, 0xd8, 0xf6, 0x98, 0x97, 0x84, 0x60, 0x9c, 0xc8, 0xa9, 0x3c, 0xf6, + 0xdc, 0x12, 0x5c, 0xe1, 0xbb, 0x0b, 0x8b, 0x98, 0x9c, 0x9d, 0x26, 0x7c, 0x4a, 0xe6, 0x46, 0x36, 0x58, 0x21, 0x4a, + 0xee, 0xca, 0xd7, 0x3b, 0xc2, 0x6c, 0x49, 0x2f, 0xe5, 0xd5, 0x03, 0x59, 0x84, 0x53, 0xcb, 0xfe, 0x92, 0x71, 0x2e, + 0x7c, 0x21, 0xcc, 0x99, 0x85, 0x7f, 0xb8, 0x74, 0x90, 0x13, 0x42, 0x3f, 0xe0, 0x6b, 0x1d, 0xf2, 0x4d, 0x54, 0xd4, + 0xfc, 0x3a, 0x05, 0xe6, 0x74, 0xaf, 0xa6, 0xa0, 0x2a, 0x20, 0x23, 0x5d, 0x34, 0x5c, 0xd9, 0x3e, 0x4e, 0xfa, 0x93, + 0xe7, 0xaa, 0xe9, 0x6f, 0x08, 0x43, 0x67, 0x41, 0xc5, 0xad, 0xfb, 0x31, 0x95, 0x82, 0x73, 0x32, 0xd8, 0xa6, 0xa3, + 0xed, 0x0e, 0x2d, 0xf6, 0x5f, 0xfd, 0x80, 0xa6, 0x7a, 0xe0, 0xdf, 0x78, 0x15, 0x29, 0x74, 0x33, 0xd0, 0x9e, 0x83, + 0x86, 0x72, 0x22, 0x57, 0x29, 0xb9, 0x9e, 0x5d, 0xd3, 0x1a, 0xb5, 0x96, 0x72, 0x41, 0x3d, 0xf1, 0x64, 0x43, 0x67, + 0xee, 0xaa, 0x5c, 0xd3, 0x9a, 0x96, 0x13, 0x11, 0x5d, 0xf3, 0x0c, 0x87, 0x82, 0x1e, 0x41, 0x9e, 0xd0, 0x27, 0xd7, + 0x54, 0x3b, 0x67, 0x73, 0x09, 0x91, 0xe9, 0xd5, 0x36, 0xa7, 0xb5, 0x55, 0xe4, 0xf3, 0x21, 0x51, 0x49, 0x22, 0x07, + 0x55, 0x4f, 0x44, 0x4b, 0xd2, 0x15, 0x93, 0x17, 0x2a, 0xfa, 0x4d, 0x4a, 0x57, 0xdb, 0x4c, 0xa6, 0xeb, 0xec, 0x53, + 0x25, 0x6c, 0x21, 0xed, 0x00, 0x4c, 0x3b, 0xca, 0x14, 0x57, 0xa9, 0xd6, 0x6a, 0xcd, 0x8d, 0x5e, 0x74, 0xac, 0x72, + 0xc1, 0x97, 0xe5, 0x1b, 0x45, 0x4e, 0xda, 0xfc, 0xcc, 0x40, 0xe8, 0x48, 0x88, 0x0b, 0xa3, 0xe3, 0x8d, 0x83, 0x42, + 0xc3, 0x23, 0xfd, 0x68, 0xb5, 0x8e, 0xf1, 0x9d, 0x63, 0x77, 0xe9, 0xa3, 0x8e, 0x8c, 0x26, 0x6b, 0xbd, 0x72, 0x73, + 0x35, 0x0c, 0x03, 0xf8, 0x43, 0x78, 0x52, 0x71, 0x15, 0x1f, 0x71, 0x5d, 0x6e, 0xed, 0xb9, 0xcc, 0x86, 0x30, 0xdb, + 0x2b, 0xd3, 0x82, 0x88, 0x23, 0x71, 0x90, 0x53, 0x5c, 0xa9, 0x2f, 0x76, 0x01, 0xb7, 0x9a, 0xfe, 0x43, 0x55, 0xa3, + 0x04, 0x9b, 0x0e, 0xe4, 0x59, 0xdf, 0xc9, 0xe9, 0xb1, 0xea, 0x29, 0x28, 0x3c, 0x5c, 0xae, 0x72, 0x84, 0xb6, 0xc6, + 0xeb, 0x0c, 0x27, 0x07, 0x74, 0x90, 0x0d, 0x31, 0xb0, 0x00, 0x77, 0xe9, 0x40, 0x70, 0x6f, 0x68, 0xa7, 0xfd, 0x06, + 0xec, 0x4b, 0xc0, 0xb7, 0xac, 0xbc, 0x33, 0xb7, 0x6d, 0x0a, 0xbd, 0x12, 0x1b, 0x59, 0xcb, 0xdd, 0x32, 0xf5, 0x1d, + 0x94, 0x57, 0x76, 0x9e, 0x0c, 0x18, 0x98, 0x71, 0xd7, 0x2a, 0xdb, 0x0b, 0x7b, 0xa7, 0x71, 0xb7, 0x67, 0x81, 0x23, + 0x96, 0xae, 0xb9, 0x7e, 0x32, 0x43, 0x92, 0x8a, 0x19, 0xa0, 0xc4, 0xd4, 0x3b, 0x57, 0xf9, 0x4a, 0x2c, 0xfb, 0x51, + 0x46, 0xbb, 0xcb, 0x5d, 0xb3, 0xef, 0x13, 0x93, 0x6e, 0x68, 0x42, 0x54, 0x57, 0xd3, 0x6a, 0x3a, 0x8f, 0x9d, 0x66, + 0xbf, 0xbd, 0x36, 0x23, 0xf5, 0x93, 0x83, 0x7b, 0x9c, 0xc0, 0xdd, 0xc5, 0x49, 0xc0, 0x64, 0xed, 0x07, 0x12, 0xb3, + 0xe6, 0xe4, 0xe5, 0x38, 0x95, 0x23, 0xb1, 0xa0, 0x3b, 0x1a, 0x61, 0xda, 0x17, 0xac, 0xc3, 0x58, 0xdd, 0x74, 0x64, + 0x22, 0x11, 0xe8, 0x32, 0x1d, 0x16, 0x93, 0x85, 0x99, 0xa5, 0x9c, 0x34, 0x55, 0xb1, 0xe9, 0x20, 0x72, 0xc9, 0x28, + 0x7b, 0x79, 0x00, 0xa1, 0xa6, 0xa3, 0x27, 0x40, 0x18, 0x8a, 0x54, 0xe0, 0xcc, 0xe8, 0x4e, 0x8e, 0x43, 0x96, 0xe7, + 0x3f, 0xc8, 0xe9, 0xb2, 0xf9, 0xc9, 0xda, 0x04, 0x71, 0x50, 0x47, 0xe4, 0xaa, 0xce, 0xa2, 0x30, 0xc8, 0xe4, 0xac, + 0xc7, 0x0d, 0x06, 0x2e, 0xe6, 0xe8, 0x80, 0x36, 0x29, 0x9e, 0x01, 0xb8, 0xc3, 0xf0, 0xa0, 0x5d, 0x7a, 0xca, 0x4d, + 0xa0, 0x57, 0xbd, 0x2a, 0x45, 0xa7, 0x7f, 0x9c, 0x93, 0x07, 0x8f, 0x35, 0x67, 0x92, 0xe3, 0xe9, 0x7f, 0xa8, 0x61, + 0x43, 0x9e, 0x25, 0x4f, 0x33, 0x76, 0x13, 0x6e, 0x12, 0xb9, 0xdd, 0xa4, 0x7c, 0x08, 0x9f, 0x7c, 0xe7, 0x0a, 0x8d, + 0x84, 0x06, 0xa4, 0x33, 0x17, 0x34, 0x5e, 0x10, 0x7c, 0xc0, 0xa8, 0x3d, 0x1f, 0x42, 0x20, 0x51, 0x65, 0x5d, 0x09, + 0xc3, 0xaa, 0xc0, 0xc8, 0x0d, 0xf0, 0x79, 0xbc, 0x20, 0x1b, 0x95, 0xe7, 0x06, 0x7d, 0x47, 0x20, 0x03, 0x1a, 0x74, + 0xdd, 0xe2, 0xd4, 0xae, 0x38, 0x71, 0x9b, 0xf5, 0x80, 0xec, 0x08, 0x4e, 0x56, 0xba, 0x76, 0x12, 0x1a, 0xdf, 0x48, + 0xf3, 0xae, 0xb3, 0xe6, 0xe6, 0xbe, 0xc0, 0x91, 0x2e, 0x01, 0xb3, 0x01, 0x86, 0xa2, 0xb9, 0x52, 0xd1, 0x21, 0xae, + 0xd4, 0x97, 0x1d, 0xef, 0x41, 0x12, 0x95, 0x3d, 0x48, 0x45, 0x1c, 0x56, 0x32, 0x8f, 0xb8, 0x43, 0xbb, 0x19, 0xf3, + 0xca, 0xe9, 0xeb, 0x6d, 0x84, 0xbe, 0x86, 0x06, 0xe2, 0x36, 0xb2, 0x62, 0x9d, 0xd3, 0x4c, 0x48, 0x18, 0x54, 0x13, + 0x4e, 0xcf, 0xfd, 0xba, 0x84, 0xb9, 0x30, 0x53, 0xcf, 0xfb, 0xb9, 0x29, 0x8f, 0xdc, 0x9f, 0xef, 0x60, 0x0b, 0x64, + 0xf6, 0x8b, 0xee, 0xa6, 0x91, 0xc2, 0x41, 0x6c, 0xf6, 0xfa, 0x79, 0x67, 0x4b, 0xc1, 0x3f, 0xaf, 0x09, 0x81, 0xd4, + 0x5d, 0xcb, 0x09, 0xdf, 0x36, 0x31, 0xc0, 0x14, 0x3c, 0x7c, 0x0e, 0x65, 0x95, 0x99, 0x6d, 0xa3, 0xf4, 0xd7, 0x38, + 0xee, 0x1a, 0x2b, 0x37, 0xe2, 0xa4, 0x3b, 0x4b, 0xd0, 0x65, 0xca, 0xf8, 0xc3, 0xe8, 0x15, 0x20, 0xef, 0xf2, 0x00, + 0xfd, 0x01, 0x09, 0xc5, 0xc8, 0x17, 0x04, 0x93, 0xd0, 0x93, 0x03, 0x55, 0xc5, 0xfe, 0x32, 0xa3, 0x3e, 0x28, 0x2d, + 0x3b, 0x93, 0x8a, 0xcc, 0x07, 0x72, 0x80, 0x8b, 0x74, 0x16, 0x24, 0xbb, 0xda, 0x94, 0x39, 0x30, 0x8f, 0xb1, 0xcd, + 0x4a, 0x90, 0x92, 0x7c, 0x14, 0x8f, 0x95, 0x4e, 0xac, 0x9b, 0xd8, 0x8f, 0x1a, 0x87, 0xa4, 0x32, 0x27, 0x8a, 0xba, + 0xf7, 0x41, 0xcf, 0x84, 0x37, 0x19, 0xe6, 0x06, 0xf5, 0x0e, 0xcf, 0x36, 0xf5, 0x9e, 0x6c, 0xde, 0xbc, 0xff, 0x64, + 0x7e, 0x4e, 0x59, 0x57, 0x48, 0xfe, 0x14, 0xf7, 0x9c, 0x93, 0x5d, 0x15, 0xad, 0xcc, 0x11, 0xb1, 0x17, 0x18, 0xb2, + 0x7e, 0xcc, 0xab, 0xe9, 0xce, 0x7d, 0x77, 0x5b, 0x51, 0x1b, 0x1e, 0x20, 0xa8, 0x32, 0x06, 0x0e, 0x75, 0x93, 0xac, + 0xdb, 0x35, 0x37, 0x1f, 0xe9, 0x19, 0x1d, 0xb4, 0x71, 0x97, 0xd6, 0x4e, 0x2c, 0x08, 0xa5, 0x13, 0xf9, 0x0e, 0x7e, + 0x78, 0x6e, 0x14, 0xe0, 0xa9, 0xb9, 0x96, 0x4c, 0x80, 0x82, 0xba, 0x17, 0xb3, 0x9d, 0x69, 0xb0, 0x84, 0x46, 0xff, + 0xf9, 0x52, 0x79, 0x94, 0x58, 0x3a, 0x62, 0x90, 0x15, 0x35, 0x71, 0x10, 0x37, 0xed, 0xa1, 0x8e, 0x53, 0x6e, 0xf4, + 0x26, 0x57, 0x93, 0x15, 0x93, 0xf6, 0x81, 0x2c, 0x5a, 0x10, 0xda, 0x92, 0xad, 0x2f, 0xdb, 0x28, 0x31, 0x2d, 0x55, + 0x04, 0xd2, 0x06, 0x28, 0x8c, 0x1e, 0xdc, 0xea, 0x54, 0xac, 0xff, 0xb7, 0x6c, 0x30, 0x15, 0xd4, 0xb4, 0x0d, 0x00, + 0x93, 0x57, 0xdd, 0xd2, 0x07, 0x07, 0x06, 0xd9, 0x43, 0x9b, 0xcd, 0x3a, 0xf4, 0x7d, 0x4c, 0x36, 0x5d, 0x23, 0xa2, + 0xcc, 0x57, 0x40, 0x91, 0xe9, 0x2c, 0x2f, 0x2c, 0xd5, 0x30, 0x9b, 0x17, 0xb0, 0xc9, 0xf7, 0xa7, 0x2f, 0xd1, 0x93, + 0x20, 0x6b, 0xc6, 0xc1, 0xe4, 0x6f, 0xcb, 0xd1, 0xe7, 0x09, 0x0f, 0x9e, 0xdc, 0xaa, 0x9f, 0x2f, 0xdf, 0x56, 0x9f, + 0xd4, 0x33, 0x04, 0xaf, 0xd3, 0x6c, 0x58, 0x61, 0xf0, 0x30, 0xec, 0xf2, 0x7f, 0xf2, 0x9c, 0xdf, 0x39, 0xbb, 0x6f, + 0xa2, 0x8c, 0x7e, 0xc4, 0x22, 0x51, 0x71, 0xc0, 0x4d, 0x14, 0x1a, 0xc4, 0xcd, 0x04, 0xd9, 0x87, 0x08, 0x50, 0x05, + 0xcc, 0xaf, 0xf6, 0xf0, 0x8f, 0x92, 0x54, 0x58, 0xc2, 0xc7, 0x09, 0x7a, 0x59, 0x02, 0x05, 0xe8, 0xb0, 0x86, 0xd9, + 0xbf, 0x7b, 0x35, 0x51, 0x4d, 0xaf, 0x08, 0x97, 0x2c, 0x65, 0xda, 0x2a, 0x71, 0x3a, 0xa8, 0x51, 0xcc, 0xf2, 0x73, + 0x27, 0xc3, 0xfd, 0x62, 0xcf, 0xe3, 0xb2, 0xca, 0xcb, 0xbe, 0x1a, 0x0a, 0xa1, 0x34, 0x7b, 0x77, 0xc4, 0x62, 0x68, + 0x78, 0x5f, 0x94, 0x07, 0x04, 0x65, 0x16, 0x4b, 0x61, 0xcb, 0xff, 0x75, 0x26, 0x50, 0x66, 0x1f, 0x6e, 0x93, 0xf8, + 0xc5, 0x51, 0xeb, 0xa4, 0x4a, 0x48, 0x68, 0x6b, 0xe2, 0x5e, 0x44, 0xb2, 0x50, 0x2c, 0x6c, 0xae, 0x79, 0x4e, 0x66, + 0x35, 0x81, 0x50, 0xac, 0xbc, 0x3f, 0xb1, 0x0c, 0xf3, 0x05, 0x3c, 0x4a, 0xa3, 0x6c, 0x2a, 0x79, 0xb4, 0xb7, 0xab, + 0xca, 0xc7, 0x9b, 0x8e, 0xcd, 0x5f, 0x11, 0x03, 0xcb, 0x30, 0xa3, 0xab, 0xda, 0xfe, 0x64, 0xb9, 0xbb, 0xd8, 0x5e, + 0x3a, 0x1a, 0x56, 0xe5, 0x05, 0x48, 0x90, 0x1e, 0x61, 0x69, 0x1b, 0x22, 0xe6, 0x1a, 0x3c, 0x75, 0xad, 0x1f, 0x37, + 0x28, 0xdc, 0xe4, 0x6d, 0xbd, 0x42, 0xdc, 0xd3, 0xc8, 0xb6, 0x1c, 0x48, 0xfe, 0x94, 0x77, 0x7f, 0xbd, 0x62, 0xac, + 0xa3, 0x47, 0x27, 0xcf, 0x5f, 0xd9, 0xdb, 0xaf, 0xec, 0xf7, 0x5e, 0xc1, 0xb0, 0x9d, 0x01, 0x26, 0x99, 0x7e, 0x8f, + 0x03, 0x70, 0xb5, 0x42, 0xbe, 0x67, 0x28, 0x1b, 0x7c, 0xbd, 0x61, 0x21, 0x97, 0xcc, 0x5c, 0xe1, 0x97, 0x8f, 0x8d, + 0xde, 0x2b, 0xaa, 0xa7, 0x71, 0x1d, 0x1e, 0x02, 0x73, 0x70, 0x58, 0x32, 0x5b, 0x1d, 0x67, 0x3d, 0xe0, 0x74, 0x4f, + 0x03, 0xf2, 0x70, 0x51, 0x79, 0xf1, 0x61, 0x70, 0x15, 0x74, 0x9d, 0x23, 0x89, 0xde, 0xac, 0xfd, 0xde, 0xd0, 0x1f, + 0xc3, 0x87, 0x44, 0x35, 0x4b, 0xe5, 0xb0, 0x60, 0xc5, 0x22, 0xe4, 0x9e, 0xca, 0xeb, 0xd5, 0x3a, 0x09, 0x45, 0xa4, + 0xdb, 0xfa, 0x3f, 0xeb, 0x1b, 0xc7, 0xc8, 0x14, 0x99, 0x51, 0x92, 0x10, 0xed, 0xed, 0x28, 0xe0, 0xa1, 0xf8, 0x26, + 0xcf, 0xcd, 0xcb, 0x63, 0xa1, 0x3b, 0xe3, 0xdf, 0x7e, 0xfe, 0xa6, 0xf0, 0x81, 0x9a, 0xbf, 0x55, 0xde, 0x54, 0xd5, + 0x56, 0x60, 0x98, 0x10, 0x68, 0xf4, 0x38, 0x96, 0x8e, 0x6f, 0x1d, 0x44, 0x7f, 0xd6, 0x2f, 0xfe, 0x55, 0xfb, 0x0c, + 0x7e, 0x67, 0xe2, 0x61, 0x44, 0xed, 0xf2, 0x35, 0x30, 0x5d, 0xe9, 0xc7, 0xd6, 0x6d, 0xe0, 0xa0, 0xed, 0xf3, 0xfc, + 0xd8, 0x3e, 0x0a, 0x7b, 0xcd, 0xaf, 0x65, 0x68, 0x18, 0xc0, 0xec, 0x04, 0x1c, 0x74, 0x6d, 0xe2, 0x6e, 0x79, 0xd4, + 0x11, 0x2b, 0x62, 0xd5, 0x27, 0xad, 0x4f, 0x01, 0x59, 0x73, 0xcc, 0x6a, 0x53, 0xfb, 0x2d, 0xd5, 0x4e, 0x99, 0x21, + 0x65, 0x4d, 0xf5, 0x82, 0xf7, 0xd8, 0x42, 0xce, 0x6f, 0x3d, 0x36, 0x47, 0xf1, 0x05, 0x16, 0xe8, 0x1b, 0x6a, 0x8f, + 0x93, 0xf2, 0x8f, 0x37, 0x40, 0x12, 0x28, 0xa3, 0xe6, 0xb9, 0x17, 0x4a, 0x1f, 0xb1, 0xd1, 0x66, 0x69, 0x86, 0xc4, + 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e, 0x2b, 0xdf, 0xcd, 0xf9, 0x3c +}; +static CONST UINT8 DecOutput011[] = { + 0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b, 0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b, 0x45, 0x60, 0xbe, + 0x9a, 0x31, 0x9f, 0xff, 0x5d, 0x66, 0x34, 0xb4, 0xdc, 0xfb, 0x9d, 0x8e, 0xee, 0x6a, 0x33, 0xa4, 0x07, 0x3c, 0xf9, + 0x4c, 0x30, 0xa1, 0x24, 0x52, 0xf9, 0x50, 0x46, 0x88, 0x20, 0x02, 0x32, 0x3a, 0x0e, 0x99, 0x63, 0xaf, 0x1f, 0x15, + 0x28, 0x2a, 0x05, 0xff, 0x57, 0x59, 0x5e, 0x18, 0xa1, 0x1f, 0xd0, 0x92, 0x5c, 0x88, 0x66, 0x1b, 0x00, 0x64, 0xa5, + 0x93, 0x8d, 0x06, 0x46, 0xb0, 0x64, 0x8b, 0x8b, 0xef, 0x99, 0x05, 0x35, 0x85, 0xb3, 0xf3, 0x33, 0xbb, 0xec, 0x66, + 0xb6, 0x3d, 0x57, 0x42, 0xe3, 0xb4, 0xc6, 0xaa, 0xb0, 0x41, 0x2a, 0xb9, 0x59, 0xa9, 0xf6, 0x3e, 0x15, 0x26, 0x12, + 0x03, 0x21, 0x4c, 0x74, 0x43, 0x13, 0x2a, 0x03, 0x27, 0x09, 0xb4, 0xfb, 0xe7, 0xb7, 0x40, 0xff, 0x5e, 0xce, 0x48, + 0x9a, 0x60, 0xe3, 0x8b, 0x80, 0x8c, 0x38, 0x2d, 0xcb, 0x93, 0x37, 0x74, 0x05, 0x52, 0x6f, 0x73, 0x3e, 0xc3, 0xbc, + 0xca, 0x72, 0x0a, 0xeb, 0xf1, 0x3b, 0xa0, 0x95, 0xdc, 0x8a, 0xc4, 0xa9, 0xdc, 0xca, 0x44, 0xd8, 0x08, 0x63, 0x6a, + 0x36, 0xd3, 0x3c, 0xb8, 0xac, 0x46, 0x7d, 0xfd, 0xaa, 0xeb, 0x3e, 0x0f, 0x45, 0x8f, 0x49, 0xda, 0x2b, 0xf2, 0x12, + 0xbd, 0xaf, 0x67, 0x8a, 0x63, 0x48, 0x4b, 0x55, 0x5f, 0x6d, 0x8c, 0xb9, 0x76, 0x34, 0x84, 0xae, 0xc2, 0xfc, 0x52, + 0x64, 0x82, 0xf7, 0xb0, 0x06, 0xf0, 0x45, 0x73, 0x12, 0x50, 0x30, 0x72, 0xea, 0x78, 0x9a, 0xa8, 0xaf, 0xb5, 0xe3, + 0xbb, 0x77, 0x52, 0xec, 0x59, 0x84, 0xbf, 0x6b, 0x8f, 0xce, 0x86, 0x5e, 0x1f, 0x23, 0xe9, 0xfb, 0x08, 0x86, 0xf7, + 0x10, 0xb9, 0xf2, 0x44, 0x96, 0x44, 0x63, 0xa9, 0xa8, 0x78, 0x00, 0x23, 0xd6, 0xc7, 0xe7, 0x6e, 0x66, 0x4f, 0xcc, + 0xee, 0x15, 0xb3, 0xbd, 0x1d, 0xa0, 0xe5, 0x9c, 0x1b, 0x24, 0x2c, 0x4d, 0x3c, 0x62, 0x35, 0x9c, 0x88, 0x59, 0x09, + 0xdd, 0x82, 0x1b, 0xcf, 0x0a, 0x83, 0x6b, 0x3f, 0xae, 0x03, 0xc4, 0xb4, 0xdd, 0x7e, 0x5b, 0x28, 0x76, 0x25, 0x96, + 0xd9, 0xc9, 0x9d, 0x5f, 0x86, 0xfa, 0xf6, 0xd7, 0xd2, 0xe6, 0x76, 0x1d, 0x0f, 0xa1, 0xdc, 0x74, 0x05, 0x1b, 0x1d, + 0xe0, 0xcd, 0x16, 0xb0, 0xa8, 0x8a, 0x34, 0x7b, 0x15, 0x11, 0x77, 0xe5, 0x7b, 0x7e, 0x20, 0xf7, 0xda, 0x38, 0xda, + 0xce, 0x70, 0xe9, 0xf5, 0x6c, 0xd9, 0xbe, 0x0c, 0x4c, 0x95, 0x4c, 0xc2, 0x9b, 0x34, 0x55, 0x55, 0xe1, 0xf3, 0x46, + 0x8e, 0x48, 0x74, 0x14, 0x4f, 0x9d, 0xc9, 0xf5, 0xe8, 0x1a, 0xf0, 0x11, 0x4a, 0xc1, 0x8d, 0xe0, 0x93, 0xa0, 0xbe, + 0x09, 0x1c, 0x2b, 0x4e, 0x0f, 0xb2, 0x87, 0x8b, 0x84, 0xfe, 0x92, 0x32, 0x14, 0xd7, 0x93, 0xdf, 0xe7, 0x44, 0xbc, + 0xc5, 0xae, 0x53, 0x69, 0xd8, 0xb3, 0x79, 0x37, 0x80, 0xe3, 0x17, 0x5c, 0xec, 0x53, 0x00, 0x9a, 0xe3, 0x8e, 0xdc, + 0x38, 0xb8, 0x66, 0xf0, 0xd3, 0xad, 0x1d, 0x02, 0x96, 0x86, 0x3e, 0x9d, 0x3b, 0x5d, 0xa5, 0x7f, 0x21, 0x10, 0xf1, + 0x1f, 0x13, 0x20, 0xf9, 0x57, 0x87, 0x20, 0xf5, 0x5f, 0xf1, 0x17, 0x48, 0x0a, 0x51, 0x5a, 0xcd, 0x19, 0x03, 0xa6, + 0x5a, 0xd1, 0x12, 0x97, 0xe9, 0x48, 0xe2, 0x1d, 0x83, 0x75, 0x50, 0xd9, 0x75, 0x7d, 0x6a, 0x82, 0xa1, 0xf9, 0x4e, + 0x54, 0x87, 0x89, 0xc9, 0x0c, 0xb7, 0x5b, 0x6a, 0x91, 0xc1, 0x9c, 0xb2, 0xa9, 0xdc, 0x9a, 0xa4, 0x49, 0x0a, 0x6d, + 0x0d, 0xbb, 0xde, 0x86, 0x44, 0xdd, 0x5d, 0x89, 0x2b, 0x96, 0x0f, 0x23, 0x95, 0xad, 0xcc, 0xa2, 0xb3, 0xb9, 0x7e, + 0x74, 0x38, 0xba, 0x9f, 0x73, 0xae, 0x5f, 0xf8, 0x68, 0xa2, 0xe0, 0xa9, 0xce, 0xbd, 0x40, 0xd4, 0x4c, 0x6b, 0xd2, + 0x56, 0x62, 0xb0, 0xcc, 0x63, 0x7e, 0x5b, 0xd3, 0xae, 0xd1, 0x75, 0xce, 0xbb, 0xb4, 0x5b, 0xa8, 0xf8, 0xb4, 0xac, + 0x71, 0x75, 0xaa, 0xc9, 0x9f, 0xbb, 0x6c, 0xad, 0x0f, 0x55, 0x5d, 0xe8, 0x85, 0x7d, 0xf9, 0x21, 0x35, 0xea, 0x92, + 0x85, 0x2b, 0x00, 0xec, 0x84, 0x90, 0x0a, 0x63, 0x96, 0xe4, 0x6b, 0xa9, 0x77, 0xb8, 0x91, 0xf8, 0x46, 0x15, 0x72, + 0x63, 0x70, 0x01, 0x40, 0xa3, 0xa5, 0x76, 0x62, 0x2b, 0xbf, 0xf1, 0xe5, 0x8d, 0x9f, 0xa3, 0xfa, 0x9b, 0x03, 0xbe, + 0xfe, 0x65, 0x6f, 0xa2, 0x29, 0x0d, 0x54, 0xb4, 0x71, 0xce, 0xa9, 0xd6, 0x3d, 0x88, 0xf9, 0xaf, 0x6b, 0xa8, 0x9e, + 0xf4, 0x16, 0x96, 0x36, 0xb9, 0x00, 0xdc, 0x10, 0xab, 0xb5, 0x08, 0x31, 0x1f, 0x00, 0xb1, 0x3c, 0xd9, 0x38, 0x3e, + 0xc6, 0x04, 0xa7, 0x4e, 0xe8, 0xae, 0xed, 0x98, 0xc2, 0xf7, 0xb9, 0x00, 0x5f, 0x8c, 0x60, 0xd1, 0xe5, 0x15, 0xf7, + 0xae, 0x1e, 0x84, 0x88, 0xd1, 0xf6, 0xbc, 0x3a, 0x89, 0x35, 0x22, 0x83, 0x7c, 0xca, 0xf0, 0x33, 0x82, 0x4c, 0x79, + 0x3c, 0xfd, 0xb1, 0xae, 0x52, 0x62, 0x55, 0xd2, 0x41, 0x60, 0xc6, 0xbb, 0xfa, 0x0e, 0x59, 0xd6, 0xa8, 0xfe, 0x5d, + 0xed, 0x47, 0x3d, 0xe0, 0xea, 0x1f, 0x6e, 0x43, 0x51, 0xec, 0x10, 0x52, 0x56, 0x77, 0x42, 0x6b, 0x52, 0x87, 0xd8, + 0xec, 0xe0, 0xaa, 0x76, 0xa5, 0x84, 0x2a, 0x22, 0x24, 0xfd, 0x92, 0x40, 0x88, 0xd5, 0x85, 0x1c, 0x1f, 0x6b, 0x47, + 0xa0, 0xc4, 0xe4, 0xef, 0xf4, 0xea, 0xd7, 0x59, 0xac, 0x2a, 0x9e, 0x8c, 0xfa, 0x1f, 0x42, 0x08, 0xfe, 0x4f, 0x74, + 0xa0, 0x26, 0xf5, 0xb3, 0x84, 0xf6, 0x58, 0x5f, 0x26, 0x66, 0x3e, 0xd7, 0xe4, 0x22, 0x91, 0x13, 0xc8, 0xac, 0x25, + 0x96, 0x23, 0xd8, 0x09, 0xea, 0x45, 0x75, 0x23, 0xb8, 0x5f, 0xc2, 0x90, 0x8b, 0x09, 0xc4, 0xfc, 0x47, 0x6c, 0x6d, + 0x0a, 0xef, 0x69, 0xa4, 0x38, 0x19, 0xcf, 0x7d, 0xf9, 0x09, 0x73, 0x9b, 0x60, 0x5a, 0xf7, 0x37, 0xb5, 0xfe, 0x9f, + 0xe3, 0x2b, 0x4c, 0x0d, 0x6e, 0x19, 0xf1, 0xd6, 0xc0, 0x70, 0xf3, 0x9d, 0x22, 0x3c, 0xf9, 0x49, 0xce, 0x30, 0x8e, + 0x44, 0xb5, 0x76, 0x15, 0x8f, 0x52, 0xfd, 0xa5, 0x04, 0xb8, 0x55, 0x6a, 0x36, 0x59, 0x7c, 0xc4, 0x48, 0xb8, 0xd7, + 0xab, 0x05, 0x66, 0xe9, 0x5e, 0x21, 0x6f, 0x6b, 0x36, 0x29, 0xbb, 0xe9, 0xe3, 0xa2, 0x9a, 0xa8, 0xcd, 0x55, 0x25, + 0x11, 0xba, 0x5a, 0x58, 0xa0, 0xde, 0xae, 0x19, 0x2a, 0x48, 0x5a, 0xff, 0x36, 0xcd, 0x6d, 0x16, 0x7a, 0x73, 0x38, + 0x46, 0xe5, 0x47, 0x59, 0xc8, 0xa2, 0xf6, 0xe2, 0x6c, 0x83, 0xc5, 0x36, 0x2c, 0x83, 0x7d, 0xb4, 0x01, 0x05, 0x69, + 0xe7, 0xaf, 0x5c, 0xc4, 0x64, 0x82, 0x12, 0x21, 0xef, 0xf7, 0xd1, 0x7d, 0xb8, 0x8d, 0x8c, 0x98, 0x7c, 0x5f, 0x7d, + 0x92, 0x88, 0xb9, 0x94, 0x07, 0x9c, 0xd8, 0xe9, 0x9c, 0x17, 0x38, 0xe3, 0x57, 0x6c, 0xe0, 0xdc, 0xa5, 0x92, 0x42, + 0xb3, 0xbd, 0x50, 0xa2, 0x7e, 0xb5, 0xb1, 0x52, 0x72, 0x03, 0x97, 0xd8, 0xaa, 0x9a, 0x1e, 0x75, 0x41, 0x11, 0xa3, + 0x4f, 0xcc, 0xd4, 0xe3, 0x73, 0xad, 0x96, 0xdc, 0x47, 0x41, 0x9f, 0xb0, 0xbe, 0x79, 0x91, 0xf5, 0xb6, 0x18, 0xfe, + 0xc2, 0x83, 0x18, 0x7d, 0x73, 0xd9, 0x4f, 0x83, 0x84, 0x03, 0xb3, 0xf0, 0x77, 0x66, 0x3d, 0x83, 0x63, 0x2e, 0x2c, + 0xf9, 0xdd, 0xa6, 0x1f, 0x89, 0x82, 0xb8, 0x23, 0x42, 0xeb, 0xe2, 0xca, 0x70, 0x82, 0x61, 0x41, 0x0a, 0x6d, 0x5f, + 0x75, 0xc5, 0xe2, 0xc4, 0x91, 0x18, 0x44, 0x22, 0xfa, 0x34, 0x10, 0xf5, 0x20, 0xdc, 0xb7, 0xdd, 0x2a, 0x20, 0x77, + 0xf5, 0xf9, 0xce, 0xdb, 0xa0, 0x0a, 0x52, 0x2a, 0x4e, 0xdd, 0xcc, 0x97, 0xdf, 0x05, 0xe4, 0x5e, 0xb7, 0xaa, 0xf0, + 0xe2, 0x80, 0xff, 0xba, 0x1a, 0x0f, 0xac, 0xdf, 0x02, 0x32, 0xe6, 0xf7, 0xc7, 0x17, 0x13, 0xb7, 0xfc, 0x98, 0x48, + 0x8c, 0x0d, 0x82, 0xc9, 0x80, 0x7a, 0xe2, 0x0a, 0xc5, 0xb4, 0xde, 0x7c, 0x3c, 0x79, 0x81, 0x0e, 0x28, 0x65, 0x79, + 0x67, 0x82, 0x69, 0x44, 0x66, 0x09, 0xf7, 0x16, 0x1a, 0xf9, 0x7d, 0x80, 0xa1, 0x79, 0x14, 0xa9, 0xc8, 0x20, 0xfb, + 0xa2, 0x46, 0xbe, 0x08, 0x35, 0x17, 0x58, 0xc1, 0x1a, 0xda, 0x2a, 0x6b, 0x2e, 0x1e, 0xe6, 0x27, 0x55, 0x7b, 0x19, + 0xe2, 0xfb, 0x64, 0xfc, 0x5e, 0x15, 0x54, 0x3c, 0xe7, 0xc2, 0x11, 0x50, 0x30, 0xb8, 0x72, 0x03, 0x0b, 0x1a, 0x9f, + 0x86, 0x27, 0x11, 0x5c, 0x06, 0x2b, 0xbd, 0x75, 0x1a, 0x0a, 0xda, 0x01, 0xfa, 0x5c, 0x4a, 0xc1, 0x80, 0x3a, 0x6e, + 0x30, 0xc8, 0x2c, 0xeb, 0x56, 0xec, 0x89, 0xfa, 0x35, 0x7b, 0xb2, 0xf0, 0x97, 0x08, 0x86, 0x53, 0xbe, 0xbd, 0x40, + 0x41, 0x38, 0x1c, 0xb4, 0x8b, 0x79, 0x2e, 0x18, 0x96, 0x94, 0xde, 0xe8, 0xca, 0xe5, 0x9f, 0x92, 0x9f, 0x15, 0x5d, + 0x56, 0x60, 0x5c, 0x09, 0xf9, 0x16, 0xf4, 0x17, 0x0f, 0xf6, 0x4c, 0xda, 0xe6, 0x67, 0x89, 0x9f, 0xca, 0x6c, 0xe7, + 0x9b, 0x04, 0x62, 0x0e, 0x26, 0xa6, 0x52, 0xbd, 0x29, 0xff, 0xc7, 0xa4, 0x96, 0xe6, 0x6a, 0x02, 0xa5, 0x2e, 0x7b, + 0xfe, 0x97, 0x68, 0x3e, 0x2e, 0x5f, 0x3b, 0x0f, 0x36, 0xd6, 0x98, 0x19, 0x59, 0x48, 0xd2, 0xc6, 0xe1, 0x55, 0x1a, + 0x6e, 0xd6, 0xed, 0x2c, 0xba, 0xc3, 0x9e, 0x64, 0xc9, 0x95, 0x86, 0x35, 0x5e, 0x3e, 0x88, 0x69, 0x99, 0x4b, 0xee, + 0xbe, 0x9a, 0x99, 0xb5, 0x6e, 0x58, 0xae, 0xdd, 0x22, 0xdb, 0xdd, 0x6b, 0xfc, 0xaf, 0x90, 0xa3, 0x3d, 0xa4, 0xc1, + 0x15, 0x92, 0x18, 0x8d, 0xd2, 0x4b, 0x7b, 0x06, 0xd1, 0x37, 0xb5, 0xe2, 0x7c, 0x2c, 0xf0, 0x25, 0xe4, 0x94, 0x2a, + 0xbd, 0xe3, 0x82, 0x70, 0x78, 0xa3, 0x82, 0x10, 0x5a, 0x90, 0xd7, 0xa4, 0xfa, 0xaf, 0x1a, 0x88, 0x59, 0xdc, 0x74, + 0x12, 0xb4, 0x8e, 0xd7, 0x19, 0x46, 0xf4, 0x84, 0x69, 0x9f, 0xbb, 0x70, 0xa8, 0x4c, 0x52, 0x81, 0xa9, 0xff, 0x76, + 0x1c, 0xae, 0xd8, 0x11, 0x3d, 0x7f, 0x7d, 0xc5, 0x12, 0x59, 0x28, 0x18, 0xc2, 0xa2, 0xb7, 0x1c, 0x88, 0xf8, 0xd6, + 0x1b, 0xa6, 0x7d, 0x9e, 0xde, 0x29, 0xf8, 0xed, 0xff, 0xeb, 0x92, 0x24, 0x4f, 0x05, 0xaa, 0xd9, 0x49, 0xba, 0x87, + 0x59, 0x51, 0xc9, 0x20, 0x5c, 0x9b, 0x74, 0xcf, 0x03, 0xd9, 0x2d, 0x34, 0xc7, 0x5b, 0xa5, 0x40, 0xb2, 0x99, 0xf5, + 0xcb, 0xb4, 0xf6, 0xb7, 0x72, 0x4a, 0xd6, 0xbd, 0xb0, 0xf3, 0x93, 0xe0, 0x1b, 0xa8, 0x04, 0x1e, 0x35, 0xd4, 0x80, + 0x20, 0xf4, 0x9c, 0x31, 0x6b, 0x45, 0xb9, 0x15, 0xb0, 0x5e, 0xdd, 0x0a, 0x33, 0x9c, 0x83, 0xcd, 0x58, 0x89, 0x50, + 0x56, 0xbb, 0x81, 0x00, 0x91, 0x32, 0xf3, 0x1b, 0x3e, 0xcf, 0x45, 0xe1, 0xf9, 0xe1, 0x2c, 0x26, 0x78, 0x93, 0x9a, + 0x60, 0x46, 0xc9, 0xb5, 0x5e, 0x6a, 0x28, 0x92, 0x87, 0x3f, 0x63, 0x7b, 0xdb, 0xf7, 0xd0, 0x13, 0x9d, 0x32, 0x40, + 0x5e, 0xcf, 0xfb, 0x79, 0x68, 0x47, 0x4c, 0xfd, 0x01, 0x17, 0xe6, 0x97, 0x93, 0x78, 0xbb, 0xa6, 0x27, 0xa3, 0xe8, + 0x1a, 0xe8, 0x94, 0x55, 0x7d, 0x08, 0xe5, 0xdc, 0x66, 0xa3, 0x69, 0xc8, 0xca, 0xc5, 0xa1, 0x84, 0x55, 0xde, 0x08, + 0x91, 0x16, 0x3a, 0x0c, 0x86, 0xab, 0x27, 0x2b, 0x64, 0x34, 0x02, 0x6c, 0x76, 0x8b, 0xc6, 0xaf, 0xcc, 0xe1, 0xd6, + 0x8c, 0x2a, 0x18, 0x3d, 0xa6, 0x1b, 0x37, 0x75, 0x45, 0x73, 0xc2, 0x75, 0xd7, 0x53, 0x78, 0x3a, 0xd6, 0xe8, 0x29, + 0xd2, 0x4a, 0xa8, 0x1e, 0x82, 0xf6, 0xb6, 0x81, 0xde, 0x21, 0xed, 0x2b, 0x56, 0xbb, 0xf2, 0xd0, 0x57, 0xc1, 0x7c, + 0xd2, 0x6a, 0xd2, 0x56, 0xf5, 0x13, 0x5f, 0x1c, 0x6a, 0x0b, 0x74, 0xfb, 0xe9, 0xfe, 0x9e, 0xea, 0x95, 0xb2, 0x46, + 0xab, 0x0a, 0xfc, 0xfd, 0xf3, 0xbb, 0x04, 0x2b, 0x76, 0x1b, 0xa4, 0x74, 0xb0, 0xc1, 0x78, 0xc3, 0x69, 0xe2, 0xb0, + 0x01, 0xe1, 0xde, 0x32, 0x4c, 0x8d, 0x1a, 0xb3, 0x38, 0x08, 0xd5, 0xfc, 0x1f, 0xdc, 0x0e, 0x2c, 0x9c, 0xb1, 0xa1, + 0x63, 0x17, 0x22, 0xf5, 0x6c, 0x93, 0x70, 0x74, 0x00, 0xf8, 0x39, 0x01, 0x94, 0xd1, 0x32, 0x23, 0x56, 0x5d, 0xa6, + 0x02, 0x76, 0x76, 0x93, 0xce, 0x2f, 0x19, 0xe9, 0x17, 0x52, 0xae, 0x6e, 0x2c, 0x6d, 0x61, 0x7f, 0x3b, 0xaa, 0xe0, + 0x52, 0x85, 0xc5, 0x65, 0xc1, 0xbb, 0x8e, 0x5b, 0x21, 0xd5, 0xc9, 0x78, 0x83, 0x07, 0x97, 0x4c, 0x62, 0x61, 0x41, + 0xd4, 0xfc, 0xc9, 0x39, 0xe3, 0x9b, 0xd0, 0xcc, 0x75, 0xc4, 0x97, 0xe6, 0xdd, 0x2a, 0x5f, 0xa6, 0xe8, 0x59, 0x6c, + 0x98, 0xb9, 0x02, 0xe2, 0xa2, 0xd6, 0x68, 0xee, 0x3b, 0x1d, 0xe3, 0x4d, 0x5b, 0x30, 0xef, 0x03, 0xf2, 0xeb, 0x18, + 0x57, 0x36, 0xe8, 0xa1, 0xf4, 0x47, 0xfb, 0xcb, 0x8f, 0xcb, 0xc8, 0xf3, 0x4f, 0x74, 0x9d, 0x9d, 0xb1, 0x8d, 0x14, + 0x44, 0xd9, 0x19, 0xb4, 0x54, 0x4f, 0x75, 0x19, 0x09, 0xa0, 0x75, 0xbc, 0x3b, 0x82, 0xc6, 0x3f, 0xb8, 0x83, 0x19, + 0x6e, 0xd6, 0x37, 0xfe, 0x6e, 0x8a, 0x4e, 0xe0, 0x4a, 0xab, 0x7b, 0xc8, 0xb4, 0x1d, 0xf4, 0xed, 0x27, 0x03, 0x65, + 0xa2, 0xa1, 0xae, 0x11, 0xe7, 0x98, 0x78, 0x48, 0x91, 0xd2, 0xd2, 0xd4, 0x23, 0x78, 0x50, 0xb1, 0x5b, 0x85, 0x10, + 0x8d, 0xca, 0x5f, 0x0f, 0x71, 0xae, 0x72, 0x9a, 0xf6, 0x25, 0x19, 0x60, 0x06, 0xf7, 0x10, 0x34, 0x18, 0x0d, 0xc9, + 0x9f, 0x7b, 0x0c, 0x9b, 0x8f, 0x91, 0x1b, 0x9f, 0xcd, 0x10, 0xee, 0x75, 0xf9, 0x97, 0x66, 0xfc, 0x4d, 0x33, 0x6e, + 0x28, 0x2b, 0x92, 0x85, 0x4f, 0xab, 0x43, 0x8d, 0x8f, 0x7d, 0x86, 0xa7, 0xc7, 0xd8, 0xd3, 0x0b, 0x8b, 0x57, 0xb6, + 0x1d, 0x95, 0x0d, 0xe9, 0xbc, 0xd9, 0x03, 0xd9, 0x10, 0x19, 0xc3, 0x46, 0x63, 0x55, 0x87, 0x61, 0x79, 0x6c, 0x95, + 0x0e, 0x9c, 0xdd, 0xca, 0xc3, 0xf3, 0x64, 0xf0, 0x7d, 0x76, 0xb7, 0x53, 0x67, 0x2b, 0x1e, 0x44, 0x56, 0x81, 0xea, + 0x8f, 0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61, 0x10, 0x1e, 0xbf, 0xec, 0xa8 +}; +static CONST UINT8 DecAssoc011[] = { 0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7 }; +static CONST UINT8 DecNonce011[] = { 0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa }; +static CONST UINT8 DecKey011[] = { 0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85, 0xf2, 0xfb, 0xed, + 0x7b, 0xd0, 0x9e, 0x97, 0xca, 0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, + 0xcc, 0x52, 0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38 }; + +static CONST UINT8 DecInput012[] = { + 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, 0xdf, 0x34, 0x56, + 0x82, 0xe2, 0xbe, 0xe5, 0xe1, 0x28, 0xd1, 0x4e, 0x5f, 0x4f, 0x01, 0x7d, 0x3f, 0x99, 0x6b, 0x30, 0x6e, 0x1a, 0x7c, + 0x4c, 0x8e, 0x62, 0x81, 0xae, 0x86, 0x3f, 0x6b, 0xd0, 0xb5, 0xa9, 0xcf, 0x50, 0xf1, 0x02, 0x12, 0xa0, 0x0b, 0x24, + 0xe9, 0xe6, 0x72, 0x89, 0x2c, 0x52, 0x1b, 0x34, 0x38, 0xf8, 0x75, 0x5f, 0xa0, 0x74, 0xe2, 0x99, 0xdd, 0xa6, 0x4b, + 0x14, 0x50, 0x4e, 0xf1, 0xbe, 0xd6, 0x9e, 0xdb, 0xb2, 0x24, 0x27, 0x74, 0x12, 0x4a, 0x78, 0x78, 0x17, 0xa5, 0x58, + 0x8e, 0x2f, 0xf9, 0xf4, 0x8d, 0xee, 0x03, 0x88, 0xae, 0xb8, 0x29, 0xa1, 0x2f, 0x4b, 0xee, 0x92, 0xbd, 0x87, 0xb3, + 0xce, 0x34, 0x21, 0x57, 0x46, 0x04, 0x49, 0x0c, 0x80, 0xf2, 0x01, 0x13, 0xa1, 0x55, 0xb3, 0xff, 0x44, 0x30, 0x3c, + 0x1c, 0xd0, 0xef, 0xbc, 0x18, 0x74, 0x26, 0xad, 0x41, 0x5b, 0x5b, 0x3e, 0x9a, 0x7a, 0x46, 0x4f, 0x16, 0xd6, 0x74, + 0x5a, 0xb7, 0x3a, 0x28, 0x31, 0xd8, 0xae, 0x26, 0xac, 0x50, 0x53, 0x86, 0xf2, 0x56, 0xd7, 0x3f, 0x29, 0xbc, 0x45, + 0x68, 0x8e, 0xcb, 0x98, 0x64, 0xdd, 0xc9, 0xba, 0xb8, 0x4b, 0x7b, 0x82, 0xdd, 0x14, 0xa7, 0xcb, 0x71, 0x72, 0x00, + 0x5c, 0xad, 0x7b, 0x6a, 0x89, 0xa4, 0x3d, 0xbf, 0xb5, 0x4b, 0x3e, 0x7c, 0x5a, 0xcf, 0xb8, 0xa1, 0xc5, 0x6e, 0xc8, + 0xb6, 0x31, 0x57, 0x7b, 0xdf, 0xa5, 0x7e, 0xb1, 0xd6, 0x42, 0x2a, 0x31, 0x36, 0xd1, 0xd0, 0x3f, 0x7a, 0xe5, 0x94, + 0xd6, 0x36, 0xa0, 0x6f, 0xb7, 0x40, 0x7d, 0x37, 0xc6, 0x55, 0x7c, 0x50, 0x40, 0x6d, 0x29, 0x89, 0xe3, 0x5a, 0xae, + 0x97, 0xe7, 0x44, 0x49, 0x6e, 0xbd, 0x81, 0x3d, 0x03, 0x93, 0x06, 0x12, 0x06, 0xe2, 0x41, 0x12, 0x4a, 0xf1, 0x6a, + 0xa4, 0x58, 0xa2, 0xfb, 0xd2, 0x15, 0xba, 0xc9, 0x79, 0xc9, 0xce, 0x5e, 0x13, 0xbb, 0xf1, 0x09, 0x04, 0xcc, 0xfd, + 0xe8, 0x51, 0x34, 0x6a, 0xe8, 0x61, 0x88, 0xda, 0xed, 0x01, 0x47, 0x84, 0xf5, 0x73, 0x25, 0xf9, 0x1c, 0x42, 0x86, + 0x07, 0xf3, 0x5b, 0x1a, 0x01, 0xb3, 0xeb, 0x24, 0x32, 0x8d, 0xf6, 0xed, 0x7c, 0x4b, 0xeb, 0x3c, 0x36, 0x42, 0x28, + 0xdf, 0xdf, 0xb6, 0xbe, 0xd9, 0x8c, 0x52, 0xd3, 0x2b, 0x08, 0x90, 0x8c, 0xe7, 0x98, 0x31, 0xe2, 0x32, 0x8e, 0xfc, + 0x11, 0x48, 0x00, 0xa8, 0x6a, 0x42, 0x4a, 0x02, 0xc6, 0x4b, 0x09, 0xf1, 0xe3, 0x49, 0xf3, 0x45, 0x1f, 0x0e, 0xbc, + 0x56, 0xe2, 0xe4, 0xdf, 0xfb, 0xeb, 0x61, 0xfa, 0x24, 0xc1, 0x63, 0x75, 0xbb, 0x47, 0x75, 0xaf, 0xe1, 0x53, 0x16, + 0x96, 0x21, 0x85, 0x26, 0x11, 0xb3, 0x76, 0xe3, 0x23, 0xa1, 0x6b, 0x74, 0x37, 0xd0, 0xde, 0x06, 0x90, 0x71, 0x5d, + 0x43, 0x88, 0x9b, 0x00, 0x54, 0xa6, 0x75, 0x2f, 0xa1, 0xc2, 0x0b, 0x73, 0x20, 0x1d, 0xb6, 0x21, 0x79, 0x57, 0x3f, + 0xfa, 0x09, 0xbe, 0x8a, 0x33, 0xc3, 0x52, 0xf0, 0x1d, 0x82, 0x31, 0xd1, 0x55, 0xb5, 0x6c, 0x99, 0x25, 0xcf, 0x5c, + 0x32, 0xce, 0xe9, 0x0d, 0xfa, 0x69, 0x2c, 0xd5, 0x0d, 0xc5, 0x6d, 0x86, 0xd0, 0x0c, 0x3b, 0x06, 0x50, 0x79, 0xe8, + 0xc3, 0xae, 0x04, 0xe6, 0xcd, 0x51, 0xe4, 0x26, 0x9b, 0x4f, 0x7e, 0xa6, 0x0f, 0xab, 0xd8, 0xe5, 0xde, 0xa9, 0x00, + 0x95, 0xbe, 0xa3, 0x9d, 0x5d, 0xb2, 0x09, 0x70, 0x18, 0x1c, 0xf0, 0xac, 0x29, 0x23, 0x02, 0x29, 0x28, 0xd2, 0x74, + 0x35, 0x57, 0x62, 0x0f, 0x24, 0xea, 0x5e, 0x33, 0xc2, 0x92, 0xf3, 0x78, 0x4d, 0x30, 0x1e, 0xa1, 0x99, 0xa9, 0x82, + 0xb0, 0x42, 0x31, 0x8d, 0xad, 0x8a, 0xbc, 0xfc, 0xd4, 0x57, 0x47, 0x3e, 0xb4, 0x50, 0xdd, 0x6e, 0x2c, 0x80, 0x4d, + 0x22, 0xf1, 0xfb, 0x57, 0xc4, 0xdd, 0x17, 0xe1, 0x8a, 0x36, 0x4a, 0xb3, 0x37, 0xca, 0xc9, 0x4e, 0xab, 0xd5, 0x69, + 0xc4, 0xf4, 0xbc, 0x0b, 0x3b, 0x44, 0x4b, 0x29, 0x9c, 0xee, 0xd4, 0x35, 0x22, 0x21, 0xb0, 0x1f, 0x27, 0x64, 0xa8, + 0x51, 0x1b, 0xf0, 0x9f, 0x19, 0x5c, 0xfb, 0x5a, 0x64, 0x74, 0x70, 0x45, 0x09, 0xf5, 0x64, 0xfe, 0x1a, 0x2d, 0xc9, + 0x14, 0x04, 0x14, 0xcf, 0xd5, 0x7d, 0x60, 0xaf, 0x94, 0x39, 0x94, 0xe2, 0x7d, 0x79, 0x82, 0xd0, 0x65, 0x3b, 0x6b, + 0x9c, 0x19, 0x84, 0xb4, 0x6d, 0xb3, 0x0c, 0x99, 0xc0, 0x56, 0xa8, 0xbd, 0x73, 0xce, 0x05, 0x84, 0x3e, 0x30, 0xaa, + 0xc4, 0x9b, 0x1b, 0x04, 0x2a, 0x9f, 0xd7, 0x43, 0x2b, 0x23, 0xdf, 0xbf, 0xaa, 0xd5, 0xc2, 0x43, 0x2d, 0x70, 0xab, + 0xdc, 0x75, 0xad, 0xac, 0xf7, 0xc0, 0xbe, 0x67, 0xb2, 0x74, 0xed, 0x67, 0x10, 0x4a, 0x92, 0x60, 0xc1, 0x40, 0x50, + 0x19, 0x8a, 0x8a, 0x8c, 0x09, 0x0e, 0x72, 0xe1, 0x73, 0x5e, 0xe8, 0x41, 0x85, 0x63, 0x9f, 0x3f, 0xd7, 0x7d, 0xc4, + 0xfb, 0x22, 0x5d, 0x92, 0x6c, 0xb3, 0x1e, 0xe2, 0x50, 0x2f, 0x82, 0xa8, 0x28, 0xc0, 0xb5, 0xd7, 0x5f, 0x68, 0x0d, + 0x2c, 0x2d, 0xaf, 0x7e, 0xfa, 0x2e, 0x08, 0x0f, 0x1f, 0x70, 0x9f, 0xe9, 0x19, 0x72, 0x55, 0xf8, 0xfb, 0x51, 0xd2, + 0x33, 0x5d, 0xa0, 0xd3, 0x2b, 0x0a, 0x6c, 0xbc, 0x4e, 0xcf, 0x36, 0x4d, 0xdc, 0x3b, 0xe9, 0x3e, 0x81, 0x7c, 0x61, + 0xdb, 0x20, 0x2d, 0x3a, 0xc3, 0xb3, 0x0c, 0x1e, 0x00, 0xb9, 0x7c, 0xf5, 0xca, 0x10, 0x5f, 0x3a, 0x71, 0xb3, 0xe4, + 0x20, 0xdb, 0x0c, 0x2a, 0x98, 0x63, 0x45, 0x00, 0x58, 0xf6, 0x68, 0xe4, 0x0b, 0xda, 0x13, 0x3b, 0x60, 0x5c, 0x76, + 0xdb, 0xb9, 0x97, 0x71, 0xe4, 0xd9, 0xb7, 0xdb, 0xbd, 0x68, 0xc7, 0x84, 0x84, 0xaa, 0x7c, 0x68, 0x62, 0x5e, 0x16, + 0xfc, 0xba, 0x72, 0xaa, 0x9a, 0xa9, 0xeb, 0x7c, 0x75, 0x47, 0x97, 0x7e, 0xad, 0xe2, 0xd9, 0x91, 0xe8, 0xe4, 0xa5, + 0x31, 0xd7, 0x01, 0x8e, 0xa2, 0x11, 0x88, 0x95, 0xb9, 0xf2, 0x9b, 0xd3, 0x7f, 0x1b, 0x81, 0x22, 0xf7, 0x98, 0x60, + 0x0a, 0x64, 0xa6, 0xc1, 0xf6, 0x49, 0xc7, 0xe3, 0x07, 0x4d, 0x94, 0x7a, 0xcf, 0x6e, 0x68, 0x0c, 0x1b, 0x3f, 0x6e, + 0x2e, 0xee, 0x92, 0xfa, 0x52, 0xb3, 0x59, 0xf8, 0xf1, 0x8f, 0x6a, 0x66, 0xa3, 0x82, 0x76, 0x4a, 0x07, 0x1a, 0xc7, + 0xdd, 0xf5, 0xda, 0x9c, 0x3c, 0x24, 0xbf, 0xfd, 0x42, 0xa1, 0x10, 0x64, 0x6a, 0x0f, 0x89, 0xee, 0x36, 0xa5, 0xce, + 0x99, 0x48, 0x6a, 0xf0, 0x9f, 0x9e, 0x69, 0xa4, 0x40, 0x20, 0xe9, 0x16, 0x15, 0xf7, 0xdb, 0x75, 0x02, 0xcb, 0xe9, + 0x73, 0x8b, 0x3b, 0x49, 0x2f, 0xf0, 0xaf, 0x51, 0x06, 0x5c, 0xdf, 0x27, 0x27, 0x49, 0x6a, 0xd1, 0xcc, 0xc7, 0xb5, + 0x63, 0xb5, 0xfc, 0xb8, 0x5c, 0x87, 0x7f, 0x84, 0xb4, 0xcc, 0x14, 0xa9, 0x53, 0xda, 0xa4, 0x56, 0xf8, 0xb6, 0x1b, + 0xcc, 0x40, 0x27, 0x52, 0x06, 0x5a, 0x13, 0x81, 0xd7, 0x3a, 0xd4, 0x3b, 0xfb, 0x49, 0x65, 0x31, 0x33, 0xb2, 0xfa, + 0xcd, 0xad, 0x58, 0x4e, 0x2b, 0xae, 0xd2, 0x20, 0xfb, 0x1a, 0x48, 0xb4, 0x3f, 0x9a, 0xd8, 0x7a, 0x35, 0x4a, 0xc8, + 0xee, 0x88, 0x5e, 0x07, 0x66, 0x54, 0xb9, 0xec, 0x9f, 0xa3, 0xe3, 0xb9, 0x37, 0xaa, 0x49, 0x76, 0x31, 0xda, 0x74, + 0x2d, 0x3c, 0xa4, 0x65, 0x10, 0x32, 0x38, 0xf0, 0xde, 0xd3, 0x99, 0x17, 0xaa, 0x71, 0xaa, 0x8f, 0x0f, 0x8c, 0xaf, + 0xa2, 0xf8, 0x5d, 0x64, 0xba, 0x1d, 0xa3, 0xef, 0x96, 0x73, 0xe8, 0xa1, 0x02, 0x8d, 0x0c, 0x6d, 0xb8, 0x06, 0x90, + 0xb8, 0x08, 0x56, 0x2c, 0xa7, 0x06, 0xc9, 0xc2, 0x38, 0xdb, 0x7c, 0x63, 0xb1, 0x57, 0x8e, 0xea, 0x7c, 0x79, 0xf3, + 0x49, 0x1d, 0xfe, 0x9f, 0xf3, 0x6e, 0xb1, 0x1d, 0xba, 0x19, 0x80, 0x1a, 0x0a, 0xd3, 0xb0, 0x26, 0x21, 0x40, 0xb1, + 0x7c, 0xf9, 0x4d, 0x8d, 0x10, 0xc1, 0x7e, 0xf4, 0xf6, 0x3c, 0xa8, 0xfd, 0x7c, 0xa3, 0x92, 0xb2, 0x0f, 0xaa, 0xcc, + 0xa6, 0x11, 0xfe, 0x04, 0xe3, 0xd1, 0x7a, 0x32, 0x89, 0xdf, 0x0d, 0xc4, 0x8f, 0x79, 0x6b, 0xca, 0x16, 0x7c, 0x6e, + 0xf9, 0xad, 0x0f, 0xf6, 0xfe, 0x27, 0xdb, 0xc4, 0x13, 0x70, 0xf1, 0x62, 0x1a, 0x4f, 0x79, 0x40, 0xc9, 0x9b, 0x8b, + 0x21, 0xea, 0x84, 0xfa, 0xf5, 0xf1, 0x89, 0xce, 0xb7, 0x55, 0x0a, 0x80, 0x39, 0x2f, 0x55, 0x36, 0x16, 0x9c, 0x7b, + 0x08, 0xbd, 0x87, 0x0d, 0xa5, 0x32, 0xf1, 0x52, 0x7c, 0xe8, 0x55, 0x60, 0x5b, 0xd7, 0x69, 0xe4, 0xfc, 0xfa, 0x12, + 0x85, 0x96, 0xea, 0x50, 0x28, 0xab, 0x8a, 0xf7, 0xbb, 0x0e, 0x53, 0x74, 0xca, 0xa6, 0x27, 0x09, 0xc2, 0xb5, 0xde, + 0x18, 0x14, 0xd9, 0xea, 0xe5, 0x29, 0x1c, 0x40, 0x56, 0xcf, 0xd7, 0xae, 0x05, 0x3f, 0x65, 0xaf, 0x05, 0x73, 0xe2, + 0x35, 0x96, 0x27, 0x07, 0x14, 0xc0, 0xad, 0x33, 0xf1, 0xdc, 0x44, 0x7a, 0x89, 0x17, 0x77, 0xd2, 0x9c, 0x58, 0x60, + 0xf0, 0x3f, 0x7b, 0x2d, 0x2e, 0x57, 0x95, 0x54, 0x87, 0xed, 0xf2, 0xc7, 0x4c, 0xf0, 0xae, 0x56, 0x29, 0x19, 0x7d, + 0x66, 0x4b, 0x9b, 0x83, 0x84, 0x42, 0x3b, 0x01, 0x25, 0x66, 0x8e, 0x02, 0xde, 0xb9, 0x83, 0x54, 0x19, 0xf6, 0x9f, + 0x79, 0x0d, 0x67, 0xc5, 0x1d, 0x7a, 0x44, 0x02, 0x98, 0xa7, 0x16, 0x1c, 0x29, 0x0d, 0x74, 0xff, 0x85, 0x40, 0x06, + 0xef, 0x2c, 0xa9, 0xc6, 0xf5, 0x53, 0x07, 0x06, 0xae, 0xe4, 0xfa, 0x5f, 0xd8, 0x39, 0x4d, 0xf1, 0x9b, 0x6b, 0xd9, + 0x24, 0x84, 0xfe, 0x03, 0x4c, 0xb2, 0x3f, 0xdf, 0xa1, 0x05, 0x9e, 0x50, 0x14, 0x5a, 0xd9, 0x1a, 0xa2, 0xa7, 0xfa, + 0xfa, 0x17, 0xf7, 0x78, 0xd6, 0xb5, 0x92, 0x61, 0x91, 0xac, 0x36, 0xfa, 0x56, 0x0d, 0x38, 0x32, 0x18, 0x85, 0x08, + 0x58, 0x37, 0xf0, 0x4b, 0xdb, 0x59, 0xe7, 0xa4, 0x34, 0xc0, 0x1b, 0x01, 0xaf, 0x2d, 0xde, 0xa1, 0xaa, 0x5d, 0xd3, + 0xec, 0xe1, 0xd4, 0xf7, 0xe6, 0x54, 0x68, 0xf0, 0x51, 0x97, 0xa7, 0x89, 0xea, 0x24, 0xad, 0xd3, 0x6e, 0x47, 0x93, + 0x8b, 0x4b, 0xb4, 0xf7, 0x1c, 0x42, 0x06, 0x67, 0xe8, 0x99, 0xf6, 0xf5, 0x7b, 0x85, 0xb5, 0x65, 0xb5, 0xb5, 0xd2, + 0x37, 0xf5, 0xf3, 0x02, 0xa6, 0x4d, 0x11, 0xa7, 0xdc, 0x51, 0x09, 0x7f, 0xa0, 0xd8, 0x88, 0x1c, 0x13, 0x71, 0xae, + 0x9c, 0xb7, 0x7b, 0x34, 0xd6, 0x4e, 0x68, 0x26, 0x83, 0x51, 0xaf, 0x1d, 0xee, 0x8b, 0xbb, 0x69, 0x43, 0x2b, 0x9e, + 0x8a, 0xbc, 0x02, 0x0e, 0xa0, 0x1b, 0xe0, 0xa8, 0x5f, 0x6f, 0xaf, 0x1b, 0x8f, 0xe7, 0x64, 0x71, 0x74, 0x11, 0x7e, + 0xa8, 0xd8, 0xf9, 0x97, 0x06, 0xc3, 0xb6, 0xfb, 0xfb, 0xb7, 0x3d, 0x35, 0x9d, 0x3b, 0x52, 0xed, 0x54, 0xca, 0xf4, + 0x81, 0x01, 0x2d, 0x1b, 0xc3, 0xa7, 0x00, 0x3d, 0x1a, 0x39, 0x54, 0xe1, 0xf6, 0xff, 0xed, 0x6f, 0x0b, 0x5a, 0x68, + 0xda, 0x58, 0xdd, 0xa9, 0xcf, 0x5c, 0x4a, 0xe5, 0x09, 0x4e, 0xde, 0x9d, 0xbc, 0x3e, 0xee, 0x5a, 0x00, 0x3b, 0x2c, + 0x87, 0x10, 0x65, 0x60, 0xdd, 0xd7, 0x56, 0xd1, 0x4c, 0x64, 0x45, 0xe4, 0x21, 0xec, 0x78, 0xf8, 0x25, 0x7a, 0x3e, + 0x16, 0x5d, 0x09, 0x53, 0x14, 0xbe, 0x4f, 0xae, 0x87, 0xd8, 0xd1, 0xaa, 0x3c, 0xf6, 0x3e, 0xa4, 0x70, 0x8c, 0x5e, + 0x70, 0xa4, 0xb3, 0x6b, 0x66, 0x73, 0xd3, 0xbf, 0x31, 0x06, 0x19, 0x62, 0x93, 0x15, 0xf2, 0x86, 0xe4, 0x52, 0x7e, + 0x53, 0x4c, 0x12, 0x38, 0xcc, 0x34, 0x7d, 0x57, 0xf6, 0x42, 0x93, 0x8a, 0xc4, 0xee, 0x5c, 0x8a, 0xe1, 0x52, 0x8f, + 0x56, 0x64, 0xf6, 0xa6, 0xd1, 0x91, 0x57, 0x70, 0xcd, 0x11, 0x76, 0xf5, 0x59, 0x60, 0x60, 0x3c, 0xc1, 0xc3, 0x0b, + 0x7f, 0x58, 0x1a, 0x50, 0x91, 0xf1, 0x68, 0x8f, 0x6e, 0x74, 0x74, 0xa8, 0x51, 0x0b, 0xf7, 0x7a, 0x98, 0x37, 0xf2, + 0x0a, 0x0e, 0xa4, 0x97, 0x04, 0xb8, 0x9b, 0xfd, 0xa0, 0xea, 0xf7, 0x0d, 0xe1, 0xdb, 0x03, 0xf0, 0x31, 0x29, 0xf8, + 0xdd, 0x6b, 0x8b, 0x5d, 0xd8, 0x59, 0xa9, 0x29, 0xcf, 0x9a, 0x79, 0x89, 0x19, 0x63, 0x46, 0x09, 0x79, 0x6a, 0x11, + 0xda, 0x63, 0x68, 0x48, 0x77, 0x23, 0xfb, 0x7d, 0x3a, 0x43, 0xcb, 0x02, 0x3b, 0x7a, 0x6d, 0x10, 0x2a, 0x9e, 0xac, + 0xf1, 0xd4, 0x19, 0xf8, 0x23, 0x64, 0x1d, 0x2c, 0x5f, 0xf2, 0xb0, 0x5c, 0x23, 0x27, 0xf7, 0x27, 0x30, 0x16, 0x37, + 0xb1, 0x90, 0xab, 0x38, 0xfb, 0x55, 0xcd, 0x78, 0x58, 0xd4, 0x7d, 0x43, 0xf6, 0x45, 0x5e, 0x55, 0x8d, 0xb1, 0x02, + 0x65, 0x58, 0xb4, 0x13, 0x4b, 0x36, 0xf7, 0xcc, 0xfe, 0x3d, 0x0b, 0x82, 0xe2, 0x12, 0x11, 0xbb, 0xe6, 0xb8, 0x3a, + 0x48, 0x71, 0xc7, 0x50, 0x06, 0x16, 0x3a, 0xe6, 0x7c, 0x05, 0xc7, 0xc8, 0x4d, 0x2f, 0x08, 0x6a, 0x17, 0x9a, 0x95, + 0x97, 0x50, 0x68, 0xdc, 0x28, 0x18, 0xc4, 0x61, 0x38, 0xb9, 0xe0, 0x3e, 0x78, 0xdb, 0x29, 0xe0, 0x9f, 0x52, 0xdd, + 0xf8, 0x4f, 0x91, 0xc1, 0xd0, 0x33, 0xa1, 0x7a, 0x8e, 0x30, 0x13, 0x82, 0x07, 0x9f, 0xd3, 0x31, 0x0f, 0x23, 0xbe, + 0x32, 0x5a, 0x75, 0xcf, 0x96, 0xb2, 0xec, 0xb5, 0x32, 0xac, 0x21, 0xd1, 0x82, 0x33, 0xd3, 0x15, 0x74, 0xbd, 0x90, + 0xf1, 0x2c, 0xe6, 0x5f, 0x8d, 0xe3, 0x02, 0xe8, 0xe9, 0xc4, 0xca, 0x96, 0xeb, 0x0e, 0xbc, 0x91, 0xf4, 0xb9, 0xea, + 0xd9, 0x1b, 0x75, 0xbd, 0xe1, 0xac, 0x2a, 0x05, 0x37, 0x52, 0x9b, 0x1b, 0x3f, 0x5a, 0xdc, 0x21, 0xc3, 0x98, 0xbb, + 0xaf, 0xa3, 0xf2, 0x00, 0xbf, 0x0d, 0x30, 0x89, 0x05, 0xcc, 0xa5, 0x76, 0xf5, 0x06, 0xf0, 0xc6, 0x54, 0x8a, 0x5d, + 0xd4, 0x1e, 0xc1, 0xf2, 0xce, 0xb0, 0x62, 0xc8, 0xfc, 0x59, 0x42, 0x9a, 0x90, 0x60, 0x55, 0xfe, 0x88, 0xa5, 0x8b, + 0xb8, 0x33, 0x0c, 0x23, 0x24, 0x0d, 0x15, 0x70, 0x37, 0x1e, 0x3d, 0xf6, 0xd2, 0xea, 0x92, 0x10, 0xb2, 0xc4, 0x51, + 0xac, 0xf2, 0xac, 0xf3, 0x6b, 0x6c, 0xaa, 0xcf, 0x12, 0xc5, 0x6c, 0x90, 0x50, 0xb5, 0x0c, 0xfc, 0x1a, 0x15, 0x52, + 0xe9, 0x26, 0xc6, 0x52, 0xa4, 0xe7, 0x81, 0x69, 0xe1, 0xe7, 0x9e, 0x30, 0x01, 0xec, 0x84, 0x89, 0xb2, 0x0d, 0x66, + 0xdd, 0xce, 0x28, 0x5c, 0xec, 0x98, 0x46, 0x68, 0x21, 0x9f, 0x88, 0x3f, 0x1f, 0x42, 0x77, 0xce, 0xd0, 0x61, 0xd4, + 0x20, 0xa7, 0xff, 0x53, 0xad, 0x37, 0xd0, 0x17, 0x35, 0xc9, 0xfc, 0xba, 0x0a, 0x78, 0x3f, 0xf2, 0xcc, 0x86, 0x89, + 0xe8, 0x4b, 0x3c, 0x48, 0x33, 0x09, 0x7f, 0xc6, 0xc0, 0xdd, 0xb8, 0xfd, 0x7a, 0x66, 0x66, 0x65, 0xeb, 0x47, 0xa7, + 0x04, 0x28, 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, 0x70, 0xcf, 0xd6 +}; +static CONST UINT8 DecOutput012[] = { + 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, 0x42, 0x34, 0x8a, + 0xa1, 0x03, 0xb7, 0xe9, 0x57, 0x46, 0xbe, 0x20, 0xe4, 0x6e, 0xb0, 0xeb, 0xff, 0xea, 0x07, 0x7e, 0xef, 0xe2, 0x55, + 0x9f, 0xe5, 0x78, 0x3a, 0xb7, 0x83, 0xc2, 0x18, 0x40, 0x7b, 0xeb, 0xcd, 0x81, 0xfb, 0x90, 0x12, 0x9e, 0x46, 0xa9, + 0xd6, 0x4a, 0xba, 0xb0, 0x62, 0xdb, 0x6b, 0x99, 0xc4, 0xdb, 0x54, 0x4b, 0xb8, 0xa5, 0x71, 0xcb, 0xcd, 0x63, 0x32, + 0x55, 0xfb, 0x31, 0xf0, 0x38, 0xf5, 0xbe, 0x78, 0xe4, 0x45, 0xce, 0x1b, 0x6a, 0x5b, 0x0e, 0xf4, 0x16, 0xe4, 0xb1, + 0x3d, 0xf6, 0x63, 0x7b, 0xa7, 0x0c, 0xde, 0x6f, 0x8f, 0x74, 0xdf, 0xe0, 0x1e, 0x9d, 0xce, 0x8f, 0x24, 0xef, 0x23, + 0x35, 0x33, 0x7b, 0x83, 0x34, 0x23, 0x58, 0x74, 0x14, 0x77, 0x1f, 0xc2, 0x4f, 0x4e, 0xc6, 0x89, 0xf9, 0x52, 0x09, + 0x37, 0x64, 0x14, 0xc4, 0x01, 0x6b, 0x9d, 0x77, 0xe8, 0x90, 0x5d, 0xa8, 0x4a, 0x2a, 0xef, 0x5c, 0x7f, 0xeb, 0xbb, + 0xb2, 0xc6, 0x93, 0x99, 0x66, 0xdc, 0x7f, 0xd4, 0x9e, 0x2a, 0xca, 0x8d, 0xdb, 0xe7, 0x20, 0xcf, 0xe4, 0x73, 0xae, + 0x49, 0x7d, 0x64, 0x0f, 0x0e, 0x28, 0x46, 0xa9, 0xa8, 0x32, 0xe4, 0x0e, 0xf6, 0x51, 0x53, 0xb8, 0x3c, 0xb1, 0xff, + 0xa3, 0x33, 0x41, 0x75, 0xff, 0xf1, 0x6f, 0xf1, 0xfb, 0xbb, 0x83, 0x7f, 0x06, 0x9b, 0xe7, 0x1b, 0x0a, 0xe0, 0x5c, + 0x33, 0x60, 0x5b, 0xdb, 0x5b, 0xed, 0xfe, 0xa5, 0x16, 0x19, 0x72, 0xa3, 0x64, 0x23, 0x00, 0x02, 0xc7, 0xf3, 0x6a, + 0x81, 0x3e, 0x44, 0x1d, 0x79, 0x15, 0x5f, 0x9a, 0xde, 0xe2, 0xfd, 0x1b, 0x73, 0xc1, 0xbc, 0x23, 0xba, 0x31, 0xd2, + 0x50, 0xd5, 0xad, 0x7f, 0x74, 0xa7, 0xc9, 0xf8, 0x3e, 0x2b, 0x26, 0x10, 0xf6, 0x03, 0x36, 0x74, 0xe4, 0x0e, 0x6a, + 0x72, 0xb7, 0x73, 0x0a, 0x42, 0x28, 0xc2, 0xad, 0x5e, 0x03, 0xbe, 0xb8, 0x0b, 0xa8, 0x5b, 0xd4, 0xb8, 0xba, 0x52, + 0x89, 0xb1, 0x9b, 0xc1, 0xc3, 0x65, 0x87, 0xed, 0xa5, 0xf4, 0x86, 0xfd, 0x41, 0x80, 0x91, 0x27, 0x59, 0x53, 0x67, + 0x15, 0x78, 0x54, 0x8b, 0x2d, 0x3d, 0xc7, 0xff, 0x02, 0x92, 0x07, 0x5f, 0x7a, 0x4b, 0x60, 0x59, 0x3c, 0x6f, 0x5c, + 0xd8, 0xec, 0x95, 0xd2, 0xfe, 0xa0, 0x3b, 0xd8, 0x3f, 0xd1, 0x69, 0xa6, 0xd6, 0x41, 0xb2, 0xf4, 0x4d, 0x12, 0xf4, + 0x58, 0x3e, 0x66, 0x64, 0x80, 0x31, 0x9b, 0xa8, 0x4c, 0x8b, 0x07, 0xb2, 0xec, 0x66, 0x94, 0x66, 0x47, 0x50, 0x50, + 0x5f, 0x18, 0x0b, 0x0e, 0xd6, 0xc0, 0x39, 0x21, 0x13, 0x9e, 0x33, 0xbc, 0x79, 0x36, 0x02, 0x96, 0x70, 0xf0, 0x48, + 0x67, 0x2f, 0x26, 0xe9, 0x6d, 0x10, 0xbb, 0xd6, 0x3f, 0xd1, 0x64, 0x7a, 0x2e, 0xbe, 0x0c, 0x61, 0xf0, 0x75, 0x42, + 0x38, 0x23, 0xb1, 0x9e, 0x9f, 0x7c, 0x67, 0x66, 0xd9, 0x58, 0x9a, 0xf1, 0xbb, 0x41, 0x2a, 0x8d, 0x65, 0x84, 0x94, + 0xfc, 0xdc, 0x6a, 0x50, 0x64, 0xdb, 0x56, 0x33, 0x76, 0x00, 0x10, 0xed, 0xbe, 0xd2, 0x12, 0xf6, 0xf6, 0x1b, 0xa2, + 0x16, 0xde, 0xae, 0x31, 0x95, 0xdd, 0xb1, 0x08, 0x7e, 0x4e, 0xee, 0xe7, 0xf9, 0xa5, 0xfb, 0x5b, 0x61, 0x43, 0x00, + 0x40, 0xf6, 0x7e, 0x02, 0x04, 0x32, 0x4e, 0x0c, 0xe2, 0x66, 0x0d, 0xd7, 0x07, 0x98, 0x0e, 0xf8, 0x72, 0x34, 0x6d, + 0x95, 0x86, 0xd7, 0xcb, 0x31, 0x54, 0x47, 0xd0, 0x38, 0x29, 0x9c, 0x5a, 0x68, 0xd4, 0x87, 0x76, 0xc9, 0xe7, 0x7e, + 0xe3, 0xf4, 0x81, 0x6d, 0x18, 0xcb, 0xc9, 0x05, 0xaf, 0xa0, 0xfb, 0x66, 0xf7, 0xf1, 0x1c, 0xc6, 0x14, 0x11, 0x4f, + 0x2b, 0x79, 0x42, 0x8b, 0xbc, 0xac, 0xe7, 0x6c, 0xfe, 0x0f, 0x58, 0xe7, 0x7c, 0x78, 0x39, 0x30, 0xb0, 0x66, 0x2c, + 0x9b, 0x6d, 0x3a, 0xe1, 0xcf, 0xc9, 0xa4, 0x0e, 0x6d, 0x6d, 0x8a, 0xa1, 0x3a, 0xe7, 0x28, 0xd4, 0x78, 0x4c, 0xa6, + 0xa2, 0x2a, 0xa6, 0x03, 0x30, 0xd7, 0xa8, 0x25, 0x66, 0x87, 0x2f, 0x69, 0x5c, 0x4e, 0xdd, 0xa5, 0x49, 0x5d, 0x37, + 0x4a, 0x59, 0xc4, 0xaf, 0x1f, 0xa2, 0xe4, 0xf8, 0xa6, 0x12, 0x97, 0xd5, 0x79, 0xf5, 0xe2, 0x4a, 0x2b, 0x5f, 0x61, + 0xe4, 0x9e, 0xe3, 0xee, 0xb8, 0xa7, 0x5b, 0x2f, 0xf4, 0x9e, 0x6c, 0xfb, 0xd1, 0xc6, 0x56, 0x77, 0xba, 0x75, 0xaa, + 0x3d, 0x1a, 0xa8, 0x0b, 0xb3, 0x68, 0x24, 0x00, 0x10, 0x7f, 0xfd, 0xd7, 0xa1, 0x8d, 0x83, 0x54, 0x4f, 0x1f, 0xd8, + 0x2a, 0xbe, 0x8a, 0x0c, 0x87, 0xab, 0xa2, 0xde, 0xc3, 0x39, 0xbf, 0x09, 0x03, 0xa5, 0xf3, 0x05, 0x28, 0xe1, 0xe1, + 0xee, 0x39, 0x70, 0x9c, 0xd8, 0x81, 0x12, 0x1e, 0x02, 0x40, 0xd2, 0x6e, 0xf0, 0xeb, 0x1b, 0x3d, 0x22, 0xc6, 0xe5, + 0xe3, 0xb4, 0x5a, 0x98, 0xbb, 0xf0, 0x22, 0x28, 0x8d, 0xe5, 0xd3, 0x16, 0x48, 0x24, 0xa5, 0xe6, 0x66, 0x0c, 0xf9, + 0x08, 0xf9, 0x7e, 0x1e, 0xe1, 0x28, 0x26, 0x22, 0xc7, 0xc7, 0x0a, 0x32, 0x47, 0xfa, 0xa3, 0xbe, 0x3c, 0xc4, 0xc5, + 0x53, 0x0a, 0xd5, 0x94, 0x4a, 0xd7, 0x93, 0xd8, 0x42, 0x99, 0xb9, 0x0a, 0xdb, 0x56, 0xf7, 0xb9, 0x1c, 0x53, 0x4f, + 0xfa, 0xd3, 0x74, 0xad, 0xd9, 0x68, 0xf1, 0x1b, 0xdf, 0x61, 0xc6, 0x5e, 0xa8, 0x48, 0xfc, 0xd4, 0x4a, 0x4c, 0x3c, + 0x32, 0xf7, 0x1c, 0x96, 0x21, 0x9b, 0xf9, 0xa3, 0xcc, 0x5a, 0xce, 0xd5, 0xd7, 0x08, 0x24, 0xf6, 0x1c, 0xfd, 0xdd, + 0x38, 0xc2, 0x32, 0xe9, 0xb8, 0xe7, 0xb6, 0xfa, 0x9d, 0x45, 0x13, 0x2c, 0x83, 0xfd, 0x4a, 0x69, 0x82, 0xcd, 0xdc, + 0xb3, 0x76, 0x0c, 0x9e, 0xd8, 0xf4, 0x1b, 0x45, 0x15, 0xb4, 0x97, 0xe7, 0x58, 0x34, 0xe2, 0x03, 0x29, 0x5a, 0xbf, + 0xb6, 0xe0, 0x5d, 0x13, 0xd9, 0x2b, 0xb4, 0x80, 0xb2, 0x45, 0x81, 0x6a, 0x2e, 0x6c, 0x89, 0x7d, 0xee, 0xbb, 0x52, + 0xdd, 0x1f, 0x18, 0xe7, 0x13, 0x6b, 0x33, 0x0e, 0xea, 0x36, 0x92, 0x77, 0x7b, 0x6d, 0x9c, 0x5a, 0x5f, 0x45, 0x7b, + 0x7b, 0x35, 0x62, 0x23, 0xd1, 0xbf, 0x0f, 0xd0, 0x08, 0x1b, 0x2b, 0x80, 0x6b, 0x7e, 0xf1, 0x21, 0x47, 0xb0, 0x57, + 0xd1, 0x98, 0x72, 0x90, 0x34, 0x1c, 0x20, 0x04, 0xff, 0x3d, 0x5c, 0xee, 0x0e, 0x57, 0x5f, 0x6f, 0x24, 0x4e, 0x3c, + 0xea, 0xfc, 0xa5, 0xa9, 0x83, 0xc9, 0x61, 0xb4, 0x51, 0x24, 0xf8, 0x27, 0x5e, 0x46, 0x8c, 0xb1, 0x53, 0x02, 0x96, + 0x35, 0xba, 0xb8, 0x4c, 0x71, 0xd3, 0x15, 0x59, 0x35, 0x22, 0x20, 0xad, 0x03, 0x9f, 0x66, 0x44, 0x3b, 0x9c, 0x35, + 0x37, 0x1f, 0x9b, 0xbb, 0xf3, 0xdb, 0x35, 0x63, 0x30, 0x64, 0xaa, 0xa2, 0x06, 0xa8, 0x5d, 0xbb, 0xe1, 0x9f, 0x70, + 0xec, 0x82, 0x11, 0x06, 0x36, 0xec, 0x8b, 0x69, 0x66, 0x24, 0x44, 0xc9, 0x4a, 0x57, 0xbb, 0x9b, 0x78, 0x13, 0xce, + 0x9c, 0x0c, 0xba, 0x92, 0x93, 0x63, 0xb8, 0xe2, 0x95, 0x0f, 0x0f, 0x16, 0x39, 0x52, 0xfd, 0x3a, 0x6d, 0x02, 0x4b, + 0xdf, 0x13, 0xd3, 0x2a, 0x22, 0xb4, 0x03, 0x7c, 0x54, 0x49, 0x96, 0x68, 0x54, 0x10, 0xfa, 0xef, 0xaa, 0x6c, 0xe8, + 0x22, 0xdc, 0x71, 0x16, 0x13, 0x1a, 0xf6, 0x28, 0xe5, 0x6d, 0x77, 0x3d, 0xcd, 0x30, 0x63, 0xb1, 0x70, 0x52, 0xa1, + 0xc5, 0x94, 0x5f, 0xcf, 0xe8, 0xb8, 0x26, 0x98, 0xf7, 0x06, 0xa0, 0x0a, 0x70, 0xfa, 0x03, 0x80, 0xac, 0xc1, 0xec, + 0xd6, 0x4c, 0x54, 0xd7, 0xfe, 0x47, 0xb6, 0x88, 0x4a, 0xf7, 0x71, 0x24, 0xee, 0xf3, 0xd2, 0xc2, 0x4a, 0x7f, 0xfe, + 0x61, 0xc7, 0x35, 0xc9, 0x37, 0x67, 0xcb, 0x24, 0x35, 0xda, 0x7e, 0xca, 0x5f, 0xf3, 0x8d, 0xd4, 0x13, 0x8e, 0xd6, + 0xcb, 0x4d, 0x53, 0x8f, 0x53, 0x1f, 0xc0, 0x74, 0xf7, 0x53, 0xb9, 0x5e, 0x23, 0x37, 0xba, 0x6e, 0xe3, 0x9d, 0x07, + 0x55, 0x25, 0x7b, 0xe6, 0x2a, 0x64, 0xd1, 0x32, 0xdd, 0x54, 0x1b, 0x4b, 0xc0, 0xe1, 0xd7, 0x69, 0x58, 0xf8, 0x93, + 0x29, 0xc4, 0xdd, 0x23, 0x2f, 0xa5, 0xfc, 0x9d, 0x7e, 0xf8, 0xd4, 0x90, 0xcd, 0x82, 0x55, 0xdc, 0x16, 0x16, 0x9f, + 0x07, 0x52, 0x9b, 0x9d, 0x25, 0xed, 0x32, 0xc5, 0x7b, 0xdf, 0xf6, 0x83, 0x46, 0x3d, 0x65, 0xb7, 0xef, 0x87, 0x7a, + 0x12, 0x69, 0x8f, 0x06, 0x7c, 0x51, 0x15, 0x4a, 0x08, 0xe8, 0xac, 0x9a, 0x0c, 0x24, 0xa7, 0x27, 0xd8, 0x46, 0x2f, + 0xe7, 0x01, 0x0e, 0x1c, 0xc6, 0x91, 0xb0, 0x6e, 0x85, 0x65, 0xf0, 0x29, 0x0d, 0x2e, 0x6b, 0x3b, 0xfb, 0x4b, 0xdf, + 0xe4, 0x80, 0x93, 0x03, 0x66, 0x46, 0x3e, 0x8a, 0x6e, 0xf3, 0x5e, 0x4d, 0x62, 0x0e, 0x49, 0x05, 0xaf, 0xd4, 0xf8, + 0x21, 0x20, 0x61, 0x1d, 0x39, 0x17, 0xf4, 0x61, 0x47, 0x95, 0xfb, 0x15, 0x2e, 0xb3, 0x4f, 0xd0, 0x5d, 0xf5, 0x7d, + 0x40, 0xda, 0x90, 0x3c, 0x6b, 0xcb, 0x17, 0x00, 0x13, 0x3b, 0x64, 0x34, 0x1b, 0xf0, 0xf2, 0xe5, 0x3b, 0xb2, 0xc7, + 0xd3, 0x5f, 0x3a, 0x44, 0xa6, 0x9b, 0xb7, 0x78, 0x0e, 0x42, 0x5d, 0x4c, 0xc1, 0xe9, 0xd2, 0xcb, 0xb7, 0x78, 0xd1, + 0xfe, 0x9a, 0xb5, 0x07, 0xe9, 0xe0, 0xbe, 0xe2, 0x8a, 0xa7, 0x01, 0x83, 0x00, 0x8c, 0x5c, 0x08, 0xe6, 0x63, 0x12, + 0x92, 0xb7, 0xb7, 0xa6, 0x19, 0x7d, 0x38, 0x13, 0x38, 0x92, 0x87, 0x24, 0xf9, 0x48, 0xb3, 0x5e, 0x87, 0x6a, 0x40, + 0x39, 0x5c, 0x3f, 0xed, 0x8f, 0xee, 0xdb, 0x15, 0x82, 0x06, 0xda, 0x49, 0x21, 0x2b, 0xb5, 0xbf, 0x32, 0x7c, 0x9f, + 0x42, 0x28, 0x63, 0xcf, 0xaf, 0x1e, 0xf8, 0xc6, 0xa0, 0xd1, 0x02, 0x43, 0x57, 0x62, 0xec, 0x9b, 0x0f, 0x01, 0x9e, + 0x71, 0xd8, 0x87, 0x9d, 0x01, 0xc1, 0x58, 0x77, 0xd9, 0xaf, 0xb1, 0x10, 0x7e, 0xdd, 0xa6, 0x50, 0x96, 0xe5, 0xf0, + 0x72, 0x00, 0x6d, 0x4b, 0xf8, 0x2a, 0x8f, 0x19, 0xf3, 0x22, 0x88, 0x11, 0x4a, 0x8b, 0x7c, 0xfd, 0xb7, 0xed, 0xe1, + 0xf6, 0x40, 0x39, 0xe0, 0xe9, 0xf6, 0x3d, 0x25, 0xe6, 0x74, 0x3c, 0x58, 0x57, 0x7f, 0xe1, 0x22, 0x96, 0x47, 0x31, + 0x91, 0xba, 0x70, 0x85, 0x28, 0x6b, 0x9f, 0x6e, 0x25, 0xac, 0x23, 0x66, 0x2f, 0x29, 0x88, 0x28, 0xce, 0x8c, 0x5c, + 0x88, 0x53, 0xd1, 0x3b, 0xcc, 0x6a, 0x51, 0xb2, 0xe1, 0x28, 0x3f, 0x91, 0xb4, 0x0d, 0x00, 0x3a, 0xe3, 0xf8, 0xc3, + 0x8f, 0xd7, 0x96, 0x62, 0x0e, 0x2e, 0xfc, 0xc8, 0x6c, 0x77, 0xa6, 0x1d, 0x22, 0xc1, 0xb8, 0xe6, 0x61, 0xd7, 0x67, + 0x36, 0x13, 0x7b, 0xbb, 0x9b, 0x59, 0x09, 0xa6, 0xdf, 0xf7, 0x6b, 0xa3, 0x40, 0x1a, 0xf5, 0x4f, 0xb4, 0xda, 0xd3, + 0xf3, 0x81, 0x93, 0xc6, 0x18, 0xd9, 0x26, 0xee, 0xac, 0xf0, 0xaa, 0xdf, 0xc5, 0x9c, 0xca, 0xc2, 0xa2, 0xcc, 0x7b, + 0x5c, 0x24, 0xb0, 0xbc, 0xd0, 0x6a, 0x4d, 0x89, 0x09, 0xb8, 0x07, 0xfe, 0x87, 0xad, 0x0a, 0xea, 0xb8, 0x42, 0xf9, + 0x5e, 0xb3, 0x3e, 0x36, 0x4c, 0xaf, 0x75, 0x9e, 0x1c, 0xeb, 0xbd, 0xbc, 0xbb, 0x80, 0x40, 0xa7, 0x3a, 0x30, 0xbf, + 0xa8, 0x44, 0xf4, 0xeb, 0x38, 0xad, 0x29, 0xba, 0x23, 0xed, 0x41, 0x0c, 0xea, 0xd2, 0xbb, 0x41, 0x18, 0xd6, 0xb9, + 0xba, 0x65, 0x2b, 0xa3, 0x91, 0x6d, 0x1f, 0xa9, 0xf4, 0xd1, 0x25, 0x8d, 0x4d, 0x38, 0xff, 0x64, 0xa0, 0xec, 0xde, + 0xa6, 0xb6, 0x79, 0xab, 0x8e, 0x33, 0x6c, 0x47, 0xde, 0xaf, 0x94, 0xa4, 0xa5, 0x86, 0x77, 0x55, 0x09, 0x92, 0x81, + 0x31, 0x76, 0xc7, 0x34, 0x22, 0x89, 0x8e, 0x3d, 0x26, 0x26, 0xd7, 0xfc, 0x1e, 0x16, 0x72, 0x13, 0x33, 0x63, 0xd5, + 0x22, 0xbe, 0xb8, 0x04, 0x34, 0x84, 0x41, 0xbb, 0x80, 0xd0, 0x9f, 0x46, 0x48, 0x07, 0xa7, 0xfc, 0x2b, 0x3a, 0x75, + 0x55, 0x8c, 0xc7, 0x6a, 0xbd, 0x7e, 0x46, 0x08, 0x84, 0x0f, 0xd5, 0x74, 0xc0, 0x82, 0x8e, 0xaa, 0x61, 0x05, 0x01, + 0xb2, 0x47, 0x6e, 0x20, 0x6a, 0x2d, 0x58, 0x70, 0x48, 0x32, 0xa7, 0x37, 0xd2, 0xb8, 0x82, 0x1a, 0x51, 0xb9, 0x61, + 0xdd, 0xfd, 0x9d, 0x6b, 0x0e, 0x18, 0x97, 0xf8, 0x45, 0x5f, 0x87, 0x10, 0xcf, 0x34, 0x72, 0x45, 0x26, 0x49, 0x70, + 0xe7, 0xa3, 0x78, 0xe0, 0x52, 0x89, 0x84, 0x94, 0x83, 0x82, 0xc2, 0x69, 0x8f, 0xe3, 0xe1, 0x3f, 0x60, 0x74, 0x88, + 0xc4, 0xf7, 0x75, 0x2c, 0xfb, 0xbd, 0xb6, 0xc4, 0x7e, 0x10, 0x0a, 0x6c, 0x90, 0x04, 0x9e, 0xc3, 0x3f, 0x59, 0x7c, + 0xce, 0x31, 0x18, 0x60, 0x57, 0x73, 0x46, 0x94, 0x7d, 0x06, 0xa0, 0x6d, 0x44, 0xec, 0xa2, 0x0a, 0x9e, 0x05, 0x15, + 0xef, 0xca, 0x5c, 0xbf, 0x00, 0xeb, 0xf7, 0x3d, 0x32, 0xd4, 0xa5, 0xef, 0x49, 0x89, 0x5e, 0x46, 0xb0, 0xa6, 0x63, + 0x5b, 0x8a, 0x73, 0xae, 0x6f, 0xd5, 0x9d, 0xf8, 0x4f, 0x40, 0xb5, 0xb2, 0x6e, 0xd3, 0xb6, 0x01, 0xa9, 0x26, 0xa2, + 0x21, 0xcf, 0x33, 0x7a, 0x3a, 0xa4, 0x23, 0x13, 0xb0, 0x69, 0x6a, 0xee, 0xce, 0xd8, 0x9d, 0x01, 0x1d, 0x50, 0xc1, + 0x30, 0x6c, 0xb1, 0xcd, 0xa0, 0xf0, 0xf0, 0xa2, 0x64, 0x6f, 0xbb, 0xbf, 0x5e, 0xe6, 0xab, 0x87, 0xb4, 0x0f, 0x4f, + 0x15, 0xaf, 0xb5, 0x25, 0xa1, 0xb2, 0xd0, 0x80, 0x2c, 0xfb, 0xf9, 0xfe, 0xd2, 0x33, 0xbb, 0x76, 0xfe, 0x7c, 0xa8, + 0x66, 0xf7, 0xe7, 0x85, 0x9f, 0x1f, 0x85, 0x57, 0x88, 0xe1, 0xe9, 0x63, 0xe4, 0xd8, 0x1c, 0xa1, 0xfb, 0xda, 0x44, + 0x05, 0x2e, 0x1d, 0x3a, 0x1c, 0xff, 0xc8, 0x3b, 0xc0, 0xfe, 0xda, 0x22, 0x0b, 0x43, 0xd6, 0x88, 0x39, 0x4c, 0x4a, + 0xa6, 0x69, 0x18, 0x93, 0x42, 0x4e, 0xb5, 0xcc, 0x66, 0x0d, 0x09, 0xf8, 0x1e, 0x7c, 0xd3, 0x3c, 0x99, 0x0d, 0x50, + 0x1d, 0x62, 0xe9, 0x57, 0x06, 0xbf, 0x19, 0x88, 0xdd, 0xad, 0x7b, 0x4f, 0xf9, 0xc7, 0x82, 0x6d, 0x8d, 0xc8, 0xc4, + 0xc5, 0x78, 0x17, 0x20, 0x15, 0xc5, 0x52, 0x41, 0xcf, 0x5b, 0xd6, 0x7f, 0x94, 0x02, 0x41, 0xe0, 0x40, 0x22, 0x03, + 0x5e, 0xd1, 0x53, 0xd4, 0x86, 0xd3, 0x2c, 0x9f, 0x0f, 0x96, 0xe3, 0x6b, 0x9a, 0x76, 0x32, 0x06, 0x47, 0x4b, 0x11, + 0xb3, 0xdd, 0x03, 0x65, 0xbd, 0x9b, 0x01, 0xda, 0x9c, 0xb9, 0x7e, 0x3f, 0x6a, 0xc4, 0x7b, 0xea, 0xd4, 0x3c, 0xb9, + 0xfb, 0x5c, 0x6b, 0x64, 0x33, 0x52, 0xba, 0x64, 0x78, 0x8f, 0xa4, 0xaf, 0x7a, 0x61, 0x8d, 0xbc, 0xc5, 0x73, 0xe9, + 0x6b, 0x58, 0x97, 0x4b, 0xbf, 0x63, 0x22, 0xd3, 0x37, 0x02, 0x54, 0xc5, 0xb9, 0x16, 0x4a, 0xf0, 0x19, 0xd8, 0x94, + 0x57, 0xb8, 0x8a, 0xb3, 0x16, 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, 0x78, 0xec, 0x00 +}; +static CONST UINT8 DecAssoc012[] = { 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, 0x77, 0xe8, 0x21, 0xff, 0x06, + 0x59, 0x35, 0xce, 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, 0xcf, 0x70, + 0xdd, 0x0a, 0x68, 0xbf, 0xd4, 0xbc, 0x16, 0x76, 0x99, 0x36, 0x1e, 0x58, 0x79, + 0x5e, 0xd4, 0x29, 0xf7, 0x33, 0x93, 0x48, 0xdb, 0x5f, 0x01, 0xae, 0x9c, 0xb6, + 0xe4, 0x88, 0x6d, 0x2b, 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 }; +static CONST UINT8 DecNonce012[] = { 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 }; +static CONST UINT8 DecKey012[] = { 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, 0x8f, 0x59, 0x8e, + 0xc5, 0x90, 0xd5, 0x27, 0x2d, 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, + 0x44, 0x1e, 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 }; + +static CONST UINT8 DecInput013[] = { + 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, 0xdf, 0x34, 0x56, + 0x82, 0xe2, 0xbe, 0xe5, 0xe1, 0x28, 0xd1, 0x4e, 0x5f, 0x4f, 0x01, 0x7d, 0x3f, 0x99, 0x6b, 0x30, 0x6e, 0x1a, 0x7c, + 0x4c, 0x8e, 0x62, 0x81, 0xae, 0x86, 0x3f, 0x6b, 0xd0, 0xb5, 0xa9, 0xcf, 0x50, 0xf1, 0x02, 0x12, 0xa0, 0x0b, 0x24, + 0xe9, 0xe6, 0x72, 0x89, 0x2c, 0x52, 0x1b, 0x34, 0x38, 0xf8, 0x75, 0x5f, 0xa0, 0x74, 0xe2, 0x99, 0xdd, 0xa6, 0x4b, + 0x14, 0x50, 0x4e, 0xf1, 0xbe, 0xd6, 0x9e, 0xdb, 0xb2, 0x24, 0x27, 0x74, 0x12, 0x4a, 0x78, 0x78, 0x17, 0xa5, 0x58, + 0x8e, 0x2f, 0xf9, 0xf4, 0x8d, 0xee, 0x03, 0x88, 0xae, 0xb8, 0x29, 0xa1, 0x2f, 0x4b, 0xee, 0x92, 0xbd, 0x87, 0xb3, + 0xce, 0x34, 0x21, 0x57, 0x46, 0x04, 0x49, 0x0c, 0x80, 0xf2, 0x01, 0x13, 0xa1, 0x55, 0xb3, 0xff, 0x44, 0x30, 0x3c, + 0x1c, 0xd0, 0xef, 0xbc, 0x18, 0x74, 0x26, 0xad, 0x41, 0x5b, 0x5b, 0x3e, 0x9a, 0x7a, 0x46, 0x4f, 0x16, 0xd6, 0x74, + 0x5a, 0xb7, 0x3a, 0x28, 0x31, 0xd8, 0xae, 0x26, 0xac, 0x50, 0x53, 0x86, 0xf2, 0x56, 0xd7, 0x3f, 0x29, 0xbc, 0x45, + 0x68, 0x8e, 0xcb, 0x98, 0x64, 0xdd, 0xc9, 0xba, 0xb8, 0x4b, 0x7b, 0x82, 0xdd, 0x14, 0xa7, 0xcb, 0x71, 0x72, 0x00, + 0x5c, 0xad, 0x7b, 0x6a, 0x89, 0xa4, 0x3d, 0xbf, 0xb5, 0x4b, 0x3e, 0x7c, 0x5a, 0xcf, 0xb8, 0xa1, 0xc5, 0x6e, 0xc8, + 0xb6, 0x31, 0x57, 0x7b, 0xdf, 0xa5, 0x7e, 0xb1, 0xd6, 0x42, 0x2a, 0x31, 0x36, 0xd1, 0xd0, 0x3f, 0x7a, 0xe5, 0x94, + 0xd6, 0x36, 0xa0, 0x6f, 0xb7, 0x40, 0x7d, 0x37, 0xc6, 0x55, 0x7c, 0x50, 0x40, 0x6d, 0x29, 0x89, 0xe3, 0x5a, 0xae, + 0x97, 0xe7, 0x44, 0x49, 0x6e, 0xbd, 0x81, 0x3d, 0x03, 0x93, 0x06, 0x12, 0x06, 0xe2, 0x41, 0x12, 0x4a, 0xf1, 0x6a, + 0xa4, 0x58, 0xa2, 0xfb, 0xd2, 0x15, 0xba, 0xc9, 0x79, 0xc9, 0xce, 0x5e, 0x13, 0xbb, 0xf1, 0x09, 0x04, 0xcc, 0xfd, + 0xe8, 0x51, 0x34, 0x6a, 0xe8, 0x61, 0x88, 0xda, 0xed, 0x01, 0x47, 0x84, 0xf5, 0x73, 0x25, 0xf9, 0x1c, 0x42, 0x86, + 0x07, 0xf3, 0x5b, 0x1a, 0x01, 0xb3, 0xeb, 0x24, 0x32, 0x8d, 0xf6, 0xed, 0x7c, 0x4b, 0xeb, 0x3c, 0x36, 0x42, 0x28, + 0xdf, 0xdf, 0xb6, 0xbe, 0xd9, 0x8c, 0x52, 0xd3, 0x2b, 0x08, 0x90, 0x8c, 0xe7, 0x98, 0x31, 0xe2, 0x32, 0x8e, 0xfc, + 0x11, 0x48, 0x00, 0xa8, 0x6a, 0x42, 0x4a, 0x02, 0xc6, 0x4b, 0x09, 0xf1, 0xe3, 0x49, 0xf3, 0x45, 0x1f, 0x0e, 0xbc, + 0x56, 0xe2, 0xe4, 0xdf, 0xfb, 0xeb, 0x61, 0xfa, 0x24, 0xc1, 0x63, 0x75, 0xbb, 0x47, 0x75, 0xaf, 0xe1, 0x53, 0x16, + 0x96, 0x21, 0x85, 0x26, 0x11, 0xb3, 0x76, 0xe3, 0x23, 0xa1, 0x6b, 0x74, 0x37, 0xd0, 0xde, 0x06, 0x90, 0x71, 0x5d, + 0x43, 0x88, 0x9b, 0x00, 0x54, 0xa6, 0x75, 0x2f, 0xa1, 0xc2, 0x0b, 0x73, 0x20, 0x1d, 0xb6, 0x21, 0x79, 0x57, 0x3f, + 0xfa, 0x09, 0xbe, 0x8a, 0x33, 0xc3, 0x52, 0xf0, 0x1d, 0x82, 0x31, 0xd1, 0x55, 0xb5, 0x6c, 0x99, 0x25, 0xcf, 0x5c, + 0x32, 0xce, 0xe9, 0x0d, 0xfa, 0x69, 0x2c, 0xd5, 0x0d, 0xc5, 0x6d, 0x86, 0xd0, 0x0c, 0x3b, 0x06, 0x50, 0x79, 0xe8, + 0xc3, 0xae, 0x04, 0xe6, 0xcd, 0x51, 0xe4, 0x26, 0x9b, 0x4f, 0x7e, 0xa6, 0x0f, 0xab, 0xd8, 0xe5, 0xde, 0xa9, 0x00, + 0x95, 0xbe, 0xa3, 0x9d, 0x5d, 0xb2, 0x09, 0x70, 0x18, 0x1c, 0xf0, 0xac, 0x29, 0x23, 0x02, 0x29, 0x28, 0xd2, 0x74, + 0x35, 0x57, 0x62, 0x0f, 0x24, 0xea, 0x5e, 0x33, 0xc2, 0x92, 0xf3, 0x78, 0x4d, 0x30, 0x1e, 0xa1, 0x99, 0xa9, 0x82, + 0xb0, 0x42, 0x31, 0x8d, 0xad, 0x8a, 0xbc, 0xfc, 0xd4, 0x57, 0x47, 0x3e, 0xb4, 0x50, 0xdd, 0x6e, 0x2c, 0x80, 0x4d, + 0x22, 0xf1, 0xfb, 0x57, 0xc4, 0xdd, 0x17, 0xe1, 0x8a, 0x36, 0x4a, 0xb3, 0x37, 0xca, 0xc9, 0x4e, 0xab, 0xd5, 0x69, + 0xc4, 0xf4, 0xbc, 0x0b, 0x3b, 0x44, 0x4b, 0x29, 0x9c, 0xee, 0xd4, 0x35, 0x22, 0x21, 0xb0, 0x1f, 0x27, 0x64, 0xa8, + 0x51, 0x1b, 0xf0, 0x9f, 0x19, 0x5c, 0xfb, 0x5a, 0x64, 0x74, 0x70, 0x45, 0x09, 0xf5, 0x64, 0xfe, 0x1a, 0x2d, 0xc9, + 0x14, 0x04, 0x14, 0xcf, 0xd5, 0x7d, 0x60, 0xaf, 0x94, 0x39, 0x94, 0xe2, 0x7d, 0x79, 0x82, 0xd0, 0x65, 0x3b, 0x6b, + 0x9c, 0x19, 0x84, 0xb4, 0x6d, 0xb3, 0x0c, 0x99, 0xc0, 0x56, 0xa8, 0xbd, 0x73, 0xce, 0x05, 0x84, 0x3e, 0x30, 0xaa, + 0xc4, 0x9b, 0x1b, 0x04, 0x2a, 0x9f, 0xd7, 0x43, 0x2b, 0x23, 0xdf, 0xbf, 0xaa, 0xd5, 0xc2, 0x43, 0x2d, 0x70, 0xab, + 0xdc, 0x75, 0xad, 0xac, 0xf7, 0xc0, 0xbe, 0x67, 0xb2, 0x74, 0xed, 0x67, 0x10, 0x4a, 0x92, 0x60, 0xc1, 0x40, 0x50, + 0x19, 0x8a, 0x8a, 0x8c, 0x09, 0x0e, 0x72, 0xe1, 0x73, 0x5e, 0xe8, 0x41, 0x85, 0x63, 0x9f, 0x3f, 0xd7, 0x7d, 0xc4, + 0xfb, 0x22, 0x5d, 0x92, 0x6c, 0xb3, 0x1e, 0xe2, 0x50, 0x2f, 0x82, 0xa8, 0x28, 0xc0, 0xb5, 0xd7, 0x5f, 0x68, 0x0d, + 0x2c, 0x2d, 0xaf, 0x7e, 0xfa, 0x2e, 0x08, 0x0f, 0x1f, 0x70, 0x9f, 0xe9, 0x19, 0x72, 0x55, 0xf8, 0xfb, 0x51, 0xd2, + 0x33, 0x5d, 0xa0, 0xd3, 0x2b, 0x0a, 0x6c, 0xbc, 0x4e, 0xcf, 0x36, 0x4d, 0xdc, 0x3b, 0xe9, 0x3e, 0x81, 0x7c, 0x61, + 0xdb, 0x20, 0x2d, 0x3a, 0xc3, 0xb3, 0x0c, 0x1e, 0x00, 0xb9, 0x7c, 0xf5, 0xca, 0x10, 0x5f, 0x3a, 0x71, 0xb3, 0xe4, + 0x20, 0xdb, 0x0c, 0x2a, 0x98, 0x63, 0x45, 0x00, 0x58, 0xf6, 0x68, 0xe4, 0x0b, 0xda, 0x13, 0x3b, 0x60, 0x5c, 0x76, + 0xdb, 0xb9, 0x97, 0x71, 0xe4, 0xd9, 0xb7, 0xdb, 0xbd, 0x68, 0xc7, 0x84, 0x84, 0xaa, 0x7c, 0x68, 0x62, 0x5e, 0x16, + 0xfc, 0xba, 0x72, 0xaa, 0x9a, 0xa9, 0xeb, 0x7c, 0x75, 0x47, 0x97, 0x7e, 0xad, 0xe2, 0xd9, 0x91, 0xe8, 0xe4, 0xa5, + 0x31, 0xd7, 0x01, 0x8e, 0xa2, 0x11, 0x88, 0x95, 0xb9, 0xf2, 0x9b, 0xd3, 0x7f, 0x1b, 0x81, 0x22, 0xf7, 0x98, 0x60, + 0x0a, 0x64, 0xa6, 0xc1, 0xf6, 0x49, 0xc7, 0xe3, 0x07, 0x4d, 0x94, 0x7a, 0xcf, 0x6e, 0x68, 0x0c, 0x1b, 0x3f, 0x6e, + 0x2e, 0xee, 0x92, 0xfa, 0x52, 0xb3, 0x59, 0xf8, 0xf1, 0x8f, 0x6a, 0x66, 0xa3, 0x82, 0x76, 0x4a, 0x07, 0x1a, 0xc7, + 0xdd, 0xf5, 0xda, 0x9c, 0x3c, 0x24, 0xbf, 0xfd, 0x42, 0xa1, 0x10, 0x64, 0x6a, 0x0f, 0x89, 0xee, 0x36, 0xa5, 0xce, + 0x99, 0x48, 0x6a, 0xf0, 0x9f, 0x9e, 0x69, 0xa4, 0x40, 0x20, 0xe9, 0x16, 0x15, 0xf7, 0xdb, 0x75, 0x02, 0xcb, 0xe9, + 0x73, 0x8b, 0x3b, 0x49, 0x2f, 0xf0, 0xaf, 0x51, 0x06, 0x5c, 0xdf, 0x27, 0x27, 0x49, 0x6a, 0xd1, 0xcc, 0xc7, 0xb5, + 0x63, 0xb5, 0xfc, 0xb8, 0x5c, 0x87, 0x7f, 0x84, 0xb4, 0xcc, 0x14, 0xa9, 0x53, 0xda, 0xa4, 0x56, 0xf8, 0xb6, 0x1b, + 0xcc, 0x40, 0x27, 0x52, 0x06, 0x5a, 0x13, 0x81, 0xd7, 0x3a, 0xd4, 0x3b, 0xfb, 0x49, 0x65, 0x31, 0x33, 0xb2, 0xfa, + 0xcd, 0xad, 0x58, 0x4e, 0x2b, 0xae, 0xd2, 0x20, 0xfb, 0x1a, 0x48, 0xb4, 0x3f, 0x9a, 0xd8, 0x7a, 0x35, 0x4a, 0xc8, + 0xee, 0x88, 0x5e, 0x07, 0x66, 0x54, 0xb9, 0xec, 0x9f, 0xa3, 0xe3, 0xb9, 0x37, 0xaa, 0x49, 0x76, 0x31, 0xda, 0x74, + 0x2d, 0x3c, 0xa4, 0x65, 0x10, 0x32, 0x38, 0xf0, 0xde, 0xd3, 0x99, 0x17, 0xaa, 0x71, 0xaa, 0x8f, 0x0f, 0x8c, 0xaf, + 0xa2, 0xf8, 0x5d, 0x64, 0xba, 0x1d, 0xa3, 0xef, 0x96, 0x73, 0xe8, 0xa1, 0x02, 0x8d, 0x0c, 0x6d, 0xb8, 0x06, 0x90, + 0xb8, 0x08, 0x56, 0x2c, 0xa7, 0x06, 0xc9, 0xc2, 0x38, 0xdb, 0x7c, 0x63, 0xb1, 0x57, 0x8e, 0xea, 0x7c, 0x79, 0xf3, + 0x49, 0x1d, 0xfe, 0x9f, 0xf3, 0x6e, 0xb1, 0x1d, 0xba, 0x19, 0x80, 0x1a, 0x0a, 0xd3, 0xb0, 0x26, 0x21, 0x40, 0xb1, + 0x7c, 0xf9, 0x4d, 0x8d, 0x10, 0xc1, 0x7e, 0xf4, 0xf6, 0x3c, 0xa8, 0xfd, 0x7c, 0xa3, 0x92, 0xb2, 0x0f, 0xaa, 0xcc, + 0xa6, 0x11, 0xfe, 0x04, 0xe3, 0xd1, 0x7a, 0x32, 0x89, 0xdf, 0x0d, 0xc4, 0x8f, 0x79, 0x6b, 0xca, 0x16, 0x7c, 0x6e, + 0xf9, 0xad, 0x0f, 0xf6, 0xfe, 0x27, 0xdb, 0xc4, 0x13, 0x70, 0xf1, 0x62, 0x1a, 0x4f, 0x79, 0x40, 0xc9, 0x9b, 0x8b, + 0x21, 0xea, 0x84, 0xfa, 0xf5, 0xf1, 0x89, 0xce, 0xb7, 0x55, 0x0a, 0x80, 0x39, 0x2f, 0x55, 0x36, 0x16, 0x9c, 0x7b, + 0x08, 0xbd, 0x87, 0x0d, 0xa5, 0x32, 0xf1, 0x52, 0x7c, 0xe8, 0x55, 0x60, 0x5b, 0xd7, 0x69, 0xe4, 0xfc, 0xfa, 0x12, + 0x85, 0x96, 0xea, 0x50, 0x28, 0xab, 0x8a, 0xf7, 0xbb, 0x0e, 0x53, 0x74, 0xca, 0xa6, 0x27, 0x09, 0xc2, 0xb5, 0xde, + 0x18, 0x14, 0xd9, 0xea, 0xe5, 0x29, 0x1c, 0x40, 0x56, 0xcf, 0xd7, 0xae, 0x05, 0x3f, 0x65, 0xaf, 0x05, 0x73, 0xe2, + 0x35, 0x96, 0x27, 0x07, 0x14, 0xc0, 0xad, 0x33, 0xf1, 0xdc, 0x44, 0x7a, 0x89, 0x17, 0x77, 0xd2, 0x9c, 0x58, 0x60, + 0xf0, 0x3f, 0x7b, 0x2d, 0x2e, 0x57, 0x95, 0x54, 0x87, 0xed, 0xf2, 0xc7, 0x4c, 0xf0, 0xae, 0x56, 0x29, 0x19, 0x7d, + 0x66, 0x4b, 0x9b, 0x83, 0x84, 0x42, 0x3b, 0x01, 0x25, 0x66, 0x8e, 0x02, 0xde, 0xb9, 0x83, 0x54, 0x19, 0xf6, 0x9f, + 0x79, 0x0d, 0x67, 0xc5, 0x1d, 0x7a, 0x44, 0x02, 0x98, 0xa7, 0x16, 0x1c, 0x29, 0x0d, 0x74, 0xff, 0x85, 0x40, 0x06, + 0xef, 0x2c, 0xa9, 0xc6, 0xf5, 0x53, 0x07, 0x06, 0xae, 0xe4, 0xfa, 0x5f, 0xd8, 0x39, 0x4d, 0xf1, 0x9b, 0x6b, 0xd9, + 0x24, 0x84, 0xfe, 0x03, 0x4c, 0xb2, 0x3f, 0xdf, 0xa1, 0x05, 0x9e, 0x50, 0x14, 0x5a, 0xd9, 0x1a, 0xa2, 0xa7, 0xfa, + 0xfa, 0x17, 0xf7, 0x78, 0xd6, 0xb5, 0x92, 0x61, 0x91, 0xac, 0x36, 0xfa, 0x56, 0x0d, 0x38, 0x32, 0x18, 0x85, 0x08, + 0x58, 0x37, 0xf0, 0x4b, 0xdb, 0x59, 0xe7, 0xa4, 0x34, 0xc0, 0x1b, 0x01, 0xaf, 0x2d, 0xde, 0xa1, 0xaa, 0x5d, 0xd3, + 0xec, 0xe1, 0xd4, 0xf7, 0xe6, 0x54, 0x68, 0xf0, 0x51, 0x97, 0xa7, 0x89, 0xea, 0x24, 0xad, 0xd3, 0x6e, 0x47, 0x93, + 0x8b, 0x4b, 0xb4, 0xf7, 0x1c, 0x42, 0x06, 0x67, 0xe8, 0x99, 0xf6, 0xf5, 0x7b, 0x85, 0xb5, 0x65, 0xb5, 0xb5, 0xd2, + 0x37, 0xf5, 0xf3, 0x02, 0xa6, 0x4d, 0x11, 0xa7, 0xdc, 0x51, 0x09, 0x7f, 0xa0, 0xd8, 0x88, 0x1c, 0x13, 0x71, 0xae, + 0x9c, 0xb7, 0x7b, 0x34, 0xd6, 0x4e, 0x68, 0x26, 0x83, 0x51, 0xaf, 0x1d, 0xee, 0x8b, 0xbb, 0x69, 0x43, 0x2b, 0x9e, + 0x8a, 0xbc, 0x02, 0x0e, 0xa0, 0x1b, 0xe0, 0xa8, 0x5f, 0x6f, 0xaf, 0x1b, 0x8f, 0xe7, 0x64, 0x71, 0x74, 0x11, 0x7e, + 0xa8, 0xd8, 0xf9, 0x97, 0x06, 0xc3, 0xb6, 0xfb, 0xfb, 0xb7, 0x3d, 0x35, 0x9d, 0x3b, 0x52, 0xed, 0x54, 0xca, 0xf4, + 0x81, 0x01, 0x2d, 0x1b, 0xc3, 0xa7, 0x00, 0x3d, 0x1a, 0x39, 0x54, 0xe1, 0xf6, 0xff, 0xed, 0x6f, 0x0b, 0x5a, 0x68, + 0xda, 0x58, 0xdd, 0xa9, 0xcf, 0x5c, 0x4a, 0xe5, 0x09, 0x4e, 0xde, 0x9d, 0xbc, 0x3e, 0xee, 0x5a, 0x00, 0x3b, 0x2c, + 0x87, 0x10, 0x65, 0x60, 0xdd, 0xd7, 0x56, 0xd1, 0x4c, 0x64, 0x45, 0xe4, 0x21, 0xec, 0x78, 0xf8, 0x25, 0x7a, 0x3e, + 0x16, 0x5d, 0x09, 0x53, 0x14, 0xbe, 0x4f, 0xae, 0x87, 0xd8, 0xd1, 0xaa, 0x3c, 0xf6, 0x3e, 0xa4, 0x70, 0x8c, 0x5e, + 0x70, 0xa4, 0xb3, 0x6b, 0x66, 0x73, 0xd3, 0xbf, 0x31, 0x06, 0x19, 0x62, 0x93, 0x15, 0xf2, 0x86, 0xe4, 0x52, 0x7e, + 0x53, 0x4c, 0x12, 0x38, 0xcc, 0x34, 0x7d, 0x57, 0xf6, 0x42, 0x93, 0x8a, 0xc4, 0xee, 0x5c, 0x8a, 0xe1, 0x52, 0x8f, + 0x56, 0x64, 0xf6, 0xa6, 0xd1, 0x91, 0x57, 0x70, 0xcd, 0x11, 0x76, 0xf5, 0x59, 0x60, 0x60, 0x3c, 0xc1, 0xc3, 0x0b, + 0x7f, 0x58, 0x1a, 0x50, 0x91, 0xf1, 0x68, 0x8f, 0x6e, 0x74, 0x74, 0xa8, 0x51, 0x0b, 0xf7, 0x7a, 0x98, 0x37, 0xf2, + 0x0a, 0x0e, 0xa4, 0x97, 0x04, 0xb8, 0x9b, 0xfd, 0xa0, 0xea, 0xf7, 0x0d, 0xe1, 0xdb, 0x03, 0xf0, 0x31, 0x29, 0xf8, + 0xdd, 0x6b, 0x8b, 0x5d, 0xd8, 0x59, 0xa9, 0x29, 0xcf, 0x9a, 0x79, 0x89, 0x19, 0x63, 0x46, 0x09, 0x79, 0x6a, 0x11, + 0xda, 0x63, 0x68, 0x48, 0x77, 0x23, 0xfb, 0x7d, 0x3a, 0x43, 0xcb, 0x02, 0x3b, 0x7a, 0x6d, 0x10, 0x2a, 0x9e, 0xac, + 0xf1, 0xd4, 0x19, 0xf8, 0x23, 0x64, 0x1d, 0x2c, 0x5f, 0xf2, 0xb0, 0x5c, 0x23, 0x27, 0xf7, 0x27, 0x30, 0x16, 0x37, + 0xb1, 0x90, 0xab, 0x38, 0xfb, 0x55, 0xcd, 0x78, 0x58, 0xd4, 0x7d, 0x43, 0xf6, 0x45, 0x5e, 0x55, 0x8d, 0xb1, 0x02, + 0x65, 0x58, 0xb4, 0x13, 0x4b, 0x36, 0xf7, 0xcc, 0xfe, 0x3d, 0x0b, 0x82, 0xe2, 0x12, 0x11, 0xbb, 0xe6, 0xb8, 0x3a, + 0x48, 0x71, 0xc7, 0x50, 0x06, 0x16, 0x3a, 0xe6, 0x7c, 0x05, 0xc7, 0xc8, 0x4d, 0x2f, 0x08, 0x6a, 0x17, 0x9a, 0x95, + 0x97, 0x50, 0x68, 0xdc, 0x28, 0x18, 0xc4, 0x61, 0x38, 0xb9, 0xe0, 0x3e, 0x78, 0xdb, 0x29, 0xe0, 0x9f, 0x52, 0xdd, + 0xf8, 0x4f, 0x91, 0xc1, 0xd0, 0x33, 0xa1, 0x7a, 0x8e, 0x30, 0x13, 0x82, 0x07, 0x9f, 0xd3, 0x31, 0x0f, 0x23, 0xbe, + 0x32, 0x5a, 0x75, 0xcf, 0x96, 0xb2, 0xec, 0xb5, 0x32, 0xac, 0x21, 0xd1, 0x82, 0x33, 0xd3, 0x15, 0x74, 0xbd, 0x90, + 0xf1, 0x2c, 0xe6, 0x5f, 0x8d, 0xe3, 0x02, 0xe8, 0xe9, 0xc4, 0xca, 0x96, 0xeb, 0x0e, 0xbc, 0x91, 0xf4, 0xb9, 0xea, + 0xd9, 0x1b, 0x75, 0xbd, 0xe1, 0xac, 0x2a, 0x05, 0x37, 0x52, 0x9b, 0x1b, 0x3f, 0x5a, 0xdc, 0x21, 0xc3, 0x98, 0xbb, + 0xaf, 0xa3, 0xf2, 0x00, 0xbf, 0x0d, 0x30, 0x89, 0x05, 0xcc, 0xa5, 0x76, 0xf5, 0x06, 0xf0, 0xc6, 0x54, 0x8a, 0x5d, + 0xd4, 0x1e, 0xc1, 0xf2, 0xce, 0xb0, 0x62, 0xc8, 0xfc, 0x59, 0x42, 0x9a, 0x90, 0x60, 0x55, 0xfe, 0x88, 0xa5, 0x8b, + 0xb8, 0x33, 0x0c, 0x23, 0x24, 0x0d, 0x15, 0x70, 0x37, 0x1e, 0x3d, 0xf6, 0xd2, 0xea, 0x92, 0x10, 0xb2, 0xc4, 0x51, + 0xac, 0xf2, 0xac, 0xf3, 0x6b, 0x6c, 0xaa, 0xcf, 0x12, 0xc5, 0x6c, 0x90, 0x50, 0xb5, 0x0c, 0xfc, 0x1a, 0x15, 0x52, + 0xe9, 0x26, 0xc6, 0x52, 0xa4, 0xe7, 0x81, 0x69, 0xe1, 0xe7, 0x9e, 0x30, 0x01, 0xec, 0x84, 0x89, 0xb2, 0x0d, 0x66, + 0xdd, 0xce, 0x28, 0x5c, 0xec, 0x98, 0x46, 0x68, 0x21, 0x9f, 0x88, 0x3f, 0x1f, 0x42, 0x77, 0xce, 0xd0, 0x61, 0xd4, + 0x20, 0xa7, 0xff, 0x53, 0xad, 0x37, 0xd0, 0x17, 0x35, 0xc9, 0xfc, 0xba, 0x0a, 0x78, 0x3f, 0xf2, 0xcc, 0x86, 0x89, + 0xe8, 0x4b, 0x3c, 0x48, 0x33, 0x09, 0x7f, 0xc6, 0xc0, 0xdd, 0xb8, 0xfd, 0x7a, 0x66, 0x66, 0x65, 0xeb, 0x47, 0xa7, + 0x04, 0x28, 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, 0x70, 0xcf, 0xd7 +}; +static CONST UINT8 DecOutput013[] = { + 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, 0x42, 0x34, 0x8a, + 0xa1, 0x03, 0xb7, 0xe9, 0x57, 0x46, 0xbe, 0x20, 0xe4, 0x6e, 0xb0, 0xeb, 0xff, 0xea, 0x07, 0x7e, 0xef, 0xe2, 0x55, + 0x9f, 0xe5, 0x78, 0x3a, 0xb7, 0x83, 0xc2, 0x18, 0x40, 0x7b, 0xeb, 0xcd, 0x81, 0xfb, 0x90, 0x12, 0x9e, 0x46, 0xa9, + 0xd6, 0x4a, 0xba, 0xb0, 0x62, 0xdb, 0x6b, 0x99, 0xc4, 0xdb, 0x54, 0x4b, 0xb8, 0xa5, 0x71, 0xcb, 0xcd, 0x63, 0x32, + 0x55, 0xfb, 0x31, 0xf0, 0x38, 0xf5, 0xbe, 0x78, 0xe4, 0x45, 0xce, 0x1b, 0x6a, 0x5b, 0x0e, 0xf4, 0x16, 0xe4, 0xb1, + 0x3d, 0xf6, 0x63, 0x7b, 0xa7, 0x0c, 0xde, 0x6f, 0x8f, 0x74, 0xdf, 0xe0, 0x1e, 0x9d, 0xce, 0x8f, 0x24, 0xef, 0x23, + 0x35, 0x33, 0x7b, 0x83, 0x34, 0x23, 0x58, 0x74, 0x14, 0x77, 0x1f, 0xc2, 0x4f, 0x4e, 0xc6, 0x89, 0xf9, 0x52, 0x09, + 0x37, 0x64, 0x14, 0xc4, 0x01, 0x6b, 0x9d, 0x77, 0xe8, 0x90, 0x5d, 0xa8, 0x4a, 0x2a, 0xef, 0x5c, 0x7f, 0xeb, 0xbb, + 0xb2, 0xc6, 0x93, 0x99, 0x66, 0xdc, 0x7f, 0xd4, 0x9e, 0x2a, 0xca, 0x8d, 0xdb, 0xe7, 0x20, 0xcf, 0xe4, 0x73, 0xae, + 0x49, 0x7d, 0x64, 0x0f, 0x0e, 0x28, 0x46, 0xa9, 0xa8, 0x32, 0xe4, 0x0e, 0xf6, 0x51, 0x53, 0xb8, 0x3c, 0xb1, 0xff, + 0xa3, 0x33, 0x41, 0x75, 0xff, 0xf1, 0x6f, 0xf1, 0xfb, 0xbb, 0x83, 0x7f, 0x06, 0x9b, 0xe7, 0x1b, 0x0a, 0xe0, 0x5c, + 0x33, 0x60, 0x5b, 0xdb, 0x5b, 0xed, 0xfe, 0xa5, 0x16, 0x19, 0x72, 0xa3, 0x64, 0x23, 0x00, 0x02, 0xc7, 0xf3, 0x6a, + 0x81, 0x3e, 0x44, 0x1d, 0x79, 0x15, 0x5f, 0x9a, 0xde, 0xe2, 0xfd, 0x1b, 0x73, 0xc1, 0xbc, 0x23, 0xba, 0x31, 0xd2, + 0x50, 0xd5, 0xad, 0x7f, 0x74, 0xa7, 0xc9, 0xf8, 0x3e, 0x2b, 0x26, 0x10, 0xf6, 0x03, 0x36, 0x74, 0xe4, 0x0e, 0x6a, + 0x72, 0xb7, 0x73, 0x0a, 0x42, 0x28, 0xc2, 0xad, 0x5e, 0x03, 0xbe, 0xb8, 0x0b, 0xa8, 0x5b, 0xd4, 0xb8, 0xba, 0x52, + 0x89, 0xb1, 0x9b, 0xc1, 0xc3, 0x65, 0x87, 0xed, 0xa5, 0xf4, 0x86, 0xfd, 0x41, 0x80, 0x91, 0x27, 0x59, 0x53, 0x67, + 0x15, 0x78, 0x54, 0x8b, 0x2d, 0x3d, 0xc7, 0xff, 0x02, 0x92, 0x07, 0x5f, 0x7a, 0x4b, 0x60, 0x59, 0x3c, 0x6f, 0x5c, + 0xd8, 0xec, 0x95, 0xd2, 0xfe, 0xa0, 0x3b, 0xd8, 0x3f, 0xd1, 0x69, 0xa6, 0xd6, 0x41, 0xb2, 0xf4, 0x4d, 0x12, 0xf4, + 0x58, 0x3e, 0x66, 0x64, 0x80, 0x31, 0x9b, 0xa8, 0x4c, 0x8b, 0x07, 0xb2, 0xec, 0x66, 0x94, 0x66, 0x47, 0x50, 0x50, + 0x5f, 0x18, 0x0b, 0x0e, 0xd6, 0xc0, 0x39, 0x21, 0x13, 0x9e, 0x33, 0xbc, 0x79, 0x36, 0x02, 0x96, 0x70, 0xf0, 0x48, + 0x67, 0x2f, 0x26, 0xe9, 0x6d, 0x10, 0xbb, 0xd6, 0x3f, 0xd1, 0x64, 0x7a, 0x2e, 0xbe, 0x0c, 0x61, 0xf0, 0x75, 0x42, + 0x38, 0x23, 0xb1, 0x9e, 0x9f, 0x7c, 0x67, 0x66, 0xd9, 0x58, 0x9a, 0xf1, 0xbb, 0x41, 0x2a, 0x8d, 0x65, 0x84, 0x94, + 0xfc, 0xdc, 0x6a, 0x50, 0x64, 0xdb, 0x56, 0x33, 0x76, 0x00, 0x10, 0xed, 0xbe, 0xd2, 0x12, 0xf6, 0xf6, 0x1b, 0xa2, + 0x16, 0xde, 0xae, 0x31, 0x95, 0xdd, 0xb1, 0x08, 0x7e, 0x4e, 0xee, 0xe7, 0xf9, 0xa5, 0xfb, 0x5b, 0x61, 0x43, 0x00, + 0x40, 0xf6, 0x7e, 0x02, 0x04, 0x32, 0x4e, 0x0c, 0xe2, 0x66, 0x0d, 0xd7, 0x07, 0x98, 0x0e, 0xf8, 0x72, 0x34, 0x6d, + 0x95, 0x86, 0xd7, 0xcb, 0x31, 0x54, 0x47, 0xd0, 0x38, 0x29, 0x9c, 0x5a, 0x68, 0xd4, 0x87, 0x76, 0xc9, 0xe7, 0x7e, + 0xe3, 0xf4, 0x81, 0x6d, 0x18, 0xcb, 0xc9, 0x05, 0xaf, 0xa0, 0xfb, 0x66, 0xf7, 0xf1, 0x1c, 0xc6, 0x14, 0x11, 0x4f, + 0x2b, 0x79, 0x42, 0x8b, 0xbc, 0xac, 0xe7, 0x6c, 0xfe, 0x0f, 0x58, 0xe7, 0x7c, 0x78, 0x39, 0x30, 0xb0, 0x66, 0x2c, + 0x9b, 0x6d, 0x3a, 0xe1, 0xcf, 0xc9, 0xa4, 0x0e, 0x6d, 0x6d, 0x8a, 0xa1, 0x3a, 0xe7, 0x28, 0xd4, 0x78, 0x4c, 0xa6, + 0xa2, 0x2a, 0xa6, 0x03, 0x30, 0xd7, 0xa8, 0x25, 0x66, 0x87, 0x2f, 0x69, 0x5c, 0x4e, 0xdd, 0xa5, 0x49, 0x5d, 0x37, + 0x4a, 0x59, 0xc4, 0xaf, 0x1f, 0xa2, 0xe4, 0xf8, 0xa6, 0x12, 0x97, 0xd5, 0x79, 0xf5, 0xe2, 0x4a, 0x2b, 0x5f, 0x61, + 0xe4, 0x9e, 0xe3, 0xee, 0xb8, 0xa7, 0x5b, 0x2f, 0xf4, 0x9e, 0x6c, 0xfb, 0xd1, 0xc6, 0x56, 0x77, 0xba, 0x75, 0xaa, + 0x3d, 0x1a, 0xa8, 0x0b, 0xb3, 0x68, 0x24, 0x00, 0x10, 0x7f, 0xfd, 0xd7, 0xa1, 0x8d, 0x83, 0x54, 0x4f, 0x1f, 0xd8, + 0x2a, 0xbe, 0x8a, 0x0c, 0x87, 0xab, 0xa2, 0xde, 0xc3, 0x39, 0xbf, 0x09, 0x03, 0xa5, 0xf3, 0x05, 0x28, 0xe1, 0xe1, + 0xee, 0x39, 0x70, 0x9c, 0xd8, 0x81, 0x12, 0x1e, 0x02, 0x40, 0xd2, 0x6e, 0xf0, 0xeb, 0x1b, 0x3d, 0x22, 0xc6, 0xe5, + 0xe3, 0xb4, 0x5a, 0x98, 0xbb, 0xf0, 0x22, 0x28, 0x8d, 0xe5, 0xd3, 0x16, 0x48, 0x24, 0xa5, 0xe6, 0x66, 0x0c, 0xf9, + 0x08, 0xf9, 0x7e, 0x1e, 0xe1, 0x28, 0x26, 0x22, 0xc7, 0xc7, 0x0a, 0x32, 0x47, 0xfa, 0xa3, 0xbe, 0x3c, 0xc4, 0xc5, + 0x53, 0x0a, 0xd5, 0x94, 0x4a, 0xd7, 0x93, 0xd8, 0x42, 0x99, 0xb9, 0x0a, 0xdb, 0x56, 0xf7, 0xb9, 0x1c, 0x53, 0x4f, + 0xfa, 0xd3, 0x74, 0xad, 0xd9, 0x68, 0xf1, 0x1b, 0xdf, 0x61, 0xc6, 0x5e, 0xa8, 0x48, 0xfc, 0xd4, 0x4a, 0x4c, 0x3c, + 0x32, 0xf7, 0x1c, 0x96, 0x21, 0x9b, 0xf9, 0xa3, 0xcc, 0x5a, 0xce, 0xd5, 0xd7, 0x08, 0x24, 0xf6, 0x1c, 0xfd, 0xdd, + 0x38, 0xc2, 0x32, 0xe9, 0xb8, 0xe7, 0xb6, 0xfa, 0x9d, 0x45, 0x13, 0x2c, 0x83, 0xfd, 0x4a, 0x69, 0x82, 0xcd, 0xdc, + 0xb3, 0x76, 0x0c, 0x9e, 0xd8, 0xf4, 0x1b, 0x45, 0x15, 0xb4, 0x97, 0xe7, 0x58, 0x34, 0xe2, 0x03, 0x29, 0x5a, 0xbf, + 0xb6, 0xe0, 0x5d, 0x13, 0xd9, 0x2b, 0xb4, 0x80, 0xb2, 0x45, 0x81, 0x6a, 0x2e, 0x6c, 0x89, 0x7d, 0xee, 0xbb, 0x52, + 0xdd, 0x1f, 0x18, 0xe7, 0x13, 0x6b, 0x33, 0x0e, 0xea, 0x36, 0x92, 0x77, 0x7b, 0x6d, 0x9c, 0x5a, 0x5f, 0x45, 0x7b, + 0x7b, 0x35, 0x62, 0x23, 0xd1, 0xbf, 0x0f, 0xd0, 0x08, 0x1b, 0x2b, 0x80, 0x6b, 0x7e, 0xf1, 0x21, 0x47, 0xb0, 0x57, + 0xd1, 0x98, 0x72, 0x90, 0x34, 0x1c, 0x20, 0x04, 0xff, 0x3d, 0x5c, 0xee, 0x0e, 0x57, 0x5f, 0x6f, 0x24, 0x4e, 0x3c, + 0xea, 0xfc, 0xa5, 0xa9, 0x83, 0xc9, 0x61, 0xb4, 0x51, 0x24, 0xf8, 0x27, 0x5e, 0x46, 0x8c, 0xb1, 0x53, 0x02, 0x96, + 0x35, 0xba, 0xb8, 0x4c, 0x71, 0xd3, 0x15, 0x59, 0x35, 0x22, 0x20, 0xad, 0x03, 0x9f, 0x66, 0x44, 0x3b, 0x9c, 0x35, + 0x37, 0x1f, 0x9b, 0xbb, 0xf3, 0xdb, 0x35, 0x63, 0x30, 0x64, 0xaa, 0xa2, 0x06, 0xa8, 0x5d, 0xbb, 0xe1, 0x9f, 0x70, + 0xec, 0x82, 0x11, 0x06, 0x36, 0xec, 0x8b, 0x69, 0x66, 0x24, 0x44, 0xc9, 0x4a, 0x57, 0xbb, 0x9b, 0x78, 0x13, 0xce, + 0x9c, 0x0c, 0xba, 0x92, 0x93, 0x63, 0xb8, 0xe2, 0x95, 0x0f, 0x0f, 0x16, 0x39, 0x52, 0xfd, 0x3a, 0x6d, 0x02, 0x4b, + 0xdf, 0x13, 0xd3, 0x2a, 0x22, 0xb4, 0x03, 0x7c, 0x54, 0x49, 0x96, 0x68, 0x54, 0x10, 0xfa, 0xef, 0xaa, 0x6c, 0xe8, + 0x22, 0xdc, 0x71, 0x16, 0x13, 0x1a, 0xf6, 0x28, 0xe5, 0x6d, 0x77, 0x3d, 0xcd, 0x30, 0x63, 0xb1, 0x70, 0x52, 0xa1, + 0xc5, 0x94, 0x5f, 0xcf, 0xe8, 0xb8, 0x26, 0x98, 0xf7, 0x06, 0xa0, 0x0a, 0x70, 0xfa, 0x03, 0x80, 0xac, 0xc1, 0xec, + 0xd6, 0x4c, 0x54, 0xd7, 0xfe, 0x47, 0xb6, 0x88, 0x4a, 0xf7, 0x71, 0x24, 0xee, 0xf3, 0xd2, 0xc2, 0x4a, 0x7f, 0xfe, + 0x61, 0xc7, 0x35, 0xc9, 0x37, 0x67, 0xcb, 0x24, 0x35, 0xda, 0x7e, 0xca, 0x5f, 0xf3, 0x8d, 0xd4, 0x13, 0x8e, 0xd6, + 0xcb, 0x4d, 0x53, 0x8f, 0x53, 0x1f, 0xc0, 0x74, 0xf7, 0x53, 0xb9, 0x5e, 0x23, 0x37, 0xba, 0x6e, 0xe3, 0x9d, 0x07, + 0x55, 0x25, 0x7b, 0xe6, 0x2a, 0x64, 0xd1, 0x32, 0xdd, 0x54, 0x1b, 0x4b, 0xc0, 0xe1, 0xd7, 0x69, 0x58, 0xf8, 0x93, + 0x29, 0xc4, 0xdd, 0x23, 0x2f, 0xa5, 0xfc, 0x9d, 0x7e, 0xf8, 0xd4, 0x90, 0xcd, 0x82, 0x55, 0xdc, 0x16, 0x16, 0x9f, + 0x07, 0x52, 0x9b, 0x9d, 0x25, 0xed, 0x32, 0xc5, 0x7b, 0xdf, 0xf6, 0x83, 0x46, 0x3d, 0x65, 0xb7, 0xef, 0x87, 0x7a, + 0x12, 0x69, 0x8f, 0x06, 0x7c, 0x51, 0x15, 0x4a, 0x08, 0xe8, 0xac, 0x9a, 0x0c, 0x24, 0xa7, 0x27, 0xd8, 0x46, 0x2f, + 0xe7, 0x01, 0x0e, 0x1c, 0xc6, 0x91, 0xb0, 0x6e, 0x85, 0x65, 0xf0, 0x29, 0x0d, 0x2e, 0x6b, 0x3b, 0xfb, 0x4b, 0xdf, + 0xe4, 0x80, 0x93, 0x03, 0x66, 0x46, 0x3e, 0x8a, 0x6e, 0xf3, 0x5e, 0x4d, 0x62, 0x0e, 0x49, 0x05, 0xaf, 0xd4, 0xf8, + 0x21, 0x20, 0x61, 0x1d, 0x39, 0x17, 0xf4, 0x61, 0x47, 0x95, 0xfb, 0x15, 0x2e, 0xb3, 0x4f, 0xd0, 0x5d, 0xf5, 0x7d, + 0x40, 0xda, 0x90, 0x3c, 0x6b, 0xcb, 0x17, 0x00, 0x13, 0x3b, 0x64, 0x34, 0x1b, 0xf0, 0xf2, 0xe5, 0x3b, 0xb2, 0xc7, + 0xd3, 0x5f, 0x3a, 0x44, 0xa6, 0x9b, 0xb7, 0x78, 0x0e, 0x42, 0x5d, 0x4c, 0xc1, 0xe9, 0xd2, 0xcb, 0xb7, 0x78, 0xd1, + 0xfe, 0x9a, 0xb5, 0x07, 0xe9, 0xe0, 0xbe, 0xe2, 0x8a, 0xa7, 0x01, 0x83, 0x00, 0x8c, 0x5c, 0x08, 0xe6, 0x63, 0x12, + 0x92, 0xb7, 0xb7, 0xa6, 0x19, 0x7d, 0x38, 0x13, 0x38, 0x92, 0x87, 0x24, 0xf9, 0x48, 0xb3, 0x5e, 0x87, 0x6a, 0x40, + 0x39, 0x5c, 0x3f, 0xed, 0x8f, 0xee, 0xdb, 0x15, 0x82, 0x06, 0xda, 0x49, 0x21, 0x2b, 0xb5, 0xbf, 0x32, 0x7c, 0x9f, + 0x42, 0x28, 0x63, 0xcf, 0xaf, 0x1e, 0xf8, 0xc6, 0xa0, 0xd1, 0x02, 0x43, 0x57, 0x62, 0xec, 0x9b, 0x0f, 0x01, 0x9e, + 0x71, 0xd8, 0x87, 0x9d, 0x01, 0xc1, 0x58, 0x77, 0xd9, 0xaf, 0xb1, 0x10, 0x7e, 0xdd, 0xa6, 0x50, 0x96, 0xe5, 0xf0, + 0x72, 0x00, 0x6d, 0x4b, 0xf8, 0x2a, 0x8f, 0x19, 0xf3, 0x22, 0x88, 0x11, 0x4a, 0x8b, 0x7c, 0xfd, 0xb7, 0xed, 0xe1, + 0xf6, 0x40, 0x39, 0xe0, 0xe9, 0xf6, 0x3d, 0x25, 0xe6, 0x74, 0x3c, 0x58, 0x57, 0x7f, 0xe1, 0x22, 0x96, 0x47, 0x31, + 0x91, 0xba, 0x70, 0x85, 0x28, 0x6b, 0x9f, 0x6e, 0x25, 0xac, 0x23, 0x66, 0x2f, 0x29, 0x88, 0x28, 0xce, 0x8c, 0x5c, + 0x88, 0x53, 0xd1, 0x3b, 0xcc, 0x6a, 0x51, 0xb2, 0xe1, 0x28, 0x3f, 0x91, 0xb4, 0x0d, 0x00, 0x3a, 0xe3, 0xf8, 0xc3, + 0x8f, 0xd7, 0x96, 0x62, 0x0e, 0x2e, 0xfc, 0xc8, 0x6c, 0x77, 0xa6, 0x1d, 0x22, 0xc1, 0xb8, 0xe6, 0x61, 0xd7, 0x67, + 0x36, 0x13, 0x7b, 0xbb, 0x9b, 0x59, 0x09, 0xa6, 0xdf, 0xf7, 0x6b, 0xa3, 0x40, 0x1a, 0xf5, 0x4f, 0xb4, 0xda, 0xd3, + 0xf3, 0x81, 0x93, 0xc6, 0x18, 0xd9, 0x26, 0xee, 0xac, 0xf0, 0xaa, 0xdf, 0xc5, 0x9c, 0xca, 0xc2, 0xa2, 0xcc, 0x7b, + 0x5c, 0x24, 0xb0, 0xbc, 0xd0, 0x6a, 0x4d, 0x89, 0x09, 0xb8, 0x07, 0xfe, 0x87, 0xad, 0x0a, 0xea, 0xb8, 0x42, 0xf9, + 0x5e, 0xb3, 0x3e, 0x36, 0x4c, 0xaf, 0x75, 0x9e, 0x1c, 0xeb, 0xbd, 0xbc, 0xbb, 0x80, 0x40, 0xa7, 0x3a, 0x30, 0xbf, + 0xa8, 0x44, 0xf4, 0xeb, 0x38, 0xad, 0x29, 0xba, 0x23, 0xed, 0x41, 0x0c, 0xea, 0xd2, 0xbb, 0x41, 0x18, 0xd6, 0xb9, + 0xba, 0x65, 0x2b, 0xa3, 0x91, 0x6d, 0x1f, 0xa9, 0xf4, 0xd1, 0x25, 0x8d, 0x4d, 0x38, 0xff, 0x64, 0xa0, 0xec, 0xde, + 0xa6, 0xb6, 0x79, 0xab, 0x8e, 0x33, 0x6c, 0x47, 0xde, 0xaf, 0x94, 0xa4, 0xa5, 0x86, 0x77, 0x55, 0x09, 0x92, 0x81, + 0x31, 0x76, 0xc7, 0x34, 0x22, 0x89, 0x8e, 0x3d, 0x26, 0x26, 0xd7, 0xfc, 0x1e, 0x16, 0x72, 0x13, 0x33, 0x63, 0xd5, + 0x22, 0xbe, 0xb8, 0x04, 0x34, 0x84, 0x41, 0xbb, 0x80, 0xd0, 0x9f, 0x46, 0x48, 0x07, 0xa7, 0xfc, 0x2b, 0x3a, 0x75, + 0x55, 0x8c, 0xc7, 0x6a, 0xbd, 0x7e, 0x46, 0x08, 0x84, 0x0f, 0xd5, 0x74, 0xc0, 0x82, 0x8e, 0xaa, 0x61, 0x05, 0x01, + 0xb2, 0x47, 0x6e, 0x20, 0x6a, 0x2d, 0x58, 0x70, 0x48, 0x32, 0xa7, 0x37, 0xd2, 0xb8, 0x82, 0x1a, 0x51, 0xb9, 0x61, + 0xdd, 0xfd, 0x9d, 0x6b, 0x0e, 0x18, 0x97, 0xf8, 0x45, 0x5f, 0x87, 0x10, 0xcf, 0x34, 0x72, 0x45, 0x26, 0x49, 0x70, + 0xe7, 0xa3, 0x78, 0xe0, 0x52, 0x89, 0x84, 0x94, 0x83, 0x82, 0xc2, 0x69, 0x8f, 0xe3, 0xe1, 0x3f, 0x60, 0x74, 0x88, + 0xc4, 0xf7, 0x75, 0x2c, 0xfb, 0xbd, 0xb6, 0xc4, 0x7e, 0x10, 0x0a, 0x6c, 0x90, 0x04, 0x9e, 0xc3, 0x3f, 0x59, 0x7c, + 0xce, 0x31, 0x18, 0x60, 0x57, 0x73, 0x46, 0x94, 0x7d, 0x06, 0xa0, 0x6d, 0x44, 0xec, 0xa2, 0x0a, 0x9e, 0x05, 0x15, + 0xef, 0xca, 0x5c, 0xbf, 0x00, 0xeb, 0xf7, 0x3d, 0x32, 0xd4, 0xa5, 0xef, 0x49, 0x89, 0x5e, 0x46, 0xb0, 0xa6, 0x63, + 0x5b, 0x8a, 0x73, 0xae, 0x6f, 0xd5, 0x9d, 0xf8, 0x4f, 0x40, 0xb5, 0xb2, 0x6e, 0xd3, 0xb6, 0x01, 0xa9, 0x26, 0xa2, + 0x21, 0xcf, 0x33, 0x7a, 0x3a, 0xa4, 0x23, 0x13, 0xb0, 0x69, 0x6a, 0xee, 0xce, 0xd8, 0x9d, 0x01, 0x1d, 0x50, 0xc1, + 0x30, 0x6c, 0xb1, 0xcd, 0xa0, 0xf0, 0xf0, 0xa2, 0x64, 0x6f, 0xbb, 0xbf, 0x5e, 0xe6, 0xab, 0x87, 0xb4, 0x0f, 0x4f, + 0x15, 0xaf, 0xb5, 0x25, 0xa1, 0xb2, 0xd0, 0x80, 0x2c, 0xfb, 0xf9, 0xfe, 0xd2, 0x33, 0xbb, 0x76, 0xfe, 0x7c, 0xa8, + 0x66, 0xf7, 0xe7, 0x85, 0x9f, 0x1f, 0x85, 0x57, 0x88, 0xe1, 0xe9, 0x63, 0xe4, 0xd8, 0x1c, 0xa1, 0xfb, 0xda, 0x44, + 0x05, 0x2e, 0x1d, 0x3a, 0x1c, 0xff, 0xc8, 0x3b, 0xc0, 0xfe, 0xda, 0x22, 0x0b, 0x43, 0xd6, 0x88, 0x39, 0x4c, 0x4a, + 0xa6, 0x69, 0x18, 0x93, 0x42, 0x4e, 0xb5, 0xcc, 0x66, 0x0d, 0x09, 0xf8, 0x1e, 0x7c, 0xd3, 0x3c, 0x99, 0x0d, 0x50, + 0x1d, 0x62, 0xe9, 0x57, 0x06, 0xbf, 0x19, 0x88, 0xdd, 0xad, 0x7b, 0x4f, 0xf9, 0xc7, 0x82, 0x6d, 0x8d, 0xc8, 0xc4, + 0xc5, 0x78, 0x17, 0x20, 0x15, 0xc5, 0x52, 0x41, 0xcf, 0x5b, 0xd6, 0x7f, 0x94, 0x02, 0x41, 0xe0, 0x40, 0x22, 0x03, + 0x5e, 0xd1, 0x53, 0xd4, 0x86, 0xd3, 0x2c, 0x9f, 0x0f, 0x96, 0xe3, 0x6b, 0x9a, 0x76, 0x32, 0x06, 0x47, 0x4b, 0x11, + 0xb3, 0xdd, 0x03, 0x65, 0xbd, 0x9b, 0x01, 0xda, 0x9c, 0xb9, 0x7e, 0x3f, 0x6a, 0xc4, 0x7b, 0xea, 0xd4, 0x3c, 0xb9, + 0xfb, 0x5c, 0x6b, 0x64, 0x33, 0x52, 0xba, 0x64, 0x78, 0x8f, 0xa4, 0xaf, 0x7a, 0x61, 0x8d, 0xbc, 0xc5, 0x73, 0xe9, + 0x6b, 0x58, 0x97, 0x4b, 0xbf, 0x63, 0x22, 0xd3, 0x37, 0x02, 0x54, 0xc5, 0xb9, 0x16, 0x4a, 0xf0, 0x19, 0xd8, 0x94, + 0x57, 0xb8, 0x8a, 0xb3, 0x16, 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, 0x78, 0xec, 0x00 +}; +static CONST UINT8 DecAssoc013[] = { 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, 0x77, 0xe8, 0x21, 0xff, 0x06, + 0x59, 0x35, 0xce, 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, 0xcf, 0x70, + 0xdd, 0x0a, 0x68, 0xbf, 0xd4, 0xbc, 0x16, 0x76, 0x99, 0x36, 0x1e, 0x58, 0x79, + 0x5e, 0xd4, 0x29, 0xf7, 0x33, 0x93, 0x48, 0xdb, 0x5f, 0x01, 0xae, 0x9c, 0xb6, + 0xe4, 0x88, 0x6d, 0x2b, 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 }; +static CONST UINT8 DecNonce013[] = { 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 }; +static CONST UINT8 DecKey013[] = { 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, 0x8f, 0x59, 0x8e, + 0xc5, 0x90, 0xd5, 0x27, 0x2d, 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, + 0x44, 0x1e, 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 }; + +static CONST struct ChaCha20Poly1305TestVector ChaCha20Poly1305DecVectors[] = { + { DecInput001, + DecOutput001, + DecAssoc001, + DecNonce001, + DecKey001, + sizeof(DecInput001), + sizeof(DecAssoc001), + sizeof(DecNonce001) }, + { DecInput002, NULL, NULL, DecNonce002, DecKey002, sizeof(DecInput002), 0, sizeof(DecNonce002) }, + { DecInput003, + NULL, + DecAssoc003, + DecNonce003, + DecKey003, + sizeof(DecInput003), + sizeof(DecAssoc003), + sizeof(DecNonce003) }, + { DecInput004, + DecOutput004, + DecAssoc004, + DecNonce004, + DecKey004, + sizeof(DecInput004), + sizeof(DecAssoc004), + sizeof(DecNonce004) }, + { DecInput005, DecOutput005, NULL, DecNonce005, DecKey005, sizeof(DecInput005), 0, sizeof(DecNonce005) }, + { DecInput006, + DecOutput006, + DecAssoc006, + DecNonce006, + DecKey006, + sizeof(DecInput006), + sizeof(DecAssoc006), + sizeof(DecNonce006) }, + { DecInput007, DecOutput007, NULL, DecNonce007, DecKey007, sizeof(DecInput007), 0, sizeof(DecNonce007) }, + { DecInput008, DecOutput008, NULL, DecNonce008, DecKey008, sizeof(DecInput008), 0, sizeof(DecNonce008) }, + { DecInput009, + DecOutput009, + DecAssoc009, + DecNonce009, + DecKey009, + sizeof(DecInput009), + sizeof(DecAssoc009), + sizeof(DecNonce009) }, + { DecInput010, + DecOutput010, + DecAssoc010, + DecNonce010, + DecKey010, + sizeof(DecInput010), + sizeof(DecAssoc010), + sizeof(DecNonce010) }, + { DecInput011, + DecOutput011, + DecAssoc011, + DecNonce011, + DecKey011, + sizeof(DecInput011), + sizeof(DecAssoc011), + sizeof(DecNonce011) }, + { DecInput012, + DecOutput012, + DecAssoc012, + DecNonce012, + DecKey012, + sizeof(DecInput012), + sizeof(DecAssoc012), + sizeof(DecNonce012) }, + { DecInput013, + DecOutput013, + DecAssoc013, + DecNonce013, + DecKey013, + sizeof(DecInput013), + sizeof(DecAssoc013), + sizeof(DecNonce013), + TRUE } +}; + +static CONST UINT8 XEncInput001[] = { + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, + 0x20, 0x64, 0x72, 0x61, 0x66, 0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, + 0x6c, 0x69, 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20, 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20, 0x6f, + 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d, 0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x6d, 0x61, + 0x79, 0x20, 0x62, 0x65, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, + 0x63, 0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f, 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x20, 0x62, + 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x20, 0x61, + 0x74, 0x20, 0x61, 0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69, + 0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72, 0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20, 0x75, 0x73, 0x65, + 0x20, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x73, + 0x20, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, 0x6c, + 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, 0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65, 0x6d, 0x20, 0x6f, 0x74, + 0x68, 0x65, 0x72, 0x20, 0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20, 0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, + 0x6b, 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d +}; +static CONST UINT8 XEncOutput001[] = { + 0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77, 0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92, 0x2e, 0xed, 0x93, + 0xcf, 0x0f, 0x71, 0x88, 0x18, 0x7a, 0x9d, 0x2d, 0x24, 0xe0, 0xf5, 0xea, 0x3d, 0x55, 0x64, 0xd7, 0xad, 0x2a, 0x1a, + 0x1f, 0x7e, 0x86, 0x6d, 0xb0, 0xce, 0x80, 0x41, 0x72, 0x86, 0x26, 0xee, 0x84, 0xd7, 0xef, 0x82, 0x9e, 0xe2, 0x60, + 0x9d, 0x5a, 0xfc, 0xf0, 0xe4, 0x19, 0x85, 0xea, 0x09, 0xc6, 0xfb, 0xb3, 0xa9, 0x50, 0x09, 0xec, 0x5e, 0x11, 0x90, + 0xa1, 0xc5, 0x4e, 0x49, 0xef, 0x50, 0xd8, 0x8f, 0xe0, 0x78, 0xd7, 0xfd, 0xb9, 0x3b, 0xc9, 0xf2, 0x91, 0xc8, 0x25, + 0xc8, 0xa7, 0x63, 0x60, 0xce, 0x10, 0xcd, 0xc6, 0x7f, 0xf8, 0x16, 0xf8, 0xe1, 0x0a, 0xd9, 0xde, 0x79, 0x50, 0x33, + 0xf2, 0x16, 0x0f, 0x17, 0xba, 0xb8, 0x5d, 0xd8, 0xdf, 0x4e, 0x51, 0xa8, 0x39, 0xd0, 0x85, 0xca, 0x46, 0x6a, 0x10, + 0xa7, 0xa3, 0x88, 0xef, 0x79, 0xb9, 0xf8, 0x24, 0xf3, 0xe0, 0x71, 0x7b, 0x76, 0x28, 0x46, 0x3a, 0x3a, 0x1b, 0x91, + 0xb6, 0xd4, 0x3e, 0x23, 0xe5, 0x44, 0x15, 0xbf, 0x60, 0x43, 0x9d, 0xa4, 0xbb, 0xd5, 0x5f, 0x89, 0xeb, 0xef, 0x8e, + 0xfd, 0xdd, 0xb4, 0x0d, 0x46, 0xf0, 0x69, 0x23, 0x63, 0xae, 0x94, 0xf5, 0x5e, 0xa5, 0xad, 0x13, 0x1c, 0x41, 0x76, + 0xe6, 0x90, 0xd6, 0x6d, 0xa2, 0x8f, 0x97, 0x4c, 0xa8, 0x0b, 0xcf, 0x8d, 0x43, 0x2b, 0x9c, 0x9b, 0xc5, 0x58, 0xa5, + 0xb6, 0x95, 0x9a, 0xbf, 0x81, 0xc6, 0x54, 0xc9, 0x66, 0x0c, 0xe5, 0x4f, 0x6a, 0x53, 0xa1, 0xe5, 0x0c, 0xba, 0x31, + 0xde, 0x34, 0x64, 0x73, 0x8a, 0x3b, 0xbd, 0x92, 0x01, 0xdb, 0x71, 0x69, 0xf3, 0x58, 0x99, 0xbc, 0xd1, 0xcb, 0x4a, + 0x05, 0xe2, 0x58, 0x9c, 0x25, 0x17, 0xcd, 0xdc, 0x83, 0xb7, 0xff, 0xfb, 0x09, 0x61, 0xad, 0xbf, 0x13, 0x5b, 0x5e, + 0xed, 0x46, 0x82, 0x6f, 0x22, 0xd8, 0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5, 0x9c +}; +static CONST UINT8 XEncAssoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; +static CONST UINT8 XEncNonce001[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 }; +static CONST UINT8 XEncKey001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, + 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, + 0x80, 0x09, 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 }; + +static CONST struct ChaCha20Poly1305TestVector XChaCha20Poly1305EncVectors[] = { { XEncInput001, + XEncOutput001, + XEncAssoc001, + XEncNonce001, + XEncKey001, + sizeof(XEncInput001), + sizeof(XEncAssoc001), + sizeof(XEncNonce001) } }; + +static CONST UINT8 XDecInput001[] = { + 0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77, 0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92, 0x2e, 0xed, 0x93, + 0xcf, 0x0f, 0x71, 0x88, 0x18, 0x7a, 0x9d, 0x2d, 0x24, 0xe0, 0xf5, 0xea, 0x3d, 0x55, 0x64, 0xd7, 0xad, 0x2a, 0x1a, + 0x1f, 0x7e, 0x86, 0x6d, 0xb0, 0xce, 0x80, 0x41, 0x72, 0x86, 0x26, 0xee, 0x84, 0xd7, 0xef, 0x82, 0x9e, 0xe2, 0x60, + 0x9d, 0x5a, 0xfc, 0xf0, 0xe4, 0x19, 0x85, 0xea, 0x09, 0xc6, 0xfb, 0xb3, 0xa9, 0x50, 0x09, 0xec, 0x5e, 0x11, 0x90, + 0xa1, 0xc5, 0x4e, 0x49, 0xef, 0x50, 0xd8, 0x8f, 0xe0, 0x78, 0xd7, 0xfd, 0xb9, 0x3b, 0xc9, 0xf2, 0x91, 0xc8, 0x25, + 0xc8, 0xa7, 0x63, 0x60, 0xce, 0x10, 0xcd, 0xc6, 0x7f, 0xf8, 0x16, 0xf8, 0xe1, 0x0a, 0xd9, 0xde, 0x79, 0x50, 0x33, + 0xf2, 0x16, 0x0f, 0x17, 0xba, 0xb8, 0x5d, 0xd8, 0xdf, 0x4e, 0x51, 0xa8, 0x39, 0xd0, 0x85, 0xca, 0x46, 0x6a, 0x10, + 0xa7, 0xa3, 0x88, 0xef, 0x79, 0xb9, 0xf8, 0x24, 0xf3, 0xe0, 0x71, 0x7b, 0x76, 0x28, 0x46, 0x3a, 0x3a, 0x1b, 0x91, + 0xb6, 0xd4, 0x3e, 0x23, 0xe5, 0x44, 0x15, 0xbf, 0x60, 0x43, 0x9d, 0xa4, 0xbb, 0xd5, 0x5f, 0x89, 0xeb, 0xef, 0x8e, + 0xfd, 0xdd, 0xb4, 0x0d, 0x46, 0xf0, 0x69, 0x23, 0x63, 0xae, 0x94, 0xf5, 0x5e, 0xa5, 0xad, 0x13, 0x1c, 0x41, 0x76, + 0xe6, 0x90, 0xd6, 0x6d, 0xa2, 0x8f, 0x97, 0x4c, 0xa8, 0x0b, 0xcf, 0x8d, 0x43, 0x2b, 0x9c, 0x9b, 0xc5, 0x58, 0xa5, + 0xb6, 0x95, 0x9a, 0xbf, 0x81, 0xc6, 0x54, 0xc9, 0x66, 0x0c, 0xe5, 0x4f, 0x6a, 0x53, 0xa1, 0xe5, 0x0c, 0xba, 0x31, + 0xde, 0x34, 0x64, 0x73, 0x8a, 0x3b, 0xbd, 0x92, 0x01, 0xdb, 0x71, 0x69, 0xf3, 0x58, 0x99, 0xbc, 0xd1, 0xcb, 0x4a, + 0x05, 0xe2, 0x58, 0x9c, 0x25, 0x17, 0xcd, 0xdc, 0x83, 0xb7, 0xff, 0xfb, 0x09, 0x61, 0xad, 0xbf, 0x13, 0x5b, 0x5e, + 0xed, 0x46, 0x82, 0x6f, 0x22, 0xd8, 0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5, 0x9c +}; +static CONST UINT8 XDecOutput001[] = { + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, + 0x20, 0x64, 0x72, 0x61, 0x66, 0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, + 0x6c, 0x69, 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20, 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20, 0x6f, + 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d, 0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x6d, 0x61, + 0x79, 0x20, 0x62, 0x65, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, + 0x63, 0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f, 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x20, 0x62, + 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x20, 0x61, + 0x74, 0x20, 0x61, 0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69, + 0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72, 0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20, 0x75, 0x73, 0x65, + 0x20, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x73, + 0x20, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, 0x6c, + 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, 0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65, 0x6d, 0x20, 0x6f, 0x74, + 0x68, 0x65, 0x72, 0x20, 0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20, 0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, + 0x6b, 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d +}; +static CONST UINT8 XDecAssoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; +static CONST UINT8 XDecNonce001[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 }; +static CONST UINT8 XDecKey001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, + 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, + 0x80, 0x09, 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 }; + +static CONST struct ChaCha20Poly1305TestVector XChaCha20Poly1305DecVectors[] = { { XDecInput001, + XDecOutput001, + XDecAssoc001, + XDecNonce001, + XDecKey001, + sizeof(XDecInput001), + sizeof(XDecAssoc001), + sizeof(XDecNonce001) } }; +#pragma data_seg() +#pragma bss_seg() + +static VOID +ChaCha20Poly1305SelftestEncryptBigNonce( + UINT8 *Dst, + CONST UINT8 *Src, + CONST SIZE_T SrcLen, + CONST UINT8 *Ad, + CONST SIZE_T AdLen, + CONST UINT8 Nonce[12], + CONST UINT8 Key[CHACHA20POLY1305_KEY_SIZE], + CONST SIMD_STATE *Simd); +static VOID +ChaCha20Poly1305SelftestEncrypt( + UINT8 *Dst, + CONST UINT8 *Src, + CONST SIZE_T SrcLen, + CONST UINT8 *Ad, + CONST SIZE_T AdLen, + CONST UINT8 *Nonce, + CONST SIZE_T NonceLen, + CONST UINT8 Key[CHACHA20POLY1305_KEY_SIZE], + CONST SIMD_STATE *Simd); +static BOOLEAN +DecryptionSuccess(BOOLEAN FuncRet, BOOLEAN ExpectFailure, BOOLEAN Equal); +static BOOLEAN +ChaCha20Poly1305Selftest(CONST SIMD_STATE *Simd); + +#ifdef ALLOC_PRAGMA +# pragma alloc_text(INIT, ChaCha20Poly1305SelftestEncryptBigNonce) +# pragma alloc_text(INIT, ChaCha20Poly1305SelftestEncrypt) +# pragma alloc_text(INIT, DecryptionSuccess) +# pragma alloc_text(INIT, ChaCha20Poly1305Selftest) +#endif + +static VOID +ChaCha20Poly1305SelftestEncryptBigNonce( + UINT8 *Dst, + CONST UINT8 *Src, + CONST SIZE_T SrcLen, + CONST UINT8 *Ad, + CONST SIZE_T AdLen, + CONST UINT8 Nonce[12], + CONST UINT8 Key[CHACHA20POLY1305_KEY_SIZE], + CONST SIMD_STATE *Simd) +{ + POLY1305_CTX Poly1305State; + CHACHA20_CTX ChaCha20State; + union + { + UINT8 Block0[POLY1305_KEY_SIZE]; + UINT64_LE Lens[2]; + } b = { { 0 } }; + + ChaCha20Init(&ChaCha20State, Key, 0); + ChaCha20State.Counter[1] = GetUnalignedLe32(Nonce + 0); + ChaCha20State.Counter[2] = GetUnalignedLe32(Nonce + 4); + ChaCha20State.Counter[3] = GetUnalignedLe32(Nonce + 8); + ChaCha20(&ChaCha20State, b.Block0, b.Block0, sizeof(b.Block0), Simd); + Poly1305Init(&Poly1305State, b.Block0, Simd); + Poly1305Update(&Poly1305State, Ad, AdLen); + Poly1305Update(&Poly1305State, Pad0, (0x10 - AdLen) & 0xf); + ChaCha20(&ChaCha20State, Dst, Src, SrcLen, Simd); + Poly1305Update(&Poly1305State, Dst, SrcLen); + Poly1305Update(&Poly1305State, Pad0, (0x10 - SrcLen) & 0xf); + b.Lens[0] = CpuToLe64(AdLen); + b.Lens[1] = CpuToLe64(SrcLen); + Poly1305Update(&Poly1305State, (UINT8 *)b.Lens, sizeof(b.Lens)); + Poly1305Final(&Poly1305State, Dst + SrcLen); + RtlSecureZeroMemory(&ChaCha20State, sizeof(ChaCha20State)); + RtlSecureZeroMemory(&b, sizeof(b)); +} + +static VOID +ChaCha20Poly1305SelftestEncrypt( + UINT8 *Dst, + CONST UINT8 *Src, + CONST SIZE_T SrcLen, + CONST UINT8 *Ad, + CONST SIZE_T AdLen, + CONST UINT8 *Nonce, + CONST SIZE_T NonceLen, + CONST UINT8 Key[CHACHA20POLY1305_KEY_SIZE], + CONST SIMD_STATE *Simd) +{ + if (NonceLen == 8) + ChaCha20Poly1305Encrypt(Dst, Src, SrcLen, Ad, AdLen, GetUnalignedLe64(Nonce), Key); + else if (NonceLen == 12) + ChaCha20Poly1305SelftestEncryptBigNonce(Dst, Src, SrcLen, Ad, AdLen, Nonce, Key, Simd); + else + NT_ASSERT(0); +} + +static BOOLEAN +DecryptionSuccess(BOOLEAN FuncRet, BOOLEAN ExpectFailure, BOOLEAN Equal) +{ + if (ExpectFailure) + return !FuncRet; + return FuncRet && Equal; +} + +#define CHACHA20POLY1305_ENABLE_SLOW_CHUNKED_TEST 0 + +static BOOLEAN +ChaCha20Poly1305Selftest(CONST SIMD_STATE *Simd) +{ + enum + { + MAXIMUM_TEST_BUFFER_LEN = 1UL << 12 + }; + UINT8 *ComputedOutput, *Input; + BOOLEAN Success = TRUE, Ret; + MDL *Mdl, *LinkedMdls[3]; + + ComputedOutput = MemAllocate(MAXIMUM_TEST_BUFFER_LEN); + Input = MemAllocate(MAXIMUM_TEST_BUFFER_LEN); + Mdl = ComputedOutput ? IoAllocateMdl(ComputedOutput, MAXIMUM_TEST_BUFFER_LEN, FALSE, FALSE, NULL) : NULL; + LinkedMdls[0] = Input ? IoAllocateMdl(Input, MAXIMUM_TEST_BUFFER_LEN, FALSE, FALSE, NULL) : NULL; + LinkedMdls[1] = Input ? IoAllocateMdl(Input, MAXIMUM_TEST_BUFFER_LEN, FALSE, FALSE, NULL) : NULL; + LinkedMdls[2] = Input ? IoAllocateMdl(Input, MAXIMUM_TEST_BUFFER_LEN, FALSE, FALSE, NULL) : NULL; + if (!ComputedOutput || !Input || !Mdl || !LinkedMdls[0] || !LinkedMdls[1] || !LinkedMdls[2]) + { + LogDebug("chacha20poly1305 self-test malloc: FAIL"); + Success = FALSE; + goto out; + } + MmBuildMdlForNonPagedPool(Mdl); + MmBuildMdlForNonPagedPool(LinkedMdls[0]); + MmBuildMdlForNonPagedPool(LinkedMdls[1]); + MmBuildMdlForNonPagedPool(LinkedMdls[2]); + LinkedMdls[0]->Next = LinkedMdls[1]; + LinkedMdls[1]->Next = LinkedMdls[2]; + + for (SIZE_T i = 0; i < ARRAYSIZE(ChaCha20Poly1305EncVectors); ++i) + { + RtlZeroMemory(ComputedOutput, MAXIMUM_TEST_BUFFER_LEN); + ChaCha20Poly1305SelftestEncrypt( + ComputedOutput, + ChaCha20Poly1305EncVectors[i].Input, + ChaCha20Poly1305EncVectors[i].InLen, + ChaCha20Poly1305EncVectors[i].Ad, + ChaCha20Poly1305EncVectors[i].AdLen, + ChaCha20Poly1305EncVectors[i].Nonce, + ChaCha20Poly1305EncVectors[i].NonceLen, + ChaCha20Poly1305EncVectors[i].Key, + Simd); + if (!RtlEqualMemory( + ComputedOutput, + ChaCha20Poly1305EncVectors[i].Output, + ChaCha20Poly1305EncVectors[i].InLen + POLY1305_MAC_SIZE)) + { + LogDebug("chacha20poly1305 encryption self-test %zu: FAIL", i + 1); + Success = FALSE; + } + } + for (SIZE_T i = 0; i < ARRAYSIZE(ChaCha20Poly1305EncVectors); ++i) + { + if (ChaCha20Poly1305EncVectors[i].NonceLen != 8) + continue; + RtlCopyMemory(ComputedOutput, ChaCha20Poly1305EncVectors[i].Input, ChaCha20Poly1305EncVectors[i].InLen); + Ret = ChaCha20Poly1305EncryptMdl( + ComputedOutput, + Mdl, + ChaCha20Poly1305EncVectors[i].InLen, + 0, + ChaCha20Poly1305EncVectors[i].Ad, + ChaCha20Poly1305EncVectors[i].AdLen, + GetUnalignedLe64(ChaCha20Poly1305EncVectors[i].Nonce), + ChaCha20Poly1305EncVectors[i].Key, + Simd); + if (!Ret || !RtlEqualMemory( + ComputedOutput, + ChaCha20Poly1305EncVectors[i].Output, + ChaCha20Poly1305EncVectors[i].InLen + POLY1305_MAC_SIZE)) + { + LogDebug("chacha20poly1305 mdl encryption self-test %zu: FAIL", i + 1); + Success = FALSE; + } + } + for (SIZE_T i = 0; i < ARRAYSIZE(ChaCha20Poly1305DecVectors); ++i) + { + RtlZeroMemory(ComputedOutput, MAXIMUM_TEST_BUFFER_LEN); + Ret = ChaCha20Poly1305Decrypt( + ComputedOutput, + ChaCha20Poly1305DecVectors[i].Input, + ChaCha20Poly1305DecVectors[i].InLen, + ChaCha20Poly1305DecVectors[i].Ad, + ChaCha20Poly1305DecVectors[i].AdLen, + GetUnalignedLe64(ChaCha20Poly1305DecVectors[i].Nonce), + ChaCha20Poly1305DecVectors[i].Key); + if (!DecryptionSuccess( + Ret, + ChaCha20Poly1305DecVectors[i].Failure, + RtlEqualMemory( + ComputedOutput, + ChaCha20Poly1305DecVectors[i].Output, + ChaCha20Poly1305DecVectors[i].InLen - POLY1305_MAC_SIZE))) + { + LogDebug("chacha20poly1305 decryption self-test %zu: FAIL", i + 1); + Success = FALSE; + } + } + for (SIZE_T i = 0; i < ARRAYSIZE(ChaCha20Poly1305DecVectors); ++i) + { + RtlCopyMemory(ComputedOutput, ChaCha20Poly1305DecVectors[i].Input, ChaCha20Poly1305DecVectors[i].InLen); + Ret = ChaCha20Poly1305DecryptMdl( + ComputedOutput, + Mdl, + ChaCha20Poly1305DecVectors[i].InLen, + 0, + ChaCha20Poly1305DecVectors[i].Ad, + ChaCha20Poly1305DecVectors[i].AdLen, + GetUnalignedLe64(ChaCha20Poly1305DecVectors[i].Nonce), + ChaCha20Poly1305DecVectors[i].Key, + Simd); + if (!DecryptionSuccess( + Ret, + ChaCha20Poly1305DecVectors[i].Failure, + RtlEqualMemory( + ComputedOutput, + ChaCha20Poly1305DecVectors[i].Output, + ChaCha20Poly1305DecVectors[i].InLen - POLY1305_MAC_SIZE))) + { + LogDebug("chacha20poly1305 mdl decryption self-test %zu: FAIL", i + 1); + Success = FALSE; + } + } + for (SIZE_T i = 0; i < ARRAYSIZE(XChaCha20Poly1305EncVectors); ++i) + { + RtlZeroMemory(ComputedOutput, MAXIMUM_TEST_BUFFER_LEN); + XChaCha20Poly1305Encrypt( + ComputedOutput, + XChaCha20Poly1305EncVectors[i].Input, + XChaCha20Poly1305EncVectors[i].InLen, + XChaCha20Poly1305EncVectors[i].Ad, + XChaCha20Poly1305EncVectors[i].AdLen, + XChaCha20Poly1305EncVectors[i].Nonce, + XChaCha20Poly1305EncVectors[i].Key); + if (!RtlEqualMemory( + ComputedOutput, + XChaCha20Poly1305EncVectors[i].Output, + XChaCha20Poly1305EncVectors[i].InLen + POLY1305_MAC_SIZE)) + { + LogDebug("xchacha20poly1305 encryption self-test %zu: FAIL", i + 1); + Success = FALSE; + } + } + for (SIZE_T i = 0; i < ARRAYSIZE(XChaCha20Poly1305DecVectors); ++i) + { + RtlZeroMemory(ComputedOutput, MAXIMUM_TEST_BUFFER_LEN); + Ret = XChaCha20Poly1305Decrypt( + ComputedOutput, + XChaCha20Poly1305DecVectors[i].Input, + XChaCha20Poly1305DecVectors[i].InLen, + XChaCha20Poly1305DecVectors[i].Ad, + XChaCha20Poly1305DecVectors[i].AdLen, + XChaCha20Poly1305DecVectors[i].Nonce, + XChaCha20Poly1305DecVectors[i].Key); + if (!DecryptionSuccess( + Ret, + XChaCha20Poly1305DecVectors[i].Failure, + RtlEqualMemory( + ComputedOutput, + XChaCha20Poly1305DecVectors[i].Output, + XChaCha20Poly1305DecVectors[i].InLen - POLY1305_MAC_SIZE))) + { + LogDebug("xchacha20poly1305 decryption self-test %zu: FAIL", i + 1); + Success = FALSE; + } + } +#pragma warning(suppress : 4127) /* The whole point is to have a conditional expression on a constant. */ + for (SIZE_T TotalLen = POLY1305_MAC_SIZE; CHACHA20POLY1305_ENABLE_SLOW_CHUNKED_TEST && TotalLen <= 1 << 10; + ++TotalLen) + { + for (SIZE_T i = 0; i <= TotalLen; ++i) + { + for (SIZE_T j = i; j <= TotalLen; ++j) + { + LinkedMdls[0]->MappedSystemVa = Input; + LinkedMdls[0]->ByteCount = i; + LinkedMdls[1]->MappedSystemVa = Input + i; + LinkedMdls[1]->ByteCount = j - i; + LinkedMdls[2]->MappedSystemVa = Input + j; + LinkedMdls[2]->ByteCount = TotalLen - j; + + RtlZeroMemory(ComputedOutput, TotalLen); + RtlZeroMemory(Input, TotalLen); + + if (!ChaCha20Poly1305EncryptMdl( + Input, LinkedMdls[0], TotalLen - POLY1305_MAC_SIZE, 0, NULL, 0, 0, EncKey001, Simd)) + goto chunkfail; + ChaCha20Poly1305Encrypt( + ComputedOutput, ComputedOutput, TotalLen - POLY1305_MAC_SIZE, NULL, 0, 0, EncKey001); + if (!RtlEqualMemory(ComputedOutput, Input, TotalLen)) + goto chunkfail; + if (!ChaCha20Poly1305Decrypt(ComputedOutput, Input, TotalLen, NULL, 0, 0, EncKey001)) + goto chunkfail; + for (SIZE_T k = 0; k < TotalLen - POLY1305_MAC_SIZE; ++k) + { + if (ComputedOutput[k]) + goto chunkfail; + } + if (!ChaCha20Poly1305DecryptMdl(Input, LinkedMdls[0], TotalLen, 0, NULL, 0, 0, EncKey001, Simd)) + goto chunkfail; + for (SIZE_T k = 0; k < TotalLen - POLY1305_MAC_SIZE; ++k) + { + if (Input[k]) + goto chunkfail; + } + continue; + + chunkfail: + LogDebug("chacha20poly1305 chunked self-test %zu/%zu/%zu: FAIL", TotalLen, i, j); + Success = FALSE; + } + } + } + +out: + if (Mdl) + IoFreeMdl(Mdl); + if (LinkedMdls[0]) + IoFreeMdl(LinkedMdls[0]); + if (LinkedMdls[1]) + IoFreeMdl(LinkedMdls[1]); + if (LinkedMdls[2]) + IoFreeMdl(LinkedMdls[2]); + MemFree(ComputedOutput); + MemFree(Input); + return Success; +} diff --git a/driver/selftest/counter.c b/driver/selftest/counter.c new file mode 100644 index 0000000..e225158 --- /dev/null +++ b/driver/selftest/counter.c @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#ifdef ALLOC_PRAGMA +# pragma alloc_text(INIT, PacketCounterSelftest) +#endif +_Use_decl_annotations_ +BOOLEAN +PacketCounterSelftest(VOID) +{ + NOISE_REPLAY_COUNTER *Counter; + ULONG TestNum = 0, i; + BOOLEAN Success = TRUE; + + Counter = MemAllocate(sizeof(*Counter)); + if (!Counter) + { + LogDebug("nonce counter self-test malloc: FAIL"); + return FALSE; + } + +#define T_INIT \ + do \ + { \ + RtlZeroMemory(Counter, sizeof(*Counter)); \ + KeInitializeSpinLock(&Counter->Lock); \ + } while (0) +#define T_LIM (COUNTER_WINDOW_SIZE + 1) +#define T(n, V) \ + do \ + { \ + ++TestNum; \ + if (CounterValidate(Counter, n) != (V)) \ + { \ + LogDebug("nonce counter self-test %u: FAIL", TestNum); \ + Success = FALSE; \ + } \ + } while (0) + + T_INIT; + /* 1 */ T(0, TRUE); + /* 2 */ T(1, TRUE); + /* 3 */ T(1, FALSE); + /* 4 */ T(9, TRUE); + /* 5 */ T(8, TRUE); + /* 6 */ T(7, TRUE); + /* 7 */ T(7, FALSE); + /* 8 */ T(T_LIM, TRUE); + /* 9 */ T(T_LIM - 1, TRUE); + /* 10 */ T(T_LIM - 1, FALSE); + /* 11 */ T(T_LIM - 2, TRUE); + /* 12 */ T(2, TRUE); + /* 13 */ T(2, FALSE); + /* 14 */ T(T_LIM + 16, TRUE); + /* 15 */ T(3, FALSE); + /* 16 */ T(T_LIM + 16, FALSE); + /* 17 */ T(T_LIM * 4, TRUE); + /* 18 */ T(T_LIM * 4 - (T_LIM - 1), TRUE); + /* 19 */ T(10, FALSE); + /* 20 */ T(T_LIM * 4 - T_LIM, FALSE); + /* 21 */ T(T_LIM * 4 - (T_LIM + 1), FALSE); + /* 22 */ T(T_LIM * 4 - (T_LIM - 2), TRUE); + /* 23 */ T(T_LIM * 4 + 1 - T_LIM, FALSE); + /* 24 */ T(0, FALSE); + /* 25 */ T(REJECT_AFTER_MESSAGES, FALSE); + /* 26 */ T(REJECT_AFTER_MESSAGES - 1, TRUE); + /* 27 */ T(REJECT_AFTER_MESSAGES, FALSE); + /* 28 */ T(REJECT_AFTER_MESSAGES - 1, FALSE); + /* 29 */ T(REJECT_AFTER_MESSAGES - 2, TRUE); + /* 30 */ T(REJECT_AFTER_MESSAGES + 1, FALSE); + /* 31 */ T(REJECT_AFTER_MESSAGES + 2, FALSE); + /* 32 */ T(REJECT_AFTER_MESSAGES - 2, FALSE); + /* 33 */ T(REJECT_AFTER_MESSAGES - 3, TRUE); + /* 34 */ T(0, FALSE); + + T_INIT; + for (i = 1; i <= COUNTER_WINDOW_SIZE; ++i) + T(i, TRUE); + T(0, TRUE); + T(0, FALSE); + + T_INIT; + for (i = 2; i <= COUNTER_WINDOW_SIZE + 1; ++i) + T(i, TRUE); + T(1, TRUE); + T(0, FALSE); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 1; i-- > 0;) + T(i, TRUE); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 2; i-- > 1;) + T(i, TRUE); + T(0, FALSE); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 1; i-- > 1;) + T(i, TRUE); + T(COUNTER_WINDOW_SIZE + 1, TRUE); + T(0, FALSE); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 1; i-- > 1;) + T(i, TRUE); + T(0, TRUE); + T(COUNTER_WINDOW_SIZE + 1, TRUE); + +#undef T +#undef T_LIM +#undef T_INIT + + if (Success) + LogDebug("nonce counter self-tests: pass"); + MemFree(Counter); + return Success; +} diff --git a/driver/selftest/ratelimiter.c b/driver/selftest/ratelimiter.c new file mode 100644 index 0000000..2859103 --- /dev/null +++ b/driver/selftest/ratelimiter.c @@ -0,0 +1,255 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include "messages.h" + +#define IS_BEFORE_UNBIASED_INTERRUPT_TIME(A) ((LONG64)(A) < (LONG64)KeQueryUnbiasedInterruptTime()) + +static UINT64 +MaximumSysTimeUnitsAtIndex(LONG Index); +static NTSTATUS +TimingsTest(IPV4HDR *Hdr4, IPV6HDR *Hdr6, LONG *Test); +static NTSTATUS +CapacityTest(IPV4HDR *Hdr4, LONG *Test); + +#ifdef ALLOC_PRAGMA +# pragma alloc_text(INIT, MaximumSysTimeUnitsAtIndex) +# pragma alloc_text(INIT, TimingsTest) +# pragma alloc_text(INIT, CapacityTest) +# pragma alloc_text(INIT, RatelimiterSelftest) +#endif + +#pragma data_seg("INITDATA") +#pragma bss_seg("INITBSS") +static CONST struct +{ + BOOLEAN Failure; + ULONG64 SysTimeUnitsToSleepBefore; +} ExpectedResults[] = { [PACKETS_BURSTABLE] = { TRUE, 0 }, + [PACKETS_BURSTABLE + 1] = { FALSE, SYS_TIME_UNITS_PER_SEC / PACKETS_PER_SECOND }, + [PACKETS_BURSTABLE + 2] = { TRUE, 0 }, + [PACKETS_BURSTABLE + 3] = { FALSE, 2 * SYS_TIME_UNITS_PER_SEC / PACKETS_PER_SECOND }, + [PACKETS_BURSTABLE + 4] = { FALSE, 0 }, + [PACKETS_BURSTABLE + 5] = { TRUE, 0 } }; +#pragma data_seg() +#pragma bss_seg() + +static UINT64 +MaximumSysTimeUnitsAtIndex(LONG Index) +{ + ULONG64 Total = 2 * SYS_TIME_UNITS_PER_SEC / PACKETS_PER_SECOND / 3; + LONG i; + + for (i = 0; i <= Index; ++i) + Total += ExpectedResults[i].SysTimeUnitsToSleepBefore; + return Total; +} + +#define IN6_NIBBLE(A, i) (((ULONG *)(A))[i]) + +static NTSTATUS +TimingsTest(IPV4HDR *Hdr4, IPV6HDR *Hdr6, LONG *Test) +{ + ULONG64 LoopStartTime; + LONG i; + SOCKADDR_IN Src4 = { .sin_family = AF_INET, .sin_addr.S_un.S_addr = Hdr4->Saddr }; + SOCKADDR_IN6 Src6 = { .sin6_family = AF_INET6, .sin6_addr = Hdr6->Saddr }; + + RatelimiterGcEntries(NULL); + RcuBarrier(); + LoopStartTime = KeQueryUnbiasedInterruptTime(); + + for (i = 0; i < ARRAYSIZE(ExpectedResults); ++i) + { + if (ExpectedResults[i].SysTimeUnitsToSleepBefore) + KeDelayExecutionThread( + KernelMode, + FALSE, + &(LARGE_INTEGER){ .QuadPart = -(LONG64)ExpectedResults[i].SysTimeUnitsToSleepBefore }); + + if (IS_BEFORE_UNBIASED_INTERRUPT_TIME(LoopStartTime + MaximumSysTimeUnitsAtIndex(i))) + return STATUS_TIMEOUT; + if (RatelimiterAllow((SOCKADDR *)&Src4) == ExpectedResults[i].Failure) + return STATUS_DISK_FULL; + ++(*Test); + + Src4.sin_addr.s_addr = Hdr4->Saddr = Htonl(Ntohl(Hdr4->Saddr) + i + 1); + if (IS_BEFORE_UNBIASED_INTERRUPT_TIME(LoopStartTime + MaximumSysTimeUnitsAtIndex(i))) + return STATUS_TIMEOUT; + if (!RatelimiterAllow((SOCKADDR *)&Src4)) + return STATUS_DISK_FULL; + ++(*Test); + + Src4.sin_addr.s_addr = Hdr4->Saddr = Htonl(Ntohl(Hdr4->Saddr) - i - 1); + + IN6_NIBBLE(&Src6.sin6_addr, 2) = IN6_NIBBLE(&Hdr6->Saddr, 2) = Htonl(i); + IN6_NIBBLE(&Src6.sin6_addr, 3) = IN6_NIBBLE(&Hdr6->Saddr, 3) = Htonl(i); + if (IS_BEFORE_UNBIASED_INTERRUPT_TIME(LoopStartTime + MaximumSysTimeUnitsAtIndex(i))) + return STATUS_TIMEOUT; + if (RatelimiterAllow((SOCKADDR *)&Src6) == ExpectedResults[i].Failure) + return STATUS_DISK_FULL; + ++(*Test); + + IN6_NIBBLE(&Src6.sin6_addr, 0) = IN6_NIBBLE(&Hdr6->Saddr, 0) = + Htonl(Ntohl(IN6_NIBBLE(&Hdr6->Saddr, 0)) + i + 1); + if (IS_BEFORE_UNBIASED_INTERRUPT_TIME(LoopStartTime + MaximumSysTimeUnitsAtIndex(i))) + return STATUS_TIMEOUT; + if (!RatelimiterAllow((SOCKADDR *)&Src6)) + return STATUS_DISK_FULL; + ++(*Test); + + IN6_NIBBLE(&Src6.sin6_addr, 0) = IN6_NIBBLE(&Hdr6->Saddr, 0) = + Htonl(Ntohl(IN6_NIBBLE(&Hdr6->Saddr, 0)) - i - 1); + + if (IS_BEFORE_UNBIASED_INTERRUPT_TIME(LoopStartTime + MaximumSysTimeUnitsAtIndex(i))) + return STATUS_TIMEOUT; + } + return STATUS_SUCCESS; +} + +static NTSTATUS +CapacityTest(IPV4HDR *Hdr4, LONG *Test) +{ + ULONG i; + SOCKADDR_IN Src4 = { .sin_family = AF_INET, .sin_addr.S_un.S_addr = Hdr4->Saddr }; + + RatelimiterGcEntries(NULL); + RcuBarrier(); + + if (ReadNoFence(&TotalEntries)) + return STATUS_DISK_FULL; + ++(*Test); + + for (i = 0; i <= MAX_ENTRIES; ++i) + { + Src4.sin_addr.s_addr = Hdr4->Saddr = Htonl(i); + if (RatelimiterAllow((SOCKADDR *)&Src4) != (i != MAX_ENTRIES)) + return STATUS_DISK_FULL; + ++(*Test); + } + return STATUS_SUCCESS; +} + +_Use_decl_annotations_ +BOOLEAN +RatelimiterSelftest(VOID) +{ + enum + { + TRIALS_BEFORE_GIVING_UP = 5000 + }; + BOOLEAN Success = FALSE; + LONG Test = 0, Trials; + NET_BUFFER_LIST *Nbl4, *Nbl6 = NULL; + IPV4HDR *Hdr4; + IPV6HDR *Hdr6 = NULL; + + NET_BUFFER_LIST_POOL_PARAMETERS NblPoolParameters = { + .Header = { .Type = NDIS_OBJECT_TYPE_DEFAULT, + .Revision = NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1, + .Size = NDIS_SIZEOF_NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1 }, + .ProtocolId = NDIS_PROTOCOL_ID_DEFAULT, + .PoolTag = MEMORY_TAG + }; + NDIS_HANDLE NblPool = NdisAllocateNetBufferListPool(NULL, &NblPoolParameters); + if (!NblPool) + goto out; + NET_BUFFER_POOL_PARAMETERS NbPoolParameters = { .Header = { .Type = NDIS_OBJECT_TYPE_DEFAULT, + .Revision = NET_BUFFER_POOL_PARAMETERS_REVISION_1, + .Size = + NDIS_SIZEOF_NET_BUFFER_POOL_PARAMETERS_REVISION_1 }, + .PoolTag = MEMORY_TAG }; + NDIS_HANDLE NbPool = NdisAllocateNetBufferPool(NULL, &NbPoolParameters); + if (!NbPool) + goto cleanupNblPool; + ++Test; + ++Test; + ++Test; + + Nbl4 = MemAllocateNetBufferList(NblPool, NbPool, 0, sizeof(*Hdr4), 0); + if (!Nbl4) + goto cleanupNofree; + NdisSetNblFlag(Nbl4, NDIS_NBL_FLAGS_IS_IPV4); + NET_BUFFER_LIST_INFO(Nbl4, NetBufferListProtocolId) = (VOID *)Htons(NDIS_ETH_TYPE_IPV4); + Hdr4 = MemGetValidatedNetBufferListData(Nbl4); + Hdr4->Saddr = Htonl(8182); + ++Test; + + Nbl6 = MemAllocateNetBufferList(NblPool, NbPool, 0, sizeof(*Hdr6), 0); + if (!Nbl6) + { + MemFreeNetBufferList(Nbl4); + goto cleanupNofree; + } + NdisSetNblFlag(Nbl6, NDIS_NBL_FLAGS_IS_IPV6); + NET_BUFFER_LIST_INFO(Nbl6, NetBufferListProtocolId) = (VOID *)Htons(NDIS_ETH_TYPE_IPV6); + Hdr6 = MemGetValidatedNetBufferListData(Nbl6); + IN6_NIBBLE(&Hdr6->Saddr, 0) = Htonl(1212); + IN6_NIBBLE(&Hdr6->Saddr, 1) = Htonl(289188); + ++Test; + + for (Trials = TRIALS_BEFORE_GIVING_UP;;) + { + LONG TestCount = 0; + NTSTATUS Ret; + + Ret = TimingsTest(Hdr4, Hdr6, &TestCount); + if (Ret == STATUS_TIMEOUT) + { + if (!Trials--) + { + Test += TestCount; + goto cleanup; + } + KeDelayExecutionThread(KernelMode, FALSE, &(LARGE_INTEGER){ .QuadPart = -SYS_TIME_UNITS_PER_SEC / 2 }); + continue; + } + else if (!NT_SUCCESS(Ret)) + { + Test += TestCount; + goto cleanup; + } + else + { + Test += TestCount; + break; + } + } + + for (Trials = TRIALS_BEFORE_GIVING_UP;;) + { + LONG TestCount = 0; + + if (!NT_SUCCESS(CapacityTest(Hdr4, &TestCount))) + { + if (!Trials--) + { + Test += TestCount; + goto cleanup; + } + KeDelayExecutionThread(KernelMode, FALSE, &(LARGE_INTEGER){ .QuadPart = -SYS_TIME_UNITS_PER_SEC / 20 }); + continue; + } + Test += TestCount; + break; + } + + Success = TRUE; + +cleanup: + MemFreeNetBufferList(Nbl4); + MemFreeNetBufferList(Nbl6); +cleanupNofree: + NdisFreeNetBufferPool(NbPool); +cleanupNblPool: + NdisFreeNetBufferListPool(NblPool); +out: + if (Success) + LogDebug("ratelimiter self-tests: pass"); + else + LogDebug("ratelimiter self-test %d: FAIL", Test); + + return Success; +} diff --git a/driver/send.c b/driver/send.c new file mode 100644 index 0000000..5fa7304 --- /dev/null +++ b/driver/send.c @@ -0,0 +1,524 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include "interlocked.h" +#include "queueing.h" +#include "timers.h" +#include "device.h" +#include "peer.h" +#include "rcu.h" +#include "socket.h" +#include "messages.h" +#include "cookie.h" +#include "logging.h" + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_not_held_(Peer->Handshake.StaticIdentity->Lock) +_Requires_lock_not_held_(Peer->Handshake.Lock) +static VOID +PacketSendHandshakeInitiation(_Inout_ WG_PEER *Peer) +{ + MESSAGE_HANDSHAKE_INITIATION Packet; + + if (!BirthdateHasExpired(ReadNoFence64(&Peer->LastSentHandshake), REKEY_TIMEOUT)) + return; /* This function is rate limited. */ + + WriteNoFence64(&Peer->LastSentHandshake, KeQueryInterruptTime()); + CHAR EndpointName[SOCKADDR_STR_MAX_LEN]; + SockaddrToString(EndpointName, &Peer->Endpoint.Addr); + LogInfoRatelimited(Peer->Device, "Sending handshake initiation to peer %llu (%s)", Peer->InternalId, EndpointName); + + if (NoiseHandshakeCreateInitiation(&Packet, &Peer->Handshake)) + { + CookieAddMacToPacket(&Packet, sizeof(Packet), Peer); + TimersAnyAuthenticatedPacketTraversal(Peer); + TimersAnyAuthenticatedPacketSent(Peer); + WriteNoFence64(&Peer->LastSentHandshake, KeQueryInterruptTime()); + SocketSendBufferToPeer(Peer, &Packet, sizeof(Packet)); + TimersHandshakeInitiated(Peer); + } +} + +_Use_decl_annotations_ +VOID +PacketHandshakeTxWorker(MULTICORE_WORKQUEUE *WorkQueue) +{ + WG_DEVICE *Wg = CONTAINING_RECORD(WorkQueue, WG_DEVICE, HandshakeTxThreads); + PEER_SERIAL_ENTRY *Entry; + + while ((Entry = PeerSerialDequeue(&Wg->HandshakeTxQueue)) != NULL) + { + WG_PEER *Peer = CONTAINING_RECORD(Entry, WG_PEER, HandshakeTxSerialEntry); + HANDSHAKE_TX_ACTION Action = InterlockedExchange16(&Peer->HandshakeTxAction, HANDSHAKE_TX_NONE); + + if (Action == HANDSHAKE_TX_SEND) + PacketSendHandshakeInitiation(Peer); + else if (Action == HANDSHAKE_TX_CLEAR) + { + NoiseHandshakeClear(&Peer->Handshake); + NoiseKeypairsClear(&Peer->Keypairs); + } + + if (!PeerSerialMaybeRetire(&Wg->HandshakeTxQueue, Entry, FALSE)) + { + ExReleaseRundownProtection(&Peer->InUse); + PeerPut(Peer); + } + } +} + +_Use_decl_annotations_ +VOID +PacketSendQueuedHandshakeInitiation(WG_PEER *Peer, BOOLEAN IsRetry) +{ + if (!IsRetry) + Peer->TimerHandshakeAttempts = 0; + + /* We check LastSentHandshake here in addition to the actual function + * we're queueing up, so that we don't queue things if not strictly + * necessary: + */ + if (!BirthdateHasExpired(ReadNoFence64(&Peer->LastSentHandshake), REKEY_TIMEOUT)) + return; + + if (!ExAcquireRundownProtection(&Peer->InUse)) + return; + PeerGet(Peer); + WriteNoFence16(&Peer->HandshakeTxAction, HANDSHAKE_TX_SEND); + + if (PeerSerialEnqueueIfNotBusy(&Peer->Device->HandshakeTxQueue, &Peer->HandshakeTxSerialEntry, TRUE)) + MulticoreWorkQueueBump(&Peer->Device->HandshakeTxThreads); + else + { + /* If the work was already on the queue, we want to drop the extra reference. */ + ExReleaseRundownProtection(&Peer->InUse); + PeerPut(Peer); + } +} + +_Use_decl_annotations_ +VOID +PacketSendHandshakeResponse(WG_PEER *Peer) +{ + MESSAGE_HANDSHAKE_RESPONSE Packet; + + WriteNoFence64(&Peer->LastSentHandshake, KeQueryInterruptTime()); + CHAR EndpointName[SOCKADDR_STR_MAX_LEN]; + SockaddrToString(EndpointName, &Peer->Endpoint.Addr); + LogInfoRatelimited(Peer->Device, "Sending handshake response to peer %llu (%s)", Peer->InternalId, EndpointName); + + if (NoiseHandshakeCreateResponse(&Packet, &Peer->Handshake)) + { + CookieAddMacToPacket(&Packet, sizeof(Packet), Peer); + if (NoiseHandshakeBeginSession(&Peer->Handshake, &Peer->Keypairs)) + { + TimersSessionDerived(Peer); + TimersAnyAuthenticatedPacketTraversal(Peer); + TimersAnyAuthenticatedPacketSent(Peer); + WriteNoFence64(&Peer->LastSentHandshake, KeQueryInterruptTime()); + SocketSendBufferToPeer(Peer, &Packet, sizeof(Packet)); + } + } +} + +_Use_decl_annotations_ +VOID +PacketSendHandshakeCookie(WG_DEVICE *Wg, CONST NET_BUFFER_LIST *InitiatingNbl, UINT32_LE SenderIndex) +{ + MESSAGE_HANDSHAKE_COOKIE Packet; + + LogInfoNblRatelimited(Wg, "Sending cookie response for denied handshake message for %s", InitiatingNbl); + CookieMessageCreate(&Packet, InitiatingNbl, SenderIndex, &Wg->CookieChecker); + SocketSendBufferAsReplyToNbl(Wg, InitiatingNbl, &Packet, sizeof(Packet)); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +static VOID +KeepKeyFresh(_Inout_ WG_PEER *Peer) +{ + KIRQL Irql; + NOISE_KEYPAIR *Keypair; + BOOLEAN Send; + + Irql = RcuReadLock(); + Keypair = RcuDereference(NOISE_KEYPAIR, Peer->Keypairs.CurrentKeypair); + Send = Keypair && ReadBooleanNoFence(&Keypair->Sending.IsValid) && + (ReadNoFence64(&Keypair->SendingCounter) > REKEY_AFTER_MESSAGES || + (Keypair->IAmTheInitiator && BirthdateHasExpired(Keypair->Sending.Birthdate, REKEY_AFTER_TIME))); + RcuReadUnlock(Irql); + + if (Send) + PacketSendQueuedHandshakeInitiation(Peer, FALSE); +} + +static ULONG +CalculateNblPadding(_In_ CONST NET_BUFFER *Nb, _In_ UINT32 Mtu) +{ + ULONG PaddedSize, LastUnit = NET_BUFFER_DATA_LENGTH(Nb); + + if (!Mtu) + return ALIGN_UP_BY_T(ULONG, LastUnit, MESSAGE_PADDING_MULTIPLE) - LastUnit; + + /* We do this modulo business with the MTU, just in case NDIS gives us + * a NB that's bigger than the MTU. In that case, we wouldn't want the + * final subtraction to overflow in the case of the PaddedSize being + * clamped. + */ + if (LastUnit > Mtu) + LastUnit %= Mtu; + + PaddedSize = min(Mtu, ALIGN_UP_BY_T(ULONG, LastUnit, MESSAGE_PADDING_MULTIPLE)); + return PaddedSize - LastUnit; +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Must_inspect_result_ +static BOOLEAN +EncryptPacket( + _In_ CONST SIMD_STATE *Simd, + _Inout_ NET_BUFFER *NbOut, + _Inout_ NET_BUFFER *NbIn, + _In_ CONST NOISE_KEYPAIR *Keypair, + _In_ UINT32 Mtu) +{ + ULONG PaddingLen = CalculateNblPadding(NbIn, Mtu); + UCHAR *OutBuffer = MemGetValidatedNetBufferData(NbOut); + *(MESSAGE_DATA *)OutBuffer = (MESSAGE_DATA){ .Header.Type = CpuToLe32(MESSAGE_TYPE_DATA), + .KeyIdx = Keypair->RemoteIndex, + .Counter = CpuToLe64(NET_BUFFER_NONCE(NbOut)) }; + OutBuffer += sizeof(MESSAGE_DATA); + + MDL *LastMdl = NULL, *OriginalNextMdl = NULL, PaddingMdl = { 0 }; + ULONG OriginalMdlLen = 0; + if (PaddingLen) + { + ULONG MdlOffset = NET_BUFFER_CURRENT_MDL_OFFSET(NbIn), Remaining = NET_BUFFER_DATA_LENGTH(NbIn), NeededLen = 0; + for (LastMdl = NET_BUFFER_CURRENT_MDL(NbIn);; LastMdl = LastMdl->Next) + { + NeededLen = Remaining + MdlOffset; + Remaining -= min(MmGetMdlByteCount(LastMdl) - MdlOffset, Remaining); + if (!Remaining) + break; + MdlOffset = 0; + if (!LastMdl->Next) + return FALSE; + } + OriginalMdlLen = MmGetMdlByteCount(LastMdl); + OriginalNextMdl = LastMdl->Next; + /* This MDL is completely bogus, but hopefully is sufficient for just returning MappedSystemVa. */ + static CONST UCHAR Padding[MESSAGE_PADDING_MULTIPLE - 1] = { 0 }; + PaddingMdl.MappedSystemVa = (PVOID)Padding; + PaddingMdl.ByteCount = PaddingLen; +#pragma warning(suppress : 28145) /* We're modifying MdlFlags manually, but that's the whole point of this hack */ + PaddingMdl.MdlFlags = MDL_MAPPED_TO_SYSTEM_VA | MDL_SOURCE_IS_NONPAGED_POOL; + LastMdl->Next = &PaddingMdl; + LastMdl->ByteCount = NeededLen; + NET_BUFFER_DATA_LENGTH(NbIn) += PaddingLen; + } + BOOLEAN Ret = ChaCha20Poly1305EncryptMdl( + OutBuffer, + NET_BUFFER_CURRENT_MDL(NbIn), + NET_BUFFER_DATA_LENGTH(NbIn), + NET_BUFFER_CURRENT_MDL_OFFSET(NbIn), + NULL, + 0, + NET_BUFFER_NONCE(NbOut), + Keypair->Sending.Key, + Simd); + NET_BUFFER_DATA_LENGTH(NbOut) = MessageDataLen(NET_BUFFER_DATA_LENGTH(NbIn)); + NET_BUFFER_DATA_OFFSET(NbOut) = NET_BUFFER_CURRENT_MDL_OFFSET(NbOut) = 0; + if (PaddingLen) + { + if (NbIn != NbOut) + NET_BUFFER_DATA_LENGTH(NbIn) -= PaddingLen; + LastMdl->Next = OriginalNextMdl; + LastMdl->ByteCount = OriginalMdlLen; + } + return Ret; +} + +_Use_decl_annotations_ +VOID +PacketSendKeepalive(WG_PEER *Peer) +{ + NET_BUFFER_LIST *Nbl; + + if (NetBufferListIsQueueEmpty(&Peer->StagedPacketQueue)) + { + Nbl = MemAllocateNetBufferList( + Peer->Device->NblPool, Peer->Device->NbPool, 0, 0, sizeof(MESSAGE_DATA) + NoiseEncryptedLen(0)); + if (!Nbl) + return; + Nbl->ParentNetBufferList = Nbl; + NetBufferListInterlockedEnqueue(&Peer->StagedPacketQueue, Nbl); + CHAR EndpointName[SOCKADDR_STR_MAX_LEN]; + SockaddrToString(EndpointName, &Peer->Endpoint.Addr); + LogInfoRatelimited(Peer->Device, "Sending keepalive packet to peer %llu (%s)", Peer->InternalId, EndpointName); + } + + PacketSendStagedPackets(Peer); +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +static VOID +PacketCreateDataDone(_Inout_ WG_PEER *Peer, _Inout_ NET_BUFFER_LIST *First) +{ + BOOLEAN IsKeepalive; + + TimersAnyAuthenticatedPacketTraversal(Peer); + TimersAnyAuthenticatedPacketSent(Peer); + + if (NT_SUCCESS(SocketSendNblsToPeer(Peer, First, &IsKeepalive)) && !IsKeepalive) + TimersDataSent(Peer); + + KeepKeyFresh(Peer); +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +static BOOLEAN +PacketPeerTxWork(_Inout_ WG_PEER *Peer, _In_ ULONG Budget) +{ + NOISE_KEYPAIR *Keypair; + PACKET_STATE State; + NET_BUFFER_LIST *First; + + while ((First = PrevQueuePeek(&Peer->TxQueue)) != NULL && + (State = ReadAcquire(NET_BUFFER_LIST_CRYPT_STATE(First))) != PACKET_STATE_UNCRYPTED) + { + if (!Budget--) + return TRUE; + PrevQueueDropPeeked(&Peer->TxQueue); + Keypair = NET_BUFFER_LIST_KEYPAIR(First); + + if (State == PACKET_STATE_CRYPTED) + PacketCreateDataDone(Peer, First); + else + FreeSendNetBufferList(Peer->Device, First, 0); + + NoiseKeypairPut(Keypair, FALSE); + ExReleaseRundownProtection(&Peer->InUse); + PeerPut(Peer); + } + return FALSE; +} + +_Use_decl_annotations_ +VOID +PacketTxWorker(MULTICORE_WORKQUEUE *WorkQueue) +{ + WG_DEVICE *Wg = CONTAINING_RECORD(WorkQueue, WG_DEVICE, TxThreads); + PEER_SERIAL_ENTRY *Entry; + + while ((Entry = PeerSerialDequeue(&Wg->TxQueue)) != NULL) + PeerSerialMaybeRetire( + &Wg->TxQueue, + Entry, + PacketPeerTxWork(CONTAINING_RECORD(Entry, WG_PEER, TxSerialEntry), PEER_XMIT_PACKETS_PER_ROUND)); +} + +_Use_decl_annotations_ +VOID +PacketEncryptWorker(MULTICORE_WORKQUEUE *WorkQueue) +{ + WG_DEVICE *Wg = CONTAINING_RECORD(WorkQueue, WG_DEVICE, EncryptThreads); + PTR_RING *Ring = &Wg->EncryptQueue; + NET_BUFFER_LIST *First; + SIMD_STATE Simd; + + SimdGet(&Simd); + while ((First = PtrRingConsume(Ring)) != NULL) + { + PACKET_STATE State = PACKET_STATE_CRYPTED; + NOISE_KEYPAIR *Keypair = NET_BUFFER_LIST_KEYPAIR(First); + WG_PEER *Peer = NET_BUFFER_LIST_PEER(First); + ULONG Mtu = ReadULongNoFence(&Wg->Mtu); + + for (NET_BUFFER_LIST *Nbl = First; Nbl; Nbl = NET_BUFFER_LIST_NEXT_NBL(Nbl)) + { + for (NET_BUFFER *NbIn = NET_BUFFER_LIST_FIRST_NB(Nbl->ParentNetBufferList), + *NbOut = NET_BUFFER_LIST_FIRST_NB(Nbl); + NbIn && NbOut; + NbIn = NET_BUFFER_NEXT_NB(NbIn), NbOut = NET_BUFFER_NEXT_NB(NbOut)) + { + if (!EncryptPacket(&Simd, NbOut, NbIn, Keypair, Mtu)) + { + State = PACKET_STATE_DEAD; + goto enqueue; + } + } + if (Nbl != Nbl->ParentNetBufferList) + { + FreeSendNetBufferList(Wg, Nbl->ParentNetBufferList, 0); + Nbl->ParentNetBufferList = Nbl; + } + } + enqueue: + _Analysis_assume_(First != NULL); + QueueEnqueuePerPeer(&Peer->Device->TxQueue, &Peer->TxSerialEntry, &Peer->Device->TxThreads, First, State); + } + SimdPut(&Simd); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +static VOID +PacketCreateData(_Inout_ WG_PEER *Peer, _In_ NET_BUFFER_LIST *First) +{ + WG_DEVICE *Wg = Peer->Device; + NTSTATUS Ret = STATUS_INVALID_PARAMETER; + + if (!ExAcquireRundownProtection(&Peer->InUse)) + goto cleanup; + + Ret = QueueEnqueuePerDeviceAndPeer(&Wg->EncryptQueue, &Peer->TxQueue, &Wg->EncryptThreads, First); + if (Ret == STATUS_PIPE_BROKEN) + QueueEnqueuePerPeer( + &Peer->Device->TxQueue, &Peer->TxSerialEntry, &Peer->Device->TxThreads, First, PACKET_STATE_DEAD); + if (NT_SUCCESS(Ret) || Ret == STATUS_PIPE_BROKEN) + return; + ExReleaseRundownProtection(&Peer->InUse); + +cleanup: + NoiseKeypairPut(NET_BUFFER_LIST_KEYPAIR(First), FALSE); + FreeSendNetBufferList(Peer->Device, First, 0); + PeerPut(Peer); +} + +_Use_decl_annotations_ +VOID +PacketPurgeStagedPackets(WG_PEER *Peer) +{ + KIRQL Irql; + + KeAcquireSpinLock(&Peer->StagedPacketQueue.Lock, &Irql); + Peer->Device->Statistics.ifOutDiscards += NetBufferListQueueLength(&Peer->StagedPacketQueue); + if (Peer->StagedPacketQueue.Head) + FreeSendNetBufferList(Peer->Device, Peer->StagedPacketQueue.Head, NDIS_SEND_COMPLETE_FLAGS_DISPATCH_LEVEL); + Peer->StagedPacketQueue.Head = Peer->StagedPacketQueue.Tail = NULL; + Peer->StagedPacketQueue.Length = 0; + KeReleaseSpinLock(&Peer->StagedPacketQueue.Lock, Irql); +} + +_Use_decl_annotations_ +VOID +PacketSendStagedPackets(WG_PEER *Peer) +{ + NOISE_KEYPAIR *Keypair; + NET_BUFFER_LIST_QUEUE Packets; + PNET_BUFFER_LIST Nbl; + KIRQL Irql; + + /* Steal the current queue into our local one. */ + NetBufferListInitQueue(&Packets); + KeAcquireSpinLock(&Peer->StagedPacketQueue.Lock, &Irql); + _Analysis_suppress_lock_checking_(Packets.Lock); /* `Packets` is private, lock is not required. */ + NetBufferListSpliceAndReinitQueue(&Peer->StagedPacketQueue, &Packets); + KeReleaseSpinLock(&Peer->StagedPacketQueue.Lock, Irql); + if (NetBufferListIsQueueEmpty(&Packets)) + return; + _Analysis_assume_(Packets.Head != NULL); + + /* First we make sure we have a valid reference to a valid key. */ + Irql = RcuReadLock(); + Keypair = NoiseKeypairGet(RcuDereference(NOISE_KEYPAIR, Peer->Keypairs.CurrentKeypair)); + RcuReadUnlock(Irql); + if (!Keypair) + goto outNokey; + if (!ReadBooleanNoFence(&Keypair->Sending.IsValid)) + goto outNokey; + if (BirthdateHasExpired(Keypair->Sending.Birthdate, REJECT_AFTER_TIME)) + goto outInvalid; + + /* After we know we have a somewhat valid key, we now try to assign + * nonces to all of the packets in the queue. If we can't assign nonces + * for all of them, we just consider it a failure and wait for the next + * handshake. + */ + for (Nbl = Packets.Head; Nbl; Nbl = NET_BUFFER_LIST_NEXT_NBL(Nbl)) + { + for (NET_BUFFER *Nb = NET_BUFFER_LIST_FIRST_NB(Nbl); Nb; Nb = NET_BUFFER_NEXT_NB(Nb)) + { + NET_BUFFER_NONCE(Nb) = InterlockedIncrement64(&Keypair->SendingCounter) - 1; + if (NET_BUFFER_NONCE(Nb) >= REJECT_AFTER_MESSAGES) + goto outInvalid; + } + } + + PeerGet(Keypair->Entry.Peer); + _Analysis_assume_(NET_BUFFER_LIST_FIRST_NB(Packets.Head)); /* Checked in SendNetBufferLists(). */ + NET_BUFFER_LIST_KEYPAIR(Packets.Head) = Keypair; + PacketCreateData(Peer, Packets.Head); + return; + +outInvalid: + WriteBooleanNoFence(&Keypair->Sending.IsValid, FALSE); +outNokey: + NoiseKeypairPut(Keypair, FALSE); + + /* We orphan the packets if we're waiting on a handshake, so that they + * don't block the pool of an upper layer. Then we put them back on the + * end of the queue. We're not too concerned about accidentally getting + * things a little out of order if packets are being added really fast, + * because this queue is for before packets can even be sent and it's + * small anyway. + */ + _Analysis_suppress_lock_checking_(Packets.Lock); /* `Packets` is private, lock is not required. */ + while ((Nbl = NetBufferListDequeue(&Packets)) != NULL) + { + NT_ASSERT(Nbl->ParentNetBufferList); + if (Nbl->ParentNetBufferList == Nbl) + goto requeueOrphan; + + for (NET_BUFFER *NbIn = NET_BUFFER_LIST_FIRST_NB(Nbl->ParentNetBufferList), + *NbOut = NET_BUFFER_LIST_FIRST_NB(Nbl); + NbIn && NbOut; + NbIn = NET_BUFFER_NEXT_NB(NbIn), NbOut = NET_BUFFER_NEXT_NB(NbOut)) + { + VOID *Dst = (UCHAR *)MemGetValidatedNetBufferData(NbOut) + sizeof(MESSAGE_DATA); + VOID *Src = NdisGetDataBuffer(NbIn, NET_BUFFER_DATA_LENGTH(NbIn), Dst, 1, 0); + if (Src != Dst) + RtlCopyMemory(Dst, Src, NET_BUFFER_DATA_LENGTH(NbIn)); + NET_BUFFER_DATA_LENGTH(NbOut) = NET_BUFFER_DATA_LENGTH(NbIn); + NET_BUFFER_DATA_OFFSET(NbOut) = NET_BUFFER_CURRENT_MDL_OFFSET(NbOut) = sizeof(MESSAGE_DATA); + } + FreeSendNetBufferList(Peer->Device, Nbl->ParentNetBufferList, 0); + Nbl->ParentNetBufferList = Nbl; + requeueOrphan: + NetBufferListInterlockedEnqueue(&Peer->StagedPacketQueue, Nbl); + } + + /* If we're exiting because there's something wrong with the key, it + * means we should initiate a new handshake. + */ + PacketSendQueuedHandshakeInitiation(Peer, FALSE); +} + +#pragma warning( \ + suppress : 6014) /* `Nbl` is returned in NdisMSendNetBufferListsComplete or freed in MemFreeNetBufferList. */ +_Use_decl_annotations_ +VOID +FreeSendNetBufferList(WG_DEVICE *Wg, NET_BUFFER_LIST *FirstNbl, ULONG SendCompleteFlags) +{ + for (NET_BUFFER_LIST *Nbl = FirstNbl, *NextNbl; Nbl; Nbl = NextNbl) + { + NextNbl = NET_BUFFER_LIST_NEXT_NBL(Nbl); + NET_BUFFER_LIST_NEXT_NBL(Nbl) = NULL; + if (Nbl->NdisPoolHandle == Wg->NblPool) + { + if (Nbl->ParentNetBufferList != Nbl) + { + NET_BUFFER_LIST_STATUS(Nbl->ParentNetBufferList) = NET_BUFFER_LIST_STATUS(Nbl); + NdisMSendNetBufferListsComplete(Wg->MiniportAdapterHandle, Nbl->ParentNetBufferList, SendCompleteFlags); + ExReleaseRundownProtection(&Wg->ItemsInFlight); + Nbl->ParentNetBufferList = NULL; + } + MemFreeNetBufferList(Nbl); + } + else + { + NdisMSendNetBufferListsComplete(Wg->MiniportAdapterHandle, Nbl, SendCompleteFlags); + ExReleaseRundownProtection(&Wg->ItemsInFlight); + } + } +} diff --git a/driver/socket.c b/driver/socket.c new file mode 100644 index 0000000..e75a91e --- /dev/null +++ b/driver/socket.c @@ -0,0 +1,991 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +/* We pretend we're Windows 8, and then hack around the limitation in Windows 7 below. */ +#include <ntifs.h> /* Must be included before <wdm.h> */ +#include <wdm.h> +#if NTDDI_VERSION == NTDDI_WIN7 +# undef NTDDI_VERSION +# define NTDDI_VERSION NTDDI_WIN8 +# include <wsk.h> +# undef NTDDI_VERSION +# define NTDDI_VERSION NTDDI_WIN7 +#endif + +#include "device.h" +#include "messages.h" +#include "peer.h" +#include "queueing.h" +#include "rcu.h" +#include "socket.h" +#include "logging.h" +#include <wsk.h> +#include <netioapi.h> +/* The headers say that UDP_NOCHECKSUM is defined in ws2tcpip.h for hysterical raisins, + * which we can't include from kernel space. + */ +#define UDP_NOCHECKSUM 1 + +static LONG RoutingGenerationV4 = 1, RoutingGenerationV6 = 1; +static HANDLE RouteNotifierV4, RouteNotifierV6; +static CONST WSK_CLIENT_DISPATCH WskAppDispatchV1 = { .Version = MAKE_WSK_VERSION(1, 0) }; +static WSK_REGISTRATION WskRegistration; +static WSK_PROVIDER_NPI WskProviderNpi; +static BOOLEAN WskHasIpv4Transport, WskHasIpv6Transport; +static NTSTATUS WskInitStatus = STATUS_RETRY; +static EX_PUSH_LOCK WskIsIniting; +static LOOKASIDE_ALIGN LOOKASIDE_LIST_EX SocketSendCtxCache; + +#define NET_BUFFER_WSK_BUF(Nb) ((WSK_BUF_LIST *)&NET_BUFFER_MINIPORT_RESERVED(Nb)[0]) +static_assert( + sizeof(NET_BUFFER_MINIPORT_RESERVED((NET_BUFFER *)0)) >= sizeof(WSK_BUF_LIST), + "WSK_BUF_LIST is too large for NB"); + +typedef struct _SOCKET_SEND_CTX +{ + IRP Irp; + IO_STACK_LOCATION IoStackData; + ENDPOINT Endpoint; + WG_DEVICE *Wg; + union + { + NET_BUFFER_LIST *FirstNbl; + WSK_BUF Buffer; + }; + BOOLEAN IsNbl; +} SOCKET_SEND_CTX; + +static IO_COMPLETION_ROUTINE SendComplete; +_Use_decl_annotations_ +static NTSTATUS +SendComplete(DEVICE_OBJECT *DeviceObject, IRP *Irp, VOID *VoidCtx) +{ + SOCKET_SEND_CTX *Ctx = VoidCtx; + _Analysis_assume_(Ctx); + if (Ctx->IsNbl) + FreeSendNetBufferList(Ctx->Wg, Ctx->FirstNbl, 0); + else + MemFreeDataAndMdlChain(Ctx->Buffer.Mdl); + ExFreeToLookasideListEx(&SocketSendCtxCache, Ctx); + return STATUS_MORE_PROCESSING_REQUIRED; +} + +#if NTDDI_VERSION == NTDDI_WIN7 +static BOOLEAN NoWskSendMessages; + +typedef struct _POLYFILLED_SOCKET_SEND_CTX +{ + IRP Irp; + IO_STACK_LOCATION IoStackData; + PIRP OriginalIrp; + LONG *RefCount; +} POLYFILLED_SOCKET_SEND_CTX; + +static IO_COMPLETION_ROUTINE PolyfilledSendComplete; +_Use_decl_annotations_ +static NTSTATUS +PolyfilledSendComplete(DEVICE_OBJECT *DeviceObject, IRP *Irp, VOID *VoidCtx) +{ + POLYFILLED_SOCKET_SEND_CTX *Ctx = VoidCtx; + _Analysis_assume_(Ctx); + if (!InterlockedDecrement(Ctx->RefCount)) + { + IO_STACK_LOCATION *Stack = IoGetNextIrpStackLocation(Ctx->OriginalIrp); + if (Stack && Stack->CompletionRoutine) + Stack->CompletionRoutine(DeviceObject, Ctx->OriginalIrp, Stack->Context); + MemFree(Ctx->RefCount); + } + MemFree(Ctx); + return STATUS_MORE_PROCESSING_REQUIRED; +} + +static NTSTATUS +PolyfilledWskSendMessages( + _In_ PWSK_SOCKET Socket, + _In_ PWSK_BUF_LIST BufferList, + _Reserved_ ULONG Flags, + _In_opt_ PSOCKADDR RemoteAddress, + _In_ ULONG ControlInfoLength, + _In_reads_bytes_opt_(ControlInfoLength) PCMSGHDR ControlInfo, + _Inout_ PIRP Irp) +{ +# pragma warning(suppress : 6014) /* `RefCount` is freed in PolyfilledSendComplete. */ + LONG *RefCount = MemAllocate(sizeof(*RefCount)); + if (!RefCount) + return STATUS_INSUFFICIENT_RESOURCES; + WriteNoFence(RefCount, 1); + for (WSK_BUF_LIST *Buf = BufferList; Buf; Buf = Buf->Next) + { + POLYFILLED_SOCKET_SEND_CTX *Ctx = MemAllocate(sizeof(*Ctx)); + if (!Ctx) + continue; + Ctx->RefCount = RefCount; + Ctx->OriginalIrp = Irp; + IoInitializeIrp(&Ctx->Irp, sizeof(Ctx->IoStackData) + sizeof(Ctx->Irp), 1); + IoSetCompletionRoutine(&Ctx->Irp, PolyfilledSendComplete, Ctx, TRUE, TRUE, TRUE); + InterlockedIncrement(RefCount); + ((WSK_PROVIDER_DATAGRAM_DISPATCH *)Socket->Dispatch) + ->WskSendTo(Socket, &Buf->Buffer, Flags, RemoteAddress, ControlInfoLength, ControlInfo, &Ctx->Irp); + } + if (!InterlockedDecrement(RefCount)) + { + IO_STACK_LOCATION *Stack = IoGetNextIrpStackLocation(Irp); + if (Stack && Stack->CompletionRoutine) + Stack->CompletionRoutine((DEVICE_OBJECT *)Socket, Irp, Stack->Context); + MemFree(RefCount); + } + return STATUS_SUCCESS; +} +#endif + +/* This function expects to have Ctx's Endpoint and *either* Buffer or FirstNbl + * fields filled; it takes care of the rest. It will free/consume Ctx when + * STATUS_SUCCESS is returned. Note that STATUS_SUCCESS means the actual + * sending might fail asynchronously later on. + */ +_IRQL_requires_max_(DISPATCH_LEVEL) +static NTSTATUS +SendAsync(_In_ WG_DEVICE *Wg, _In_ __drv_aliasesMem SOCKET_SEND_CTX *Ctx) +{ + SOCKET *Socket = NULL; + NTSTATUS Status; + + Ctx->Wg = Wg; + IoInitializeIrp(&Ctx->Irp, sizeof(Ctx->IoStackData) + sizeof(Ctx->Irp), 1); + IoSetCompletionRoutine(&Ctx->Irp, SendComplete, Ctx, TRUE, TRUE, TRUE); + + KIRQL Irql = RcuReadLock(); + if (Ctx->Endpoint.Addr.si_family == AF_INET) + Socket = RcuDereference(SOCKET, Wg->Sock4); + else if (Ctx->Endpoint.Addr.si_family == AF_INET6) + Socket = RcuDereference(SOCKET, Wg->Sock6); + if (!Socket) + { + Status = STATUS_NETWORK_UNREACHABLE; + goto cleanup; + } + PFN_WSK_SEND_MESSAGES WskSendMessages = ((WSK_PROVIDER_DATAGRAM_DISPATCH *)Socket->Sock->Dispatch)->WskSendMessages; +#if NTDDI_VERSION == NTDDI_WIN7 + if (NoWskSendMessages) + WskSendMessages = PolyfilledWskSendMessages; +#endif + if (Ctx->IsNbl) + WskSendMessages( + Socket->Sock, + NET_BUFFER_WSK_BUF(NET_BUFFER_LIST_FIRST_NB(Ctx->FirstNbl)), + 0, + (PSOCKADDR)&Ctx->Endpoint.Addr, + (ULONG)Ctx->Endpoint.SrcCmsghdr.cmsg_len, + &Ctx->Endpoint.SrcCmsghdr, + &Ctx->Irp); + else + ((WSK_PROVIDER_DATAGRAM_DISPATCH *)Socket->Sock->Dispatch) + ->WskSendTo( + Socket->Sock, + &Ctx->Buffer, + 0, + (PSOCKADDR)&Ctx->Endpoint.Addr, + (ULONG)Ctx->Endpoint.SrcCmsghdr.cmsg_len, + &Ctx->Endpoint.SrcCmsghdr, + &Ctx->Irp); + Status = STATUS_SUCCESS; +cleanup: + RcuReadUnlock(Irql); + return Status; +} + +static BOOLEAN +CidrMaskMatchV4(_In_ CONST IN_ADDR *Addr, _In_ CONST IP_ADDRESS_PREFIX *Prefix) +{ + return Prefix->PrefixLength == 0 || + (Addr->s_addr & (~0U << (32 - Prefix->PrefixLength))) == Prefix->Prefix.Ipv4.sin_addr.s_addr; +} + +static BOOLEAN +CidrMaskMatchV6(_In_ CONST IN6_ADDR *Addr, _In_ CONST IP_ADDRESS_PREFIX *Prefix) +{ + if (Prefix->PrefixLength == 0) + return TRUE; + ULONG WholeParts = Prefix->PrefixLength / 32; + ULONG LeftoverBits = Prefix->PrefixLength % 32; + if (!RtlEqualMemory(&Prefix->Prefix.Ipv6.sin6_addr, Addr, WholeParts * sizeof(UINT32))) + return FALSE; + if (WholeParts == 4 || LeftoverBits == 0) + return TRUE; + return (((UINT32 *)Addr)[WholeParts] & (~0U << (32 - LeftoverBits))) == + ((UINT32 *)&Prefix->Prefix.Ipv6.sin6_addr)[WholeParts]; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +_IRQL_raises_(DISPATCH_LEVEL) +_Requires_lock_not_held_(Peer->EndpointLock) +_Acquires_lock_(Peer->EndpointLock) +static NTSTATUS +SocketResolvePeerEndpointSrc(_Inout_ WG_PEER *Peer, _Out_ _At_(*Irql, _IRQL_saves_) KIRQL *Irql) +{ + NTSTATUS Status = STATUS_INVALID_PARAMETER; + ENDPOINT *Endpoint = &Peer->Endpoint; + UINT32 UpdateGeneration; + + /* TODO: We should probably cache the results of this to avoid a DoS, + * whereby a client sends pings that change src address, resulting in new + * lookups on the pong. + */ +retry: + *Irql = ExAcquireSpinLockShared(&Peer->EndpointLock); + UpdateGeneration = Endpoint->UpdateGeneration; + if (Endpoint->Addr.si_family == AF_INET && + Endpoint->RoutingGeneration == (UINT32)ReadNoFence(&RoutingGenerationV4) && Endpoint->Src4.ipi_ifindex) + return STATUS_SUCCESS; + if (Endpoint->Addr.si_family == AF_INET6 && + Endpoint->RoutingGeneration == (UINT32)ReadNoFence(&RoutingGenerationV6) && Endpoint->Src6.ipi6_ifindex) + return STATUS_SUCCESS; + SOCKADDR_INET SrcAddr = { 0 }; + ExReleaseSpinLockShared(&Peer->EndpointLock, *Irql); + ULONG BestIndex = 0, BestCidr = 0, BestMetric = ~0UL; + NET_LUID BestLuid = { 0 }; + MIB_IPFORWARD_TABLE2 *Table; + Status = GetIpForwardTable2(Endpoint->Addr.si_family, &Table); + if (!NT_SUCCESS(Status)) + return Status; + for (ULONG i = 0; i < Table->NumEntries; ++i) + { + if (Table->Table[i].InterfaceLuid.Value == Peer->Device->InterfaceLuid.Value) + continue; + if (Table->Table[i].DestinationPrefix.PrefixLength < BestCidr) + continue; + if (Endpoint->Addr.si_family == AF_INET && + !CidrMaskMatchV4(&Endpoint->Addr.Ipv4.sin_addr, &Table->Table[i].DestinationPrefix)) + continue; + if (Endpoint->Addr.si_family == AF_INET6 && + !CidrMaskMatchV6(&Endpoint->Addr.Ipv6.sin6_addr, &Table->Table[i].DestinationPrefix)) + continue; + MIB_IPINTERFACE_ROW Interface = { .Family = Endpoint->Addr.si_family, + .InterfaceLuid = Table->Table[i].InterfaceLuid }; + if (!NT_SUCCESS(GetIpInterfaceEntry(&Interface))) + continue; + ULONG Metric = Table->Table[i].Metric + Interface.Metric; + if (Table->Table[i].DestinationPrefix.PrefixLength == BestCidr && Metric > BestMetric) + continue; + BestCidr = Table->Table[i].DestinationPrefix.PrefixLength; + BestMetric = Metric; + BestIndex = Table->Table[i].InterfaceIndex; + BestLuid = Table->Table[i].InterfaceLuid; + } + /* We disable wg-loop routing for now, to prevent stack overflow. TODO: revisit later. */ + Status = STATUS_SUCCESS; + MuAcquirePushLockShared(&DeviceListLock); + WG_DEVICE *Wg; + LIST_FOR_EACH_ENTRY (Wg, &DeviceList, WG_DEVICE, DeviceList) + { + if (Wg->InterfaceLuid.Value == BestLuid.Value) + { + LogInfoRatelimited(Peer->Device, "Dropping packet that would egress via interface %u", Wg->InterfaceIndex); + Status = STATUS_RECURSIVE_DISPATCH; + break; + } + } + MuReleasePushLockShared(&DeviceListLock); + if (NT_SUCCESS(Status) && Table->NumEntries && BestIndex) + Status = GetBestRoute2(NULL, BestIndex, NULL, &Endpoint->Addr, 0, &Table->Table[0], &SrcAddr); + FreeMibTable(Table); + if (!NT_SUCCESS(Status)) + return Status; + *Irql = ExAcquireSpinLockExclusive(&Peer->EndpointLock); + if (Endpoint->UpdateGeneration != UpdateGeneration) + { + ExReleaseSpinLockExclusive(&Peer->EndpointLock, *Irql); + goto retry; + } + if (Endpoint->Addr.si_family == AF_INET) + { + Endpoint->RoutingGeneration = ReadNoFence(&RoutingGenerationV4); + Endpoint->Src4.ipi_addr = SrcAddr.Ipv4.sin_addr; + Endpoint->Src4.ipi_ifindex = BestIndex; + Endpoint->SrcCmsghdr.cmsg_len = WSA_CMSG_LEN(sizeof(Endpoint->Src4)); + Endpoint->SrcCmsghdr.cmsg_level = IPPROTO_IP; + Endpoint->SrcCmsghdr.cmsg_type = IP_PKTINFO; + } + else if (Endpoint->Addr.si_family == AF_INET6) + { + Endpoint->RoutingGeneration = ReadNoFence(&RoutingGenerationV6); + Endpoint->Src6.ipi6_addr = SrcAddr.Ipv6.sin6_addr; + Endpoint->Src6.ipi6_ifindex = BestIndex; + Endpoint->SrcCmsghdr.cmsg_len = WSA_CMSG_LEN(sizeof(Endpoint->Src6)); + Endpoint->SrcCmsghdr.cmsg_level = IPPROTO_IPV6; + Endpoint->SrcCmsghdr.cmsg_type = IPV6_PKTINFO; + } + else + BestIndex = 0; + ++Endpoint->UpdateGeneration; + ExReleaseSpinLockExclusive(&Peer->EndpointLock, *Irql); + if (!BestIndex) + return STATUS_BAD_NETWORK_PATH; + *Irql = ExAcquireSpinLockShared(&Peer->EndpointLock); + if (Endpoint->UpdateGeneration != UpdateGeneration) + { + ExReleaseSpinLockShared(&Peer->EndpointLock, *Irql); + goto retry; + } + return STATUS_SUCCESS; +} + +#pragma warning(suppress : 28194) /* `Nbl` is aliased in Ctx->Nbl or freed on failure. */ +#pragma warning(suppress : 28167) /* IRQL is either not raised on SocketResolvePeerEndpointSrc failure, or \ + restored by ExReleaseSpinLockShared */ +_Use_decl_annotations_ +NTSTATUS +SocketSendNblsToPeer(WG_PEER *Peer, NET_BUFFER_LIST *First, BOOLEAN *AllKeepalive) +{ + if (!First) + return STATUS_ALREADY_COMPLETE; + NTSTATUS Status = STATUS_INSUFFICIENT_RESOURCES; + SOCKET_SEND_CTX *Ctx = ExAllocateFromLookasideListEx(&SocketSendCtxCache); + if (!Ctx) + goto cleanupNbls; + KIRQL Irql; + Status = SocketResolvePeerEndpointSrc(Peer, &Irql); /* Takes read-side EndpointLock */ + if (!NT_SUCCESS(Status)) + goto cleanupCtx; + Ctx->Endpoint = Peer->Endpoint; + ExReleaseSpinLockShared(&Peer->EndpointLock, Irql); + Ctx->IsNbl = TRUE; + Ctx->FirstNbl = First; + *AllKeepalive = TRUE; + + WSK_BUF_LIST *FirstWskBuf = NULL, *LastWskBuf = NULL; + ULONG64 DataLength = 0, Packets = 0; + for (NET_BUFFER_LIST *Nbl = First; Nbl; Nbl = NET_BUFFER_LIST_NEXT_NBL(Nbl)) + { + for (NET_BUFFER *Nb = NET_BUFFER_LIST_FIRST_NB(Nbl); Nb; Nb = NET_BUFFER_NEXT_NB(Nb)) + { + NET_BUFFER_WSK_BUF(Nb)->Buffer.Mdl = NET_BUFFER_CURRENT_MDL(Nb); + NET_BUFFER_WSK_BUF(Nb)->Buffer.Length = NET_BUFFER_DATA_LENGTH(Nb); + NET_BUFFER_WSK_BUF(Nb)->Buffer.Offset = NET_BUFFER_CURRENT_MDL_OFFSET(Nb); + NET_BUFFER_WSK_BUF(Nb)->Next = NULL; + *(LastWskBuf ? &LastWskBuf->Next : &FirstWskBuf) = NET_BUFFER_WSK_BUF(Nb); + LastWskBuf = NET_BUFFER_WSK_BUF(Nb); + DataLength += NET_BUFFER_DATA_LENGTH(Nb); + ++Packets; + if (NET_BUFFER_DATA_LENGTH(Nb) != MessageDataLen(0)) + *AllKeepalive = FALSE; + } + } + Status = SendAsync(Peer->Device, Ctx); + if (!NT_SUCCESS(Status)) + goto cleanupCtx; + Peer->TxBytes += DataLength; + Peer->Device->Statistics.ifHCOutOctets += DataLength; + Peer->Device->Statistics.ifHCOutUcastOctets += DataLength; + Peer->Device->Statistics.ifHCOutUcastPkts += Packets; + return STATUS_SUCCESS; + +cleanupCtx: + ExFreeToLookasideListEx(&SocketSendCtxCache, Ctx); +cleanupNbls: + FreeSendNetBufferList(Peer->Device, First, 0); + return Status; +} + +#pragma warning(suppress : 28167) /* IRQL is either not raised on SocketResolvePeerEndpointSrc failure, or \ + restored by ExReleaseSpinLockShared */ +_Use_decl_annotations_ +NTSTATUS +SocketSendBufferToPeer(WG_PEER *Peer, CONST VOID *Buffer, ULONG Len) +{ + NTSTATUS Status; + SOCKET_SEND_CTX *Ctx = ExAllocateFromLookasideListEx(&SocketSendCtxCache); + if (!Ctx) + return STATUS_INSUFFICIENT_RESOURCES; + Ctx->IsNbl = FALSE; + Ctx->Buffer.Length = Len; + Ctx->Buffer.Offset = 0; + Ctx->Buffer.Mdl = MemAllocateDataAndMdlChain(Len); + if (!Ctx->Buffer.Mdl) + { + Status = STATUS_INSUFFICIENT_RESOURCES; + goto cleanupCtx; + } + RtlCopyMemory(MmGetMdlVirtualAddress(Ctx->Buffer.Mdl), Buffer, Len); + KIRQL Irql; + Status = SocketResolvePeerEndpointSrc(Peer, &Irql); /* Takes read-side endpoint_lock */ + if (!NT_SUCCESS(Status)) + goto cleanupMdl; + Ctx->Endpoint = Peer->Endpoint; + ExReleaseSpinLockShared(&Peer->EndpointLock, Irql); + Status = SendAsync(Peer->Device, Ctx); + if (!NT_SUCCESS(Status)) + goto cleanupMdl; + Peer->TxBytes += Len; + return STATUS_SUCCESS; +cleanupMdl: + MemFreeDataAndMdlChain(Ctx->Buffer.Mdl); +cleanupCtx: + ExFreeToLookasideListEx(&SocketSendCtxCache, Ctx); + return Status; +} + +_Use_decl_annotations_ +NTSTATUS +SocketSendBufferAsReplyToNbl(WG_DEVICE *Wg, CONST NET_BUFFER_LIST *InNbl, CONST VOID *Buffer, ULONG Len) +{ + NTSTATUS Status; + SOCKET_SEND_CTX *Ctx = ExAllocateFromLookasideListEx(&SocketSendCtxCache); + if (!Ctx) + return STATUS_INSUFFICIENT_RESOURCES; + Ctx->IsNbl = FALSE; + Ctx->Buffer.Length = Len; + Ctx->Buffer.Offset = 0; + Ctx->Buffer.Mdl = MemAllocateDataAndMdlChain(Len); + if (!Ctx->Buffer.Mdl) + { + Status = STATUS_INSUFFICIENT_RESOURCES; + goto cleanupCtx; + } + RtlCopyMemory(MmGetMdlVirtualAddress(Ctx->Buffer.Mdl), Buffer, Len); + Status = SocketEndpointFromNbl(&Ctx->Endpoint, InNbl); + if (!NT_SUCCESS(Status)) + goto cleanupMdl; + Status = SendAsync(Wg, Ctx); + if (!NT_SUCCESS(Status)) + goto cleanupMdl; + return STATUS_SUCCESS; +cleanupMdl: + MemFreeDataAndMdlChain(Ctx->Buffer.Mdl); +cleanupCtx: + ExFreeToLookasideListEx(&SocketSendCtxCache, Ctx); + return Status; +} + +_Post_maybenull_ +static VOID * +FindInCmsgHdr(_In_ WSK_DATAGRAM_INDICATION *Data, _In_ CONST INT Level, _In_ CONST INT Type) +{ + SIZE_T Len = Data->ControlInfoLength; + WSACMSGHDR *Hdr = Data->ControlInfo; + + while (Len > 0 && Hdr) + { + if (Hdr->cmsg_level == Level && Hdr->cmsg_type == Type) + return (VOID *)WSA_CMSG_DATA(Hdr); + Len -= WSA_CMSGHDR_ALIGN(Hdr->cmsg_len); + Hdr = (WSACMSGHDR *)((UCHAR *)Hdr + WSA_CMSGHDR_ALIGN(Hdr->cmsg_len)); + } + return NULL; +} + +_Use_decl_annotations_ +NTSTATUS +SocketEndpointFromNbl(ENDPOINT *Endpoint, CONST NET_BUFFER_LIST *Nbl) +{ + WSK_DATAGRAM_INDICATION *Data = NET_BUFFER_LIST_DATAGRAM_INDICATION(Nbl); + SOCKADDR *Addr = Data->RemoteAddress; + VOID *Pktinfo; + RtlZeroMemory(Endpoint, sizeof(*Endpoint)); + if (Addr->sa_family == AF_INET && (Pktinfo = FindInCmsgHdr(Data, IPPROTO_IP, IP_PKTINFO)) != NULL) + { + Endpoint->Addr.Ipv4 = *(SOCKADDR_IN *)Addr; + Endpoint->Src4 = *(IN_PKTINFO *)Pktinfo; + Endpoint->RoutingGeneration = ReadNoFence(&RoutingGenerationV4); + Endpoint->SrcCmsghdr.cmsg_len = WSA_CMSG_LEN(sizeof(Endpoint->Src4)); + Endpoint->SrcCmsghdr.cmsg_level = IPPROTO_IP; + Endpoint->SrcCmsghdr.cmsg_type = IP_PKTINFO; + } + else if (Addr->sa_family == AF_INET6 && (Pktinfo = FindInCmsgHdr(Data, IPPROTO_IPV6, IPV6_PKTINFO)) != NULL) + { + Endpoint->Addr.Ipv6 = *(SOCKADDR_IN6 *)Addr; + Endpoint->Src6 = *(IN6_PKTINFO *)Pktinfo; + Endpoint->RoutingGeneration = ReadNoFence(&RoutingGenerationV6); + Endpoint->SrcCmsghdr.cmsg_len = WSA_CMSG_LEN(sizeof(Endpoint->Src6)); + Endpoint->SrcCmsghdr.cmsg_level = IPPROTO_IPV6; + Endpoint->SrcCmsghdr.cmsg_type = IPV6_PKTINFO; + } + else + return STATUS_INVALID_ADDRESS; + return STATUS_SUCCESS; +} + +static inline BOOLEAN +Ipv6AddrEq(_In_ CONST IN6_ADDR *A1, _In_ CONST IN6_ADDR *A2) +{ + UINT64 *B1 = (UINT64 *)A1, *B2 = (UINT64 *)A2; + return ((B1[0] ^ B2[0]) | (B1[1] ^ B2[1])) == 0; +} + +static BOOLEAN +EndpointEq(_In_ CONST ENDPOINT *A, _In_ CONST ENDPOINT *B) +{ + return (A->Addr.si_family == AF_INET && B->Addr.si_family == AF_INET && + A->Addr.Ipv4.sin_port == B->Addr.Ipv4.sin_port && + A->Addr.Ipv4.sin_addr.s_addr == B->Addr.Ipv4.sin_addr.s_addr && + A->Src4.ipi_addr.s_addr == B->Src4.ipi_addr.s_addr && A->Src4.ipi_ifindex == B->Src4.ipi_ifindex) || + (A->Addr.si_family == AF_INET6 && B->Addr.si_family == AF_INET6 && + A->Addr.Ipv6.sin6_port == B->Addr.Ipv6.sin6_port && + Ipv6AddrEq(&A->Addr.Ipv6.sin6_addr, &B->Addr.Ipv6.sin6_addr) && + A->Addr.Ipv6.sin6_scope_id == B->Addr.Ipv6.sin6_scope_id && + Ipv6AddrEq(&A->Src6.ipi6_addr, &B->Src6.ipi6_addr) && A->Src6.ipi6_ifindex == B->Src6.ipi6_ifindex) || + !A->Addr.si_family && !B->Addr.si_family; +} + +_Use_decl_annotations_ +VOID +SocketSetPeerEndpoint(WG_PEER *Peer, CONST ENDPOINT *Endpoint) +{ + KIRQL Irql; + + /* First we check unlocked, in order to optimize, since it's pretty rare + * that an endpoint will change. If we happen to be mid-write, and two + * CPUs wind up writing the same thing or something slightly different, + * it doesn't really matter much either. + */ + if (EndpointEq(Endpoint, &Peer->Endpoint)) + return; + Irql = ExAcquireSpinLockExclusive(&Peer->EndpointLock); + if (Endpoint->Addr.si_family == AF_INET) + { + Peer->Endpoint.Addr.Ipv4 = Endpoint->Addr.Ipv4; + Peer->Endpoint.Src4 = Endpoint->Src4; + Peer->Endpoint.SrcCmsghdr.cmsg_len = WSA_CMSG_LEN(sizeof(Endpoint->Src4)); + Peer->Endpoint.SrcCmsghdr.cmsg_level = IPPROTO_IP; + Peer->Endpoint.SrcCmsghdr.cmsg_type = IP_PKTINFO; + } + else if (Endpoint->Addr.si_family == AF_INET6) + { + Peer->Endpoint.Addr.Ipv6 = Endpoint->Addr.Ipv6; + Peer->Endpoint.Src6 = Endpoint->Src6; + Peer->Endpoint.SrcCmsghdr.cmsg_len = WSA_CMSG_LEN(sizeof(Endpoint->Src6)); + Peer->Endpoint.SrcCmsghdr.cmsg_level = IPPROTO_IPV6; + Peer->Endpoint.SrcCmsghdr.cmsg_type = IPV6_PKTINFO; + } + else + { + goto out; + } + Peer->Endpoint.RoutingGeneration = Endpoint->RoutingGeneration; + ++Peer->Endpoint.UpdateGeneration; +out: + ExReleaseSpinLockExclusive(&Peer->EndpointLock, Irql); +} + +_Use_decl_annotations_ +VOID +SocketSetPeerEndpointFromNbl(WG_PEER *Peer, CONST NET_BUFFER_LIST *Nbl) +{ + ENDPOINT Endpoint; + + if (NT_SUCCESS(SocketEndpointFromNbl(&Endpoint, Nbl))) + SocketSetPeerEndpoint(Peer, &Endpoint); +} + +_Use_decl_annotations_ +VOID +SocketClearPeerEndpointSrc(WG_PEER *Peer) +{ + KIRQL Irql; + + Irql = ExAcquireSpinLockExclusive(&Peer->EndpointLock); + Peer->Endpoint.RoutingGeneration = 0; + ++Peer->Endpoint.UpdateGeneration; + RtlZeroMemory(&Peer->Endpoint.Src6, sizeof(Peer->Endpoint.Src6)); + ExReleaseSpinLockExclusive(&Peer->EndpointLock, Irql); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +_Must_inspect_result_ +static NTSTATUS WSKAPI +Receive(_In_opt_ PVOID SocketContext, _In_ ULONG Flags, _In_opt_ WSK_DATAGRAM_INDICATION *DataIndication) +{ + SOCKET *Socket = SocketContext; + if (!Socket || !Socket->Sock || !DataIndication) + return STATUS_SUCCESS; + WG_DEVICE *Wg = Socket->Device; + NET_BUFFER_LIST *First = NULL, **Link = &First; + for (WSK_DATAGRAM_INDICATION *DataIndicationNext; DataIndication; DataIndication = DataIndicationNext) + { + DataIndicationNext = DataIndication->Next; + DataIndication->Next = NULL; + NET_BUFFER_LIST *Nbl = NULL; + ULONG Length; + if (!NT_SUCCESS(RtlSIZETToULong(DataIndication->Buffer.Length, &Length))) + goto skipDatagramIndication; + Nbl = MemAllocateNetBufferList(Wg->NblPool, Wg->NbPool, 0, Length, 0); + if (!Nbl || !ReadBooleanNoFence(&Wg->IsUp) || !ExAcquireRundownProtection(&Socket->ItemsInFlight)) + goto skipDatagramIndication; + NET_BUFFER_LIST_DATAGRAM_INDICATION(Nbl) = DataIndication; + DataIndication->Next = (VOID *)Socket; + *Link = Nbl; + Link = &NET_BUFFER_LIST_NEXT_NBL(Nbl); + continue; + + skipDatagramIndication: + ((WSK_PROVIDER_DATAGRAM_DISPATCH *)Socket->Sock->Dispatch)->WskRelease(Socket->Sock, DataIndication); + if (Nbl) + MemFreeNetBufferList(Nbl); + ++Wg->Statistics.ifInDiscards; + } + if (First) + PacketReceive(Wg, First); + return STATUS_PENDING; +} + +static IO_COMPLETION_ROUTINE RaiseEventOnComplete; +_Use_decl_annotations_ +static NTSTATUS +RaiseEventOnComplete(PDEVICE_OBJECT DeviceObject, PIRP Irp, PVOID Context) +{ + _Analysis_assume_(Context); + KeSetEvent((KEVENT *)Context, IO_NETWORK_INCREMENT, FALSE); + return STATUS_MORE_PROCESSING_REQUIRED; +} + +_IRQL_requires_max_(APC_LEVEL) +static VOID +CloseSocket(_Frees_ptr_opt_ SOCKET *Socket) +{ + if (!Socket) + return; + ExWaitForRundownProtectionRelease(&Socket->ItemsInFlight); + if (Socket->Sock) + { + KEVENT Done; + KeInitializeEvent(&Done, SynchronizationEvent, FALSE); + IRP *Irp = IoAllocateIrp(1, FALSE); + if (!Irp) + goto freeIt; + IoSetCompletionRoutine(Irp, RaiseEventOnComplete, &Done, TRUE, TRUE, TRUE); + NTSTATUS Status = ((WSK_PROVIDER_DATAGRAM_DISPATCH *)Socket->Sock->Dispatch)->WskCloseSocket(Socket->Sock, Irp); + if (Status == STATUS_PENDING) + { + KeWaitForSingleObject(&Done, Executive, KernelMode, FALSE, NULL); + Status = Irp->IoStatus.Status; + } + IoFreeIrp(Irp); + } +freeIt: + MemFree(Socket); +} + +_IRQL_requires_max_(APC_LEVEL) +static NTSTATUS +SetSockOpt( + _In_ WSK_SOCKET *Sock, + _In_ ULONG Level, + _In_ ULONG Option, + _In_reads_bytes_(Len) VOID *Input, + _In_ ULONG Len) +{ + KEVENT Done; + KeInitializeEvent(&Done, SynchronizationEvent, FALSE); + IRP *Irp = IoAllocateIrp(1, FALSE); + if (!Irp) + return STATUS_INSUFFICIENT_RESOURCES; + IoSetCompletionRoutine(Irp, RaiseEventOnComplete, &Done, TRUE, TRUE, TRUE); + NTSTATUS Status = ((WSK_PROVIDER_DATAGRAM_DISPATCH *)Sock->Dispatch) + ->WskControlSocket(Sock, WskSetOption, Option, Level, Len, Input, 0, NULL, NULL, Irp); + if (Status == STATUS_PENDING) + { + KeWaitForSingleObject(&Done, Executive, KernelMode, FALSE, NULL); + Status = Irp->IoStatus.Status; + } + IoFreeIrp(Irp); + return Status; +} + +_IRQL_requires_max_(APC_LEVEL) +static NTSTATUS +CreateAndBindSocket(_In_ WG_DEVICE *Wg, _Inout_ SOCKADDR *Sa, _Out_ SOCKET **RetSocket) +{ + NTSTATUS Status = STATUS_INSUFFICIENT_RESOURCES; + SOCKET *Socket = MemAllocate(sizeof(*Socket)); + if (!Socket) + return Status; + Socket->Device = Wg; + Socket->Sock = NULL; + ExInitializeRundownProtection(&Socket->ItemsInFlight); + IRP *Irp = IoAllocateIrp(1, FALSE); + if (!Irp) + goto cleanupSocket; + KEVENT Done; + KeInitializeEvent(&Done, SynchronizationEvent, FALSE); + IoSetCompletionRoutine(Irp, RaiseEventOnComplete, &Done, TRUE, TRUE, TRUE); + static CONST WSK_CLIENT_DATAGRAM_DISPATCH WskClientDatagramDispatch = { .WskReceiveFromEvent = Receive }; + Status = WskProviderNpi.Dispatch->WskSocket( + WskProviderNpi.Client, + Sa->sa_family, + SOCK_DGRAM, + IPPROTO_UDP, + WSK_FLAG_DATAGRAM_SOCKET, + Socket, + &WskClientDatagramDispatch, + Wg->SocketOwnerProcess, + NULL, + NULL, + Irp); + if (Status == STATUS_PENDING) + { + KeWaitForSingleObject(&Done, Executive, KernelMode, FALSE, NULL); + Status = Irp->IoStatus.Status; + } + if (!NT_SUCCESS(Status)) + goto cleanupIrp; + WSK_SOCKET *Sock = (WSK_SOCKET *)Irp->IoStatus.Information; + WritePointerNoFence(&Socket->Sock, Sock); + ULONG True = TRUE; + + if (Sa->sa_family == AF_INET) + { + Status = SetSockOpt(Sock, IPPROTO_UDP, UDP_NOCHECKSUM, &True, sizeof(True)); + if (!NT_SUCCESS(Status)) + goto cleanupIrp; + } + else if (Sa->sa_family == AF_INET6) + { + Status = SetSockOpt(Sock, IPPROTO_IPV6, IPV6_V6ONLY, &True, sizeof(True)); + if (!NT_SUCCESS(Status)) + goto cleanupIrp; + } + + Status = SetSockOpt( + Sock, + Sa->sa_family == AF_INET6 ? IPPROTO_IPV6 : IPPROTO_IP, + Sa->sa_family == AF_INET6 ? IPV6_PKTINFO : IP_PKTINFO, + &True, + sizeof(True)); + if (!NT_SUCCESS(Status)) + goto cleanupIrp; + + IoReuseIrp(Irp, STATUS_UNSUCCESSFUL); + IoSetCompletionRoutine(Irp, RaiseEventOnComplete, &Done, TRUE, TRUE, TRUE); + Status = ((WSK_PROVIDER_DATAGRAM_DISPATCH *)Sock->Dispatch)->WskBind(Sock, Sa, 0, Irp); + if (Status == STATUS_PENDING) + { + KeWaitForSingleObject(&Done, Executive, KernelMode, FALSE, NULL); + Status = Irp->IoStatus.Status; + } + if (!NT_SUCCESS(Status)) + { + CHAR Address[SOCKADDR_STR_MAX_LEN]; + SockaddrToString(Address, (SOCKADDR_INET *)Sa); + LogErr(Wg, "Could not bind socket to %s (%#x)", Address, Status); + goto cleanupIrp; + } + + IoReuseIrp(Irp, STATUS_UNSUCCESSFUL); + IoSetCompletionRoutine(Irp, RaiseEventOnComplete, &Done, TRUE, TRUE, TRUE); + Status = ((WSK_PROVIDER_DATAGRAM_DISPATCH *)Sock->Dispatch)->WskGetLocalAddress(Sock, Sa, Irp); + if (Status == STATUS_PENDING) + { + KeWaitForSingleObject(&Done, Executive, KernelMode, FALSE, NULL); + Status = Irp->IoStatus.Status; + } + if (!NT_SUCCESS(Status)) + goto cleanupIrp; + + IoFreeIrp(Irp); + *RetSocket = Socket; + return STATUS_SUCCESS; + +cleanupIrp: + IoFreeIrp(Irp); +cleanupSocket: + CloseSocket(Socket); + return Status; +} + +_Use_decl_annotations_ +NTSTATUS +SocketInit(WG_DEVICE *Wg, UINT16 Port) +{ + NTSTATUS Status; + SOCKADDR_IN Sa4 = { .sin_family = AF_INET, .sin_addr.s_addr = Htonl(INADDR_ANY), .sin_port = Htons(Port) }; + SOCKADDR_IN6 Sa6 = { .sin6_family = AF_INET6, .sin6_addr = IN6ADDR_ANY_INIT }; + SOCKET *New4 = NULL, *New6 = NULL; + LONG Retries = 0; + +retry: + if (WskHasIpv4Transport) + { + Status = CreateAndBindSocket(Wg, (SOCKADDR *)&Sa4, &New4); + if (!NT_SUCCESS(Status)) + goto out; + } + + if (WskHasIpv6Transport) + { + Sa6.sin6_port = Sa4.sin_port; + Status = CreateAndBindSocket(Wg, (SOCKADDR *)&Sa6, &New6); + if (!NT_SUCCESS(Status)) + { + CloseSocket(New4); + New4 = NULL; + if (Status == STATUS_ADDRESS_ALREADY_EXISTS && !Port && Retries++ < 100) + goto retry; + goto out; + } + } + + SocketReinit( + Wg, + New4, + New6, + WskHasIpv4Transport ? Ntohs(Sa4.sin_port) + : WskHasIpv6Transport ? Ntohs(Sa6.sin6_port) + : Port); + Status = STATUS_SUCCESS; +out: + return Status; +} + +_Use_decl_annotations_ +VOID +SocketReinit(WG_DEVICE *Wg, SOCKET *New4, SOCKET *New6, UINT16 Port) +{ + MuAcquirePushLockExclusive(&Wg->SocketUpdateLock); + SOCKET *Old4 = RcuDereferenceProtected(SOCKET, Wg->Sock4, &Wg->SocketUpdateLock); + SOCKET *Old6 = RcuDereferenceProtected(SOCKET, Wg->Sock6, &Wg->SocketUpdateLock); + RcuAssignPointer(Wg->Sock4, New4); + RcuAssignPointer(Wg->Sock6, New6); + if (New4 || New6) + Wg->IncomingPort = Port; + MuReleasePushLockExclusive(&Wg->SocketUpdateLock); + RcuSynchronize(); + CloseSocket(Old4); + CloseSocket(Old6); +} + +static VOID +RouteNotification( + _In_ VOID *CallerContext, + _In_opt_ MIB_IPFORWARD_ROW2 *Row, + _In_ MIB_NOTIFICATION_TYPE NotificationType) +{ + InterlockedAdd((LONG *)CallerContext, 2); +} + +_Use_decl_annotations_ +NTSTATUS +WskInit(VOID) +{ + NTSTATUS Status = ReadNoFence(&WskInitStatus); + if (Status != STATUS_RETRY) + return Status; + MuAcquirePushLockExclusive(&WskIsIniting); + Status = ReadNoFence(&WskInitStatus); + if (Status != STATUS_RETRY) + goto cleanupIniting; + +#if NTDDI_VERSION == NTDDI_WIN7 + RTL_OSVERSIONINFOW OsVersionInfo = { .dwOSVersionInfoSize = sizeof(OsVersionInfo) }; + NoWskSendMessages = + NT_SUCCESS(RtlGetVersion(&OsVersionInfo)) && + (OsVersionInfo.dwMajorVersion < 6 || (OsVersionInfo.dwMajorVersion == 6 && OsVersionInfo.dwMinorVersion < 2)); +#endif + + Status = ExInitializeLookasideListEx( + &SocketSendCtxCache, NULL, NULL, NonPagedPool, 0, sizeof(SOCKET_SEND_CTX), MEMORY_TAG, 0); + if (!NT_SUCCESS(Status)) + goto cleanupIniting; + WSK_CLIENT_NPI WskClientNpi = { .Dispatch = &WskAppDispatchV1 }; + Status = WskRegister(&WskClientNpi, &WskRegistration); + if (!NT_SUCCESS(Status)) + goto cleanupLookaside; + Status = WskCaptureProviderNPI(&WskRegistration, WSK_INFINITE_WAIT, &WskProviderNpi); + if (!NT_SUCCESS(Status)) + goto cleanupWskRegister; + SIZE_T WskTransportsSize = 0x10 * sizeof(WSK_TRANSPORT); + for (;;) + { + WSK_TRANSPORT *WskTransports = MemAllocate(WskTransportsSize); + if (!WskTransports) + { + Status = STATUS_INSUFFICIENT_RESOURCES; + goto cleanupWskProviderNPI; + } + Status = WskProviderNpi.Dispatch->WskControlClient( + WskProviderNpi.Client, + WSK_TRANSPORT_LIST_QUERY, + 0, + NULL, + WskTransportsSize, + WskTransports, + &WskTransportsSize, + NULL); + if (NT_SUCCESS(Status)) + { + for (SIZE_T i = 0, n = WskTransportsSize / sizeof(*WskTransports); i < n; ++i) + { + if (WskTransports[i].SocketType == SOCK_DGRAM && WskTransports[i].Protocol == IPPROTO_UDP) + { + if (WskTransports[i].AddressFamily == AF_UNSPEC) + { + WskHasIpv4Transport = TRUE; + WskHasIpv6Transport = TRUE; + } + else if (WskTransports[i].AddressFamily == AF_INET) + WskHasIpv4Transport = TRUE; + else if (WskTransports[i].AddressFamily == AF_INET6) + WskHasIpv6Transport = TRUE; + } + } + MemFree(WskTransports); + break; + } + MemFree(WskTransports); + if (Status != STATUS_BUFFER_OVERFLOW) + goto cleanupWskProviderNPI; + } + WSK_EVENT_CALLBACK_CONTROL WskEventCallbackControl = { .NpiId = &NPI_WSK_INTERFACE_ID, + .EventMask = WSK_EVENT_RECEIVE_FROM }; + Status = WskProviderNpi.Dispatch->WskControlClient( + WskProviderNpi.Client, + WSK_SET_STATIC_EVENT_CALLBACKS, + sizeof(WskEventCallbackControl), + &WskEventCallbackControl, + 0, + NULL, + NULL, + NULL); + if (!NT_SUCCESS(Status)) + goto cleanupWskProviderNPI; + + Status = NotifyRouteChange2(AF_INET, RouteNotification, &RoutingGenerationV4, FALSE, &RouteNotifierV4); + if (!NT_SUCCESS(Status)) + goto cleanupWskProviderNPI; + Status = NotifyRouteChange2(AF_INET6, RouteNotification, &RoutingGenerationV6, FALSE, &RouteNotifierV6); + if (!NT_SUCCESS(Status)) + goto cleanupRouteNotifierV4; + + WriteNoFence(&WskInitStatus, STATUS_SUCCESS); + MuReleasePushLockExclusive(&WskIsIniting); + return STATUS_SUCCESS; + +cleanupRouteNotifierV4: + CancelMibChangeNotify2(RouteNotifierV4); +cleanupWskProviderNPI: + WskReleaseProviderNPI(&WskRegistration); +cleanupWskRegister: + WskDeregister(&WskRegistration); +cleanupLookaside: + ExDeleteLookasideListEx(&SocketSendCtxCache); +cleanupIniting: + WriteNoFence(&WskInitStatus, Status); + MuReleasePushLockExclusive(&WskIsIniting); + return Status; +} + +_Use_decl_annotations_ +VOID WskUnload(VOID) +{ + MuAcquirePushLockExclusive(&WskIsIniting); + if (ReadNoFence(&WskInitStatus) != STATUS_SUCCESS) + goto out; + CancelMibChangeNotify2(RouteNotifierV6); + CancelMibChangeNotify2(RouteNotifierV4); + WskReleaseProviderNPI(&WskRegistration); + WskDeregister(&WskRegistration); + ExDeleteLookasideListEx(&SocketSendCtxCache); +out: + MuReleasePushLockExclusive(&WskIsIniting); +} diff --git a/driver/socket.h b/driver/socket.h new file mode 100644 index 0000000..223c3ec --- /dev/null +++ b/driver/socket.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +#include <wsk.h> + +typedef struct _SOCKET +{ + WSK_SOCKET *Sock; + WG_DEVICE *Device; + EX_RUNDOWN_REF ItemsInFlight; +} SOCKET; + +_IRQL_requires_max_(PASSIVE_LEVEL) +NTSTATUS +SocketSendNblsToPeer(_Inout_ WG_PEER *Peer, _In_ __drv_aliasesMem NET_BUFFER_LIST *First, _Out_ BOOLEAN *AllKeepalive); + +_IRQL_requires_max_(PASSIVE_LEVEL) +NTSTATUS +SocketSendBufferToPeer(_Inout_ WG_PEER *Peer, _In_reads_bytes_(Len) CONST VOID *Data, _In_ ULONG Len); + +_IRQL_requires_max_(DISPATCH_LEVEL) +NTSTATUS +SocketSendBufferAsReplyToNbl( + _Inout_ WG_DEVICE *Wg, + _In_ CONST NET_BUFFER_LIST *InNbl, + _In_reads_bytes_(Len) CONST VOID *Buffer, + _In_ ULONG Len); + +NTSTATUS +SocketEndpointFromNbl(_Out_ ENDPOINT *Endpoint, _In_ CONST NET_BUFFER_LIST *Nbl); + +_Requires_lock_not_held_(Peer->EndpointLock) +VOID +SocketSetPeerEndpoint(_Inout_ WG_PEER *Peer, _In_ CONST ENDPOINT *Endpoint); + +_Requires_lock_not_held_(Peer->EndpointLock) +VOID +SocketSetPeerEndpointFromNbl(_Inout_ WG_PEER *Peer, _In_ CONST NET_BUFFER_LIST *Nbl); + +_Requires_lock_not_held_(Peer->EndpointLock) +VOID +SocketClearPeerEndpointSrc(_Inout_ WG_PEER *Peer); + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_not_held_(Wg->SocketUpdateLock) +NTSTATUS +SocketInit(_Inout_ WG_DEVICE *Wg, _In_ UINT16 Port); + +_IRQL_requires_max_(APC_LEVEL) +_Requires_lock_not_held_(Wg->SocketUpdateLock) +VOID +SocketReinit( + _Inout_ WG_DEVICE *Wg, + _In_opt_ __drv_aliasesMem SOCKET *New4, + _In_opt_ __drv_aliasesMem SOCKET *New6, + _In_ UINT16 Port); + +_IRQL_requires_max_(PASSIVE_LEVEL) +NTSTATUS +WskInit(VOID); + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID WskUnload(VOID); diff --git a/driver/timers.c b/driver/timers.c new file mode 100644 index 0000000..50f0366 --- /dev/null +++ b/driver/timers.c @@ -0,0 +1,352 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include "interlocked.h" +#include "device.h" +#include "peer.h" +#include "queueing.h" +#include "rcu.h" +#include "socket.h" +#include "timers.h" +#include "logging.h" + +typedef _Function_class_(TIMER_CALLBACK) +_IRQL_requires_(DISPATCH_LEVEL) +_IRQL_requires_same_ +VOID +TIMER_CALLBACK(_In_ TIMER *); +typedef TIMER_CALLBACK *PTIMER_CALLBACK; + +static KDEFERRED_ROUTINE TimerDpcCallback; +_Use_decl_annotations_ +static VOID +TimerDpcCallback(KDPC *Dpc, PVOID DeferredContext, PVOID SystemArgument1, PVOID SystemArgument2) +{ + TIMER *Timer = CONTAINING_RECORD(Dpc, TIMER, Dpc); + _Analysis_assume_(DeferredContext != NULL); + ((PTIMER_CALLBACK)DeferredContext)(Timer); + WriteBooleanNoFence(&Timer->Pending, FALSE); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +static VOID +TimerInit(_Out_ TIMER *Timer, _In_ PTIMER_CALLBACK Callback) +{ + KeInitializeDpc(&Timer->Dpc, TimerDpcCallback, (PVOID)Callback); + KeInitializeTimer(&Timer->Timer); + Timer->Pending = FALSE; +} + +static BOOLEAN +TimerIsPending(_In_ CONST TIMER *Timer) +{ + return ReadBooleanNoFence(&Timer->Pending); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +static VOID +TimerMod(_Inout_ TIMER *Timer, _In_ LONG64 Expires) +{ + WriteBooleanNoFence(&Timer->Pending, TRUE); + KeSetCoalescableTimer(&Timer->Timer, (LARGE_INTEGER){ .QuadPart = Expires }, 0, 320, &Timer->Dpc); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +static VOID +TimerDelete(_In_ TIMER *Timer) +{ + if (!KeCancelTimer(&Timer->Timer)) + KeRemoveQueueDpc(&Timer->Dpc); + WriteBooleanNoFence(&Timer->Pending, FALSE); +} + +static ULONG JitterSeed; + +_IRQL_requires_max_(APC_LEVEL) +_Ret_range_(0, Range - 1) +static ULONG +GenerateJitter(_In_ ULONG Range) +{ + return (ULONG)(((UINT64)RtlRandomEx(&JitterSeed) * Range) >> 32); +} + +/* + * - Timer for retransmitting the handshake if we don't hear back after + * `REKEY_TIMEOUT + jitter` ms. + * + * - Timer for sending empty packet if we have received a packet but after have + * not sent one for `KEEPALIVE_TIMEOUT` ms. + * + * - Timer for initiating new handshake if we have sent a packet but after have + * not received one (even empty) for `(KEEPALIVE_TIMEOUT + REKEY_TIMEOUT) + + * jitter` ms. + * + * - Timer for zeroing out all ephemeral keys after `(REJECT_AFTER_TIME * 3)` ms + * if no new keys have been received. + * + * - Timer for, if enabled, sending an empty authenticated packet every user- + * specified seconds. + */ + +_IRQL_requires_max_(DISPATCH_LEVEL) +static inline VOID +ModPeerTimer(_In_ WG_PEER *Peer, _Inout_ TIMER *Timer, _In_ LONG64 Expires) +{ + if (!ReadBooleanNoFence(&Peer->Device->IsUp) || !ExAcquireRundownProtection(&Peer->InUse)) + return; + TimerMod(Timer, Expires); + ExReleaseRundownProtection(&Peer->InUse); +} + +static TIMER_CALLBACK ExpiredRetransmitHandshake; +_Use_decl_annotations_ +static VOID +ExpiredRetransmitHandshake(TIMER *Timer) +{ + WG_PEER *Peer = CONTAINING_RECORD(Timer, WG_PEER, TimerRetransmitHandshake); + + if (Peer->TimerHandshakeAttempts > MAX_TIMER_HANDSHAKES) + { + CHAR EndpointName[SOCKADDR_STR_MAX_LEN]; + SockaddrToString(EndpointName, &Peer->Endpoint.Addr); + LogInfo( + Peer->Device, + "Handshake for peer %llu (%s) did not complete after %d attempts, giving up", + Peer->InternalId, + EndpointName, + MAX_TIMER_HANDSHAKES + 2); + + TimerDelete(&Peer->TimerSendKeepalive); + /* We drop all packets without a keypair and don't try again, + * if we try unsuccessfully for too long to make a handshake. + */ + PacketPurgeStagedPackets(Peer); + + /* We set a timer for destroying any residue that might be left + * of a partial exchange. + */ + if (!TimerIsPending(&Peer->TimerZeroKeyMaterial)) + ModPeerTimer(Peer, &Peer->TimerZeroKeyMaterial, -SEC_TO_SYS_TIME_UNITS(REJECT_AFTER_TIME * 3)); + } + else + { + ++Peer->TimerHandshakeAttempts; + CHAR EndpointName[SOCKADDR_STR_MAX_LEN]; + SockaddrToString(EndpointName, &Peer->Endpoint.Addr); + LogInfo( + Peer->Device, + "Handshake for peer %llu (%s) did not complete after %d seconds, retrying (try %u)", + Peer->InternalId, + EndpointName, + REKEY_TIMEOUT, + Peer->TimerHandshakeAttempts + 1); + + /* We clear the endpoint address src address, in case this is + * the cause of trouble. + */ + SocketClearPeerEndpointSrc(Peer); + + PacketSendQueuedHandshakeInitiation(Peer, TRUE); + } +} + +static TIMER_CALLBACK ExpiredSendKeepalive; +_Use_decl_annotations_ +static VOID +ExpiredSendKeepalive(TIMER *Timer) +{ + WG_PEER *Peer = CONTAINING_RECORD(Timer, WG_PEER, TimerSendKeepalive); + + PacketSendKeepalive(Peer); + if (Peer->TimerNeedAnotherKeepalive) + { + Peer->TimerNeedAnotherKeepalive = FALSE; + ModPeerTimer(Peer, &Peer->TimerSendKeepalive, -SEC_TO_SYS_TIME_UNITS(KEEPALIVE_TIMEOUT)); + } +} + +static TIMER_CALLBACK ExpiredNewHandshake; +_Use_decl_annotations_ +static VOID +ExpiredNewHandshake(TIMER *Timer) +{ + WG_PEER *Peer = CONTAINING_RECORD(Timer, WG_PEER, TimerNewHandshake); + + CHAR EndpointStr[SOCKADDR_STR_MAX_LEN]; + SockaddrToString(EndpointStr, &Peer->Endpoint.Addr); + LogInfo( + Peer->Device, + "Retrying handshake with peer %llu (%s) because we stopped hearing back after %d seconds", + Peer->InternalId, + EndpointStr, + KEEPALIVE_TIMEOUT + REKEY_TIMEOUT); + /* We clear the endpoint address src address, in case this is the cause + * of trouble. + */ + SocketClearPeerEndpointSrc(Peer); + PacketSendQueuedHandshakeInitiation(Peer, FALSE); +} + +static TIMER_CALLBACK ExpiredZeroKeyMaterial; +_Use_decl_annotations_ +static VOID +ExpiredZeroKeyMaterial(TIMER *Timer) +{ + WG_PEER *Peer = CONTAINING_RECORD(Timer, WG_PEER, TimerZeroKeyMaterial); + + if (!ExAcquireRundownProtection(&Peer->InUse)) + return; + + CHAR EndpointName[SOCKADDR_STR_MAX_LEN]; + SockaddrToString(EndpointName, &Peer->Endpoint.Addr); + LogInfo( + Peer->Device, + "Zeroing out all keys for peer %llu (%s), since we haven't received a new one in %d seconds", + Peer->InternalId, + EndpointName, + REJECT_AFTER_TIME * 3); + + PeerGet(Peer); + WriteNoFence16(&Peer->HandshakeTxAction, HANDSHAKE_TX_CLEAR); + + if (PeerSerialEnqueueIfNotBusy(&Peer->Device->HandshakeTxQueue, &Peer->HandshakeTxSerialEntry, TRUE)) + MulticoreWorkQueueBump(&Peer->Device->HandshakeTxThreads); + else + { + /* If the work was already on the queue, we want to drop the extra reference. */ + ExReleaseRundownProtection(&Peer->InUse); + PeerPut(Peer); + } +} + +static TIMER_CALLBACK ExpiredSendPersistentKeepalive; +_Use_decl_annotations_ +static VOID +ExpiredSendPersistentKeepalive(TIMER *Timer) +{ + WG_PEER *Peer = CONTAINING_RECORD(Timer, WG_PEER, TimerPersistentKeepalive); + + if (Peer->PersistentKeepaliveInterval) + PacketSendKeepalive(Peer); +} + +/* Should be called after an authenticated data packet is sent. */ +_Use_decl_annotations_ +VOID +TimersDataSent(WG_PEER *Peer) +{ + if (!TimerIsPending(&Peer->TimerNewHandshake)) + ModPeerTimer( + Peer, + &Peer->TimerNewHandshake, + -(SEC_TO_SYS_TIME_UNITS(KEEPALIVE_TIMEOUT + REKEY_TIMEOUT) + + GenerateJitter(REKEY_TIMEOUT_JITTER_MAX_SYS_TIME_UNITS))); +} + +/* Should be called after an authenticated data packet is received. */ +_Use_decl_annotations_ +VOID +TimersDataReceived(WG_PEER *Peer) +{ + if (ReadBooleanNoFence(&Peer->Device->IsUp)) + { + if (!TimerIsPending(&Peer->TimerSendKeepalive)) + ModPeerTimer(Peer, &Peer->TimerSendKeepalive, -SEC_TO_SYS_TIME_UNITS(KEEPALIVE_TIMEOUT)); + else + Peer->TimerNeedAnotherKeepalive = TRUE; + } +} + +/* Should be called after any type of authenticated packet is sent, whether + * keepalive, data, or handshake. + */ +_Use_decl_annotations_ +VOID +TimersAnyAuthenticatedPacketSent(WG_PEER *Peer) +{ + TimerDelete(&Peer->TimerSendKeepalive); +} + +/* Should be called after any type of authenticated packet is received, whether + * keepalive, data, or handshake. + */ +_Use_decl_annotations_ +VOID +TimersAnyAuthenticatedPacketReceived(WG_PEER *Peer) +{ + TimerDelete(&Peer->TimerNewHandshake); +} + +/* Should be called after a handshake initiation message is sent. */ +_Use_decl_annotations_ +VOID +TimersHandshakeInitiated(WG_PEER *Peer) +{ + ModPeerTimer( + Peer, + &Peer->TimerRetransmitHandshake, + -(SEC_TO_SYS_TIME_UNITS(REKEY_TIMEOUT) + GenerateJitter(REKEY_TIMEOUT_JITTER_MAX_SYS_TIME_UNITS))); +} + +/* Should be called after a handshake response message is received and processed + * or when getting key confirmation via the first data message. + */ +_Use_decl_annotations_ +VOID +TimersHandshakeComplete(WG_PEER *Peer) +{ + TimerDelete(&Peer->TimerRetransmitHandshake); + Peer->TimerHandshakeAttempts = 0; + Peer->SentLastminuteHandshake = FALSE; + KeQuerySystemTime(&Peer->WalltimeLastHandshake); +} + +/* Should be called after an ephemeral key is created, which is before sending a + * handshake response or after receiving a handshake response. + */ +_Use_decl_annotations_ +VOID +TimersSessionDerived(WG_PEER *Peer) +{ + ModPeerTimer(Peer, &Peer->TimerZeroKeyMaterial, -SEC_TO_SYS_TIME_UNITS(REJECT_AFTER_TIME * 3)); +} + +/* Should be called before a packet with authentication, whether + * keepalive, data, or handshakem is sent, or after one is received. + */ +_Use_decl_annotations_ +VOID +TimersAnyAuthenticatedPacketTraversal(WG_PEER *Peer) +{ + if (Peer->PersistentKeepaliveInterval) + ModPeerTimer(Peer, &Peer->TimerPersistentKeepalive, -SEC_TO_SYS_TIME_UNITS(Peer->PersistentKeepaliveInterval)); +} + +_Use_decl_annotations_ +VOID +TimersInit(WG_PEER *Peer) +{ + TimerInit(&Peer->TimerRetransmitHandshake, ExpiredRetransmitHandshake); + TimerInit(&Peer->TimerSendKeepalive, ExpiredSendKeepalive); + TimerInit(&Peer->TimerNewHandshake, ExpiredNewHandshake); + TimerInit(&Peer->TimerZeroKeyMaterial, ExpiredZeroKeyMaterial); + TimerInit(&Peer->TimerPersistentKeepalive, ExpiredSendPersistentKeepalive); + Peer->TimerHandshakeAttempts = 0; + Peer->SentLastminuteHandshake = FALSE; + Peer->TimerNeedAnotherKeepalive = FALSE; + CryptoRandom((UCHAR *)&JitterSeed, sizeof(JitterSeed)); +} + +_Use_decl_annotations_ +VOID +TimersStop(WG_PEER *Peer) +{ + TimerDelete(&Peer->TimerRetransmitHandshake); + TimerDelete(&Peer->TimerSendKeepalive); + TimerDelete(&Peer->TimerNewHandshake); + TimerDelete(&Peer->TimerZeroKeyMaterial); + TimerDelete(&Peer->TimerPersistentKeepalive); + KeFlushQueuedDpcs(); +} diff --git a/driver/timers.h b/driver/timers.h new file mode 100644 index 0000000..9600ea6 --- /dev/null +++ b/driver/timers.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +#define SYS_TIME_UNITS_PER_SEC 10000000 /* System time unit is 100 ns. */ +#define SEC_TO_SYS_TIME_UNITS(Sec) ((LONG64)(Sec)*SYS_TIME_UNITS_PER_SEC) + +typedef struct _TIMER +{ + KTIMER Timer; + KDPC Dpc; + BOOLEAN Pending; +} TIMER; + +typedef struct _WG_PEER WG_PEER; + +_IRQL_requires_max_(APC_LEVEL) +VOID +TimersDataSent(_Inout_ WG_PEER *Peer); + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +TimersDataReceived(_Inout_ WG_PEER *Peer); + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +TimersAnyAuthenticatedPacketSent(_Inout_ WG_PEER *Peer); + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +TimersAnyAuthenticatedPacketReceived(_Inout_ WG_PEER *Peer); + +_IRQL_requires_max_(APC_LEVEL) +VOID +TimersHandshakeInitiated(_Inout_ WG_PEER *Peer); + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +TimersHandshakeComplete(_Inout_ WG_PEER *Peer); + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +TimersSessionDerived(_Inout_ WG_PEER *Peer); + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +TimersAnyAuthenticatedPacketTraversal(_Inout_ WG_PEER *Peer); + +_IRQL_requires_max_(DISPATCH_LEVEL) +VOID +TimersInit(_Inout_ WG_PEER *Peer); + +_IRQL_requires_max_(APC_LEVEL) +VOID +TimersStop(_Inout_ WG_PEER *Peer); + +static inline BOOLEAN +BirthdateHasExpired(_In_ CONST UINT64 BirthdaySysTimeUnits, _In_ CONST UINT64 ExpirationSeconds) +{ + return (INT64)(BirthdaySysTimeUnits + SEC_TO_SYS_TIME_UNITS(ExpirationSeconds)) <= (INT64)KeQueryInterruptTime(); +} diff --git a/driver/undocumented.h b/driver/undocumented.h new file mode 100644 index 0000000..f7a55ca --- /dev/null +++ b/driver/undocumented.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#pragma once + +#include <ntifs.h> /* Must be included before <wdm.h> */ +#include <wdm.h> + +typedef enum +{ + SystemExtendedHandleInformation = 0x40 +} SYSTEM_INFORMATION_CLASS; + +typedef struct _SYSTEM_HANDLE_TABLE_ENTRY_INFO_EX +{ + PVOID Object; + HANDLE UniqueProcessId; + HANDLE HandleValue; + ACCESS_MASK GrantedAccess; + USHORT CreatorBackTraceIndex; + USHORT ObjectTypeIndex; + ULONG HandleAttributes; + ULONG Reserved; +} SYSTEM_HANDLE_TABLE_ENTRY_INFO_EX, *PSYSTEM_HANDLE_TABLE_ENTRY_INFO_EX; + +typedef struct _SYSTEM_HANDLE_INFORMATION_EX +{ + ULONG_PTR NumberOfHandles; + ULONG_PTR Reserved; + SYSTEM_HANDLE_TABLE_ENTRY_INFO_EX Handles[ANYSIZE_ARRAY]; +} SYSTEM_HANDLE_INFORMATION_EX, *PSYSTEM_HANDLE_INFORMATION_EX; + +NTSYSAPI +NTSTATUS +NTAPI +ZwQuerySystemInformation( + SYSTEM_INFORMATION_CLASS SystemInformationClass, + PVOID SystemInformation, + ULONG SystemInformationLength, + ULONG *ReturnLength); + +NTSYSAPI +NTSTATUS +NTAPI +ZwYieldExecution(VOID); + +NTSYSAPI +BOOLEAN +NTAPI +SystemPrng(_Out_writes_bytes_all_(Len) PVOID RandomData, _In_ SIZE_T Len); diff --git a/driver/wireguard.inf b/driver/wireguard.inf new file mode 100644 index 0000000..09d870a --- /dev/null +++ b/driver/wireguard.inf @@ -0,0 +1,67 @@ +; SPDX-License-Identifier: GPL-2.0 +; +; Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + +[Version] +Signature = "$Windows NT$" +Class = Net +ClassGUID = {4D36E972-E325-11CE-BFC1-08002BE10318} +Provider = %WireGuard.CompanyName% +CatalogFile.NT = wireguard.cat +PnpLockDown = 1 + +[Manufacturer] +%WireGuard.CompanyName% = %WireGuard.Name%, NT$ARCH$ + +[SourceDisksNames] +1 = %WireGuard.DiskDesc%, "", , + +[SourceDisksFiles] +wireguard.sys = 1 + +[DestinationDirs] +DefaultDestDir = 12 +WireGuard.CopyFiles.Sys = 12 + +[WireGuard.CopyFiles.Sys] +wireguard.sys, , , 0x00004002 ; COPYFLG_IN_USE_RENAME | COPYFLG_NOSKIP + +[WireGuard.NT$ARCH$] +%WireGuard.DeviceDesc% = WireGuard.Install, WireGuard + +[WireGuard.Install] +Characteristics = 0x1 ; NCF_VIRTUAL +AddReg = WireGuard.Ndi +CopyFiles = WireGuard.CopyFiles.Sys +*IfType = 53 ; IF_TYPE_PROP_VIRTUAL +*MediaType = 19 ; NdisMediumIP +*PhysicalMediaType = 0 ; NdisPhysicalMediumUnspecified +EnableDhcp = 0 ; Disable DHCP + +[WireGuard.Install.Services] +AddService = WireGuard, 2, WireGuard.Service, WireGuard.EventLog ; 2=SPSVCINST_ASSOCSERVICE + +[WireGuard.Ndi] +HKR, , DeviceVxDs, , wireguard.sys +HKR, Ndi, Service, 0, WireGuard +HKR, Ndi\Interfaces, UpperRange, , "ndis5" +HKR, Ndi\Interfaces, LowerRange, , "nolower" + +[WireGuard.Service] +DisplayName = %WireGuard.Name% +Description = %WireGuard.DeviceDesc% +ServiceType = 1 ; SERVICE_KERNEL_DRIVER +StartType = 3 ; SERVICE_DEMAND_START +ErrorControl = 1 ; SERVICE_ERROR_NORMAL +ServiceBinary = %12%\wireguard.sys +LoadOrderGroup = NDIS + +[WireGuard.EventLog] +HKR, , EventMessageFile, 0x00020000, "%11%\IoLogMsg.dll;%12%\wireguard.sys" +HKR, , TypesSupported, 0x00010001, 7 + +[Strings] +WireGuard.Name = "WireGuard" +WireGuard.DiskDesc = "WireGuard Driver Install Disk" +WireGuard.DeviceDesc = "WireGuard Tunnel" +WireGuard.CompanyName = "WireGuard LLC" diff --git a/driver/wireguard.rc b/driver/wireguard.rc new file mode 100644 index 0000000..25d8e6d --- /dev/null +++ b/driver/wireguard.rc @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. + */ + +#include <windows.h> +#include <ntverp.h> + +#pragma code_page(1252) + +#define STRINGIZE(x) #x +#define EXPAND(x) STRINGIZE(x) + +VS_VERSION_INFO VERSIONINFO +FILEVERSION WIREGUARD_VERSION_MAJ, WIREGUARD_VERSION_MIN, WIREGUARD_VERSION_REL, 0 +PRODUCTVERSION WIREGUARD_VERSION_MAJ, WIREGUARD_VERSION_MIN, WIREGUARD_VERSION_REL, 0 +FILEOS VOS_NT_WINDOWS32 +FILETYPE VFT_DRV +FILESUBTYPE VFT2_DRV_SYSTEM +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904b0" + BEGIN + VALUE "CompanyName", "WireGuard LLC" + VALUE "FileDescription", "WireGuard Driver" + VALUE "FileVersion", EXPAND(WIREGUARD_VERSION) + VALUE "InternalName", "wireguard.sys" + VALUE "LegalCopyright", "Copyright \xa9 2015-2021 WireGuard LLC. All Rights Reserved." + VALUE "OriginalFilename", "wireguard.sys" + VALUE "ProductName", "WireGuard Driver" + VALUE "ProductVersion", EXPAND(WIREGUARD_VERSION) + VALUE "Comments", "https://www.wireguard.com/" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1200 + END +END diff --git a/example/example.c b/example/example.c new file mode 100644 index 0000000..828a8d0 --- /dev/null +++ b/example/example.c @@ -0,0 +1,474 @@ +/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+#include <winsock2.h>
+#include <Windows.h>
+#include <ws2ipdef.h>
+#include <ws2tcpip.h>
+#include <iphlpapi.h>
+#include <bcrypt.h>
+#include <wincrypt.h>
+#include <sysinfoapi.h>
+#include <winternl.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include "wireguard.h"
+
+static WIREGUARD_CREATE_ADAPTER_FUNC *WireGuardCreateAdapter;
+static WIREGUARD_DELETE_ADAPTER_FUNC *WireGuardDeleteAdapter;
+static WIREGUARD_DELETE_POOL_DRIVER_FUNC *WireGuardDeletePoolDriver;
+static WIREGUARD_ENUM_ADAPTERS_FUNC *WireGuardEnumAdapters;
+static WIREGUARD_FREE_ADAPTER_FUNC *WireGuardFreeAdapter;
+static WIREGUARD_OPEN_ADAPTER_FUNC *WireGuardOpenAdapter;
+static WIREGUARD_GET_ADAPTER_LUID_FUNC *WireGuardGetAdapterLUID;
+static WIREGUARD_GET_ADAPTER_NAME_FUNC *WireGuardGetAdapterName;
+static WIREGUARD_SET_ADAPTER_NAME_FUNC *WireGuardSetAdapterName;
+static WIREGUARD_GET_RUNNING_DRIVER_VERSION_FUNC *WireGuardGetRunningDriverVersion;
+static WIREGUARD_SET_LOGGER_FUNC *WireGuardSetLogger;
+static WIREGUARD_SET_ADAPTER_LOGGING_FUNC *WireGuardSetAdapterLogging;
+static WIREGUARD_GET_ADAPTER_STATE_FUNC *WireGuardGetAdapterState;
+static WIREGUARD_SET_ADAPTER_STATE_FUNC *WireGuardSetAdapterState;
+static WIREGUARD_GET_CONFIGURATION_FUNC *WireGuardGetConfiguration;
+static WIREGUARD_SET_CONFIGURATION_FUNC *WireGuardSetConfiguration;
+
+static HMODULE
+InitializeWireGuardNT(void)
+{
+ HMODULE WireGuardDll =
+ LoadLibraryExW(L"wireguard.dll", NULL, LOAD_LIBRARY_SEARCH_APPLICATION_DIR | LOAD_LIBRARY_SEARCH_SYSTEM32);
+ if (!WireGuardDll)
+ return NULL;
+#define X(Name, Type) ((Name = (Type *)GetProcAddress(WireGuardDll, #Name)) == NULL)
+ if (X(WireGuardCreateAdapter, WIREGUARD_CREATE_ADAPTER_FUNC) ||
+ X(WireGuardDeleteAdapter, WIREGUARD_DELETE_ADAPTER_FUNC) ||
+ X(WireGuardDeletePoolDriver, WIREGUARD_DELETE_POOL_DRIVER_FUNC) ||
+ X(WireGuardEnumAdapters, WIREGUARD_ENUM_ADAPTERS_FUNC) ||
+ X(WireGuardFreeAdapter, WIREGUARD_FREE_ADAPTER_FUNC) || X(WireGuardOpenAdapter, WIREGUARD_OPEN_ADAPTER_FUNC) ||
+ X(WireGuardGetAdapterLUID, WIREGUARD_GET_ADAPTER_LUID_FUNC) ||
+ X(WireGuardGetAdapterName, WIREGUARD_GET_ADAPTER_NAME_FUNC) ||
+ X(WireGuardSetAdapterName, WIREGUARD_SET_ADAPTER_NAME_FUNC) ||
+ X(WireGuardGetRunningDriverVersion, WIREGUARD_GET_RUNNING_DRIVER_VERSION_FUNC) ||
+ X(WireGuardSetLogger, WIREGUARD_SET_LOGGER_FUNC) ||
+ X(WireGuardSetAdapterLogging, WIREGUARD_SET_ADAPTER_LOGGING_FUNC) ||
+ X(WireGuardGetAdapterState, WIREGUARD_GET_ADAPTER_STATE_FUNC) ||
+ X(WireGuardSetAdapterState, WIREGUARD_SET_ADAPTER_STATE_FUNC) ||
+ X(WireGuardGetConfiguration, WIREGUARD_GET_CONFIGURATION_FUNC) ||
+ X(WireGuardSetConfiguration, WIREGUARD_SET_CONFIGURATION_FUNC))
+#undef X
+ {
+ DWORD LastError = GetLastError();
+ FreeLibrary(WireGuardDll);
+ SetLastError(LastError);
+ return NULL;
+ }
+ return WireGuardDll;
+}
+
+static void CALLBACK
+ConsoleLogger(_In_ WIREGUARD_LOGGER_LEVEL Level, _In_z_ const WCHAR *LogLine)
+{
+ FILETIME Timestamp;
+ GetSystemTimePreciseAsFileTime(&Timestamp);
+ SYSTEMTIME SystemTime;
+ FileTimeToSystemTime(&Timestamp, &SystemTime);
+ WCHAR LevelMarker;
+ switch (Level)
+ {
+ case WIREGUARD_LOG_INFO:
+ LevelMarker = L'+';
+ break;
+ case WIREGUARD_LOG_WARN:
+ LevelMarker = L'-';
+ break;
+ case WIREGUARD_LOG_ERR:
+ LevelMarker = L'!';
+ break;
+ default:
+ return;
+ }
+ fwprintf(
+ stderr,
+ L"%04u-%02u-%02u %02u:%02u:%02u.%04u [%c] %s\n",
+ SystemTime.wYear,
+ SystemTime.wMonth,
+ SystemTime.wDay,
+ SystemTime.wHour,
+ SystemTime.wMinute,
+ SystemTime.wSecond,
+ SystemTime.wMilliseconds,
+ LevelMarker,
+ LogLine);
+}
+
+static DWORD
+LogError(_In_z_ const WCHAR *Prefix, _In_ DWORD Error)
+{
+ WCHAR *SystemMessage = NULL, *FormattedMessage = NULL;
+ FormatMessageW(
+ FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_MAX_WIDTH_MASK,
+ NULL,
+ HRESULT_FROM_SETUPAPI(Error),
+ MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+ (void *)&SystemMessage,
+ 0,
+ NULL);
+ FormatMessageW(
+ FORMAT_MESSAGE_FROM_STRING | FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_ARGUMENT_ARRAY |
+ FORMAT_MESSAGE_MAX_WIDTH_MASK,
+ SystemMessage ? L"%1: %3(Code 0x%2!08X!)" : L"%1: Code 0x%2!08X!",
+ 0,
+ 0,
+ (void *)&FormattedMessage,
+ 0,
+ (va_list *)(DWORD_PTR[]){ (DWORD_PTR)Prefix, (DWORD_PTR)Error, (DWORD_PTR)SystemMessage });
+ if (FormattedMessage)
+ ConsoleLogger(WIREGUARD_LOG_ERR, FormattedMessage);
+ LocalFree(FormattedMessage);
+ LocalFree(SystemMessage);
+ return Error;
+}
+
+static DWORD
+LogLastError(_In_z_ const WCHAR *Prefix)
+{
+ DWORD LastError = GetLastError();
+ LogError(Prefix, LastError);
+ SetLastError(LastError);
+ return LastError;
+}
+
+static void
+Log(_In_ WIREGUARD_LOGGER_LEVEL Level, _In_z_ const WCHAR *Format, ...)
+{
+ WCHAR LogLine[0x200];
+ va_list args;
+ va_start(args, Format);
+ _vsnwprintf_s(LogLine, _countof(LogLine), _TRUNCATE, Format, args);
+ va_end(args);
+ ConsoleLogger(Level, LogLine);
+}
+
+_Must_inspect_result_
+_Return_type_success_(return != FALSE)
+static BOOL
+GenerateKeyPair(
+ _Out_writes_bytes_all_(WIREGUARD_KEY_LENGTH) BYTE PublicKey[WIREGUARD_KEY_LENGTH],
+ _Out_writes_bytes_all_(WIREGUARD_KEY_LENGTH) BYTE PrivateKey[WIREGUARD_KEY_LENGTH])
+{
+ BCRYPT_ALG_HANDLE Algorithm;
+ BCRYPT_KEY_HANDLE Key;
+ NTSTATUS Status;
+ struct
+ {
+ BCRYPT_ECCKEY_BLOB Header;
+ BYTE Public[32];
+ BYTE Unused[32];
+ BYTE Private[32];
+ } ExportedKey;
+ ULONG Bytes;
+
+ Status = BCryptOpenAlgorithmProvider(&Algorithm, BCRYPT_ECDH_ALGORITHM, NULL, 0);
+ if (!NT_SUCCESS(Status))
+ goto out;
+
+ Status = BCryptSetProperty(
+ Algorithm, BCRYPT_ECC_CURVE_NAME, (PUCHAR)BCRYPT_ECC_CURVE_25519, sizeof(BCRYPT_ECC_CURVE_25519), 0);
+ if (!NT_SUCCESS(Status))
+ goto cleanupProvider;
+
+ Status = BCryptGenerateKeyPair(Algorithm, &Key, 255, 0);
+ if (!NT_SUCCESS(Status))
+ goto cleanupProvider;
+
+ Status = BCryptFinalizeKeyPair(Key, 0);
+ if (!NT_SUCCESS(Status))
+ goto cleanupKey;
+
+ Status = BCryptExportKey(Key, NULL, BCRYPT_ECCPRIVATE_BLOB, (PUCHAR)&ExportedKey, sizeof(ExportedKey), &Bytes, 0);
+ if (!NT_SUCCESS(Status))
+ goto cleanupKey;
+
+ memcpy(PublicKey, ExportedKey.Public, WIREGUARD_KEY_LENGTH);
+ memcpy(PrivateKey, ExportedKey.Private, WIREGUARD_KEY_LENGTH);
+ SecureZeroMemory(&ExportedKey, sizeof(ExportedKey));
+
+cleanupKey:
+ BCryptDestroyKey(Key);
+cleanupProvider:
+ BCryptCloseAlgorithmProvider(Algorithm, 0);
+out:
+ SetLastError(RtlNtStatusToDosError(Status));
+ return NT_SUCCESS(Status);
+}
+
+static HANDLE QuitEvent;
+
+static BOOL WINAPI
+CtrlHandler(_In_ DWORD CtrlType)
+{
+ switch (CtrlType)
+ {
+ case CTRL_C_EVENT:
+ case CTRL_BREAK_EVENT:
+ case CTRL_CLOSE_EVENT:
+ case CTRL_LOGOFF_EVENT:
+ case CTRL_SHUTDOWN_EVENT:
+ Log(WIREGUARD_LOG_INFO, L"Cleaning up and shutting down");
+ SetEvent(QuitEvent);
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static BOOL CALLBACK
+PrintAdapter(_In_ WIREGUARD_ADAPTER_HANDLE Adapter, _In_ LPARAM Param)
+{
+ WCHAR szAdapterName[MAX_ADAPTER_NAME];
+ if (WireGuardGetAdapterName(Adapter, szAdapterName))
+ Log(WIREGUARD_LOG_INFO, L"Existing WireGuard adapter: %s", szAdapterName);
+ return TRUE;
+}
+
+_Return_type_success_(return != FALSE)
+static BOOL
+TalkToDemoServer(
+ _In_reads_bytes_(InputLength) const CHAR *Input,
+ _In_ DWORD InputLength,
+ _Out_writes_bytes_(*OutputLength) CHAR *Output,
+ _Inout_ DWORD *OutputLength,
+ _Out_ SOCKADDR_INET *ResolvedDemoServer)
+{
+ SOCKET Socket = INVALID_SOCKET;
+ ADDRINFOW Hints = { .ai_family = AF_UNSPEC, .ai_socktype = SOCK_STREAM, .ai_protocol = IPPROTO_TCP }, *Resolution;
+ BOOL Ret = FALSE;
+
+ if (GetAddrInfoW(L"demo.wireguard.com", L"42912", &Hints, &Resolution))
+ return FALSE;
+ for (ADDRINFOW *Candidate = Resolution; Candidate; Candidate = Candidate->ai_next)
+ {
+ if (Candidate->ai_family != AF_INET && Candidate->ai_family != AF_INET6)
+ continue;
+ Socket = socket(Candidate->ai_family, Candidate->ai_socktype, Candidate->ai_protocol);
+ if (Socket == INVALID_SOCKET)
+ goto cleanupResolution;
+ if (connect(Socket, Candidate->ai_addr, (int)Candidate->ai_addrlen) == SOCKET_ERROR)
+ {
+ closesocket(Socket);
+ Socket = INVALID_SOCKET;
+ }
+ memcpy(ResolvedDemoServer, Candidate->ai_addr, Candidate->ai_addrlen);
+ break;
+ }
+ if (Socket == INVALID_SOCKET)
+ goto cleanupResolution;
+ if (send(Socket, Input, InputLength, 0) == SOCKET_ERROR)
+ goto cleanupSocket;
+ if ((*OutputLength = recv(Socket, Output, *OutputLength, 0)) == SOCKET_ERROR)
+ goto cleanupSocket;
+ Ret = TRUE;
+cleanupSocket:
+ closesocket(Socket);
+cleanupResolution:
+ FreeAddrInfoW(Resolution);
+ return Ret;
+}
+
+int __cdecl main(void)
+{
+ DWORD LastError;
+ WSADATA WsaData;
+ if (WSAStartup(MAKEWORD(2, 2), &WsaData))
+ return LogError(L"Failed to initialize Winsock", GetLastError());
+ HMODULE WireGuard = InitializeWireGuardNT();
+ if (!WireGuard)
+ {
+ LastError = LogError(L"Failed to initialize WireGuardNT", GetLastError());
+ goto cleanupWinsock;
+ }
+ WireGuardSetLogger(ConsoleLogger);
+ Log(WIREGUARD_LOG_INFO, L"WireGuardNT library loaded");
+ WireGuardEnumAdapters(L"Example", PrintAdapter, 0);
+
+ struct
+ {
+ WIREGUARD_INTERFACE Interface;
+ WIREGUARD_PEER DemoServer;
+ WIREGUARD_ALLOWED_IP AllV4;
+ } Config = { .Interface = { .Flags = WIREGUARD_INTERFACE_HAS_PRIVATE_KEY, .PeersCount = 1 },
+ .DemoServer = { .Flags = WIREGUARD_PEER_HAS_PUBLIC_KEY | WIREGUARD_PEER_HAS_ENDPOINT,
+ .AllowedIPsCount = 1 },
+ .AllV4 = { .AddressFamily = AF_INET } };
+
+ Log(WIREGUARD_LOG_INFO, L"Generating keypair");
+ BYTE PublicKey[WIREGUARD_KEY_LENGTH];
+ if (!GenerateKeyPair(PublicKey, Config.Interface.PrivateKey))
+ {
+ LastError = LogError(L"Failed to generate keypair", GetLastError());
+ goto cleanupWireGuard;
+ }
+ CHAR PublicKeyString[46] = { 0 };
+ DWORD Bytes = sizeof(PublicKeyString);
+ CryptBinaryToStringA(
+ PublicKey, sizeof(PublicKey), CRYPT_STRING_BASE64 | CRYPT_STRING_NOCR, PublicKeyString, &Bytes);
+ CHAR ServerResponse[256] = { 0 };
+ Log(WIREGUARD_LOG_INFO, L"Talking to demo server");
+ Bytes = sizeof(ServerResponse) - 1;
+ if (!TalkToDemoServer(
+ PublicKeyString, (DWORD)strlen(PublicKeyString), ServerResponse, &Bytes, &Config.DemoServer.Endpoint))
+ {
+ LastError = LogError(L"Failed to talk to demo server", GetLastError());
+ goto cleanupWireGuard;
+ }
+
+ CHAR *Colon1 = strchr(ServerResponse, ':');
+ CHAR *Colon2 = Colon1 ? strchr(Colon1 + 1, ':') : NULL;
+ CHAR *Colon3 = Colon2 ? strchr(Colon2 + 1, ':') : NULL;
+ if (!Colon1 || !Colon2 || !Colon3)
+ {
+ LastError = LogError(L"Failed to parse demo server response", ERROR_UNDEFINED_CHARACTER);
+ goto cleanupWireGuard;
+ }
+ if (Bytes && ServerResponse[--Bytes] == '\n')
+ ServerResponse[Bytes] = '\0';
+ *Colon1 = *Colon2 = *Colon3 = '\0';
+
+ MIB_UNICASTIPADDRESS_ROW AddressRow;
+ InitializeUnicastIpAddressEntry(&AddressRow);
+ AddressRow.Address.Ipv4.sin_family = AF_INET;
+ AddressRow.OnLinkPrefixLength = 24; /* This is a /24 network */
+ Bytes = sizeof(Config.DemoServer.PublicKey);
+ if (strcmp(ServerResponse, "OK") || InetPtonA(AF_INET, Colon3 + 1, &AddressRow.Address.Ipv4.sin_addr) != 1 ||
+ !CryptStringToBinaryA(Colon1 + 1, 0, CRYPT_STRING_BASE64, Config.DemoServer.PublicKey, &Bytes, NULL, NULL))
+ {
+ LastError = LogError(L"Failed to parse demo server response", ERROR_UNDEFINED_CHARACTER);
+ goto cleanupWireGuard;
+ }
+ if (Config.DemoServer.Endpoint.si_family == AF_INET)
+ Config.DemoServer.Endpoint.Ipv4.sin_port = htons((u_short)atoi(Colon2 + 1));
+ else if (Config.DemoServer.Endpoint.si_family == AF_INET6)
+ Config.DemoServer.Endpoint.Ipv6.sin6_port = htons((u_short)atoi(Colon2 + 1));
+
+ QuitEvent = CreateEventW(NULL, TRUE, FALSE, NULL);
+ if (!QuitEvent)
+ {
+ LastError = LogError(L"Failed to create event", GetLastError());
+ goto cleanupWireGuard;
+ }
+ if (!SetConsoleCtrlHandler(CtrlHandler, TRUE))
+ {
+ LastError = LogError(L"Failed to set console handler", GetLastError());
+ goto cleanupQuit;
+ }
+
+ GUID ExampleGuid = { 0xdeadc001, 0xbeef, 0xbabe, { 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef } };
+ WIREGUARD_ADAPTER_HANDLE Adapter = WireGuardOpenAdapter(L"Example", L"Demo");
+ if (Adapter && !WireGuardDeleteAdapter(Adapter, NULL))
+ {
+ LastError = GetLastError();
+ LogError(L"Failed to delete already existing adapter", LastError);
+ goto cleanupQuit;
+ }
+ WireGuardFreeAdapter(Adapter);
+ Adapter = WireGuardCreateAdapter(L"Example", L"Demo", &ExampleGuid, NULL);
+ if (!Adapter)
+ {
+ LastError = GetLastError();
+ LogError(L"Failed to create adapter", LastError);
+ goto cleanupQuit;
+ }
+
+ if (!WireGuardSetAdapterLogging(Adapter, WIREGUARD_ADAPTER_LOG_ON))
+ LogError(L"Failed to enable adapter logging", GetLastError());
+
+ DWORD Version = WireGuardGetRunningDriverVersion();
+ Log(WIREGUARD_LOG_INFO, L"WireGuardNT v%u.%u loaded", (Version >> 16) & 0xff, (Version >> 0) & 0xff);
+
+ WireGuardGetAdapterLUID(Adapter, &AddressRow.InterfaceLuid);
+ LastError = CreateUnicastIpAddressEntry(&AddressRow);
+ if (LastError != ERROR_SUCCESS && LastError != ERROR_OBJECT_ALREADY_EXISTS)
+ {
+ LogError(L"Failed to set IP address", LastError);
+ goto cleanupAdapter;
+ }
+ MIB_IPFORWARD_ROW2 DefaultRoute = { 0 };
+ InitializeIpForwardEntry(&DefaultRoute);
+ DefaultRoute.InterfaceLuid = AddressRow.InterfaceLuid;
+ DefaultRoute.DestinationPrefix.Prefix.si_family = AF_INET;
+ DefaultRoute.NextHop.si_family = AF_INET;
+ DefaultRoute.Metric = 0;
+ LastError = CreateIpForwardEntry2(&DefaultRoute);
+ if (LastError != ERROR_SUCCESS && LastError != ERROR_OBJECT_ALREADY_EXISTS)
+ {
+ LogError(L"Failed to set default route", LastError);
+ goto cleanupAdapter;
+ }
+ MIB_IPINTERFACE_ROW IpInterface = { 0 };
+ InitializeIpInterfaceEntry(&IpInterface);
+ IpInterface.InterfaceLuid = AddressRow.InterfaceLuid;
+ IpInterface.Family = AF_INET;
+ LastError = GetIpInterfaceEntry(&IpInterface);
+ if (LastError != ERROR_SUCCESS)
+ {
+ LogError(L"Failed to get IP interface", LastError);
+ goto cleanupAdapter;
+ }
+ IpInterface.UseAutomaticMetric = FALSE;
+ IpInterface.Metric = 0;
+ IpInterface.NlMtu = 1420;
+ IpInterface.SitePrefixLength = 0;
+ LastError = SetIpInterfaceEntry(&IpInterface);
+ if (LastError != ERROR_SUCCESS)
+ {
+ LogError(L"Failed to set metric and MTU", LastError);
+ goto cleanupAdapter;
+ }
+
+ Log(WIREGUARD_LOG_INFO, L"Setting configuration and adapter up");
+ if (!WireGuardSetConfiguration(Adapter, &Config.Interface, sizeof(Config)) ||
+ !WireGuardSetAdapterState(Adapter, WIREGUARD_ADAPTER_STATE_UP))
+ {
+ LastError = LogError(L"Failed to set configuration and adapter up", GetLastError());
+ goto cleanupAdapter;
+ }
+
+ do
+ {
+ Bytes = sizeof(Config);
+ if (!WireGuardGetConfiguration(Adapter, &Config.Interface, &Bytes) || !Config.Interface.PeersCount)
+ {
+ LastError = LogError(L"Failed to get configuration", GetLastError());
+ goto cleanupAdapter;
+ }
+ FILETIME Timestamp;
+ GetSystemTimePreciseAsFileTime(&Timestamp);
+ SYSTEMTIME SystemTime;
+ FileTimeToSystemTime(&Timestamp, &SystemTime);
+ fwprintf(
+ stderr,
+ L"%04u-%02u-%02u %02u:%02u:%02u.%04u [#] RX: %llu, TX: %llu\r",
+ SystemTime.wYear,
+ SystemTime.wMonth,
+ SystemTime.wDay,
+ SystemTime.wHour,
+ SystemTime.wMinute,
+ SystemTime.wSecond,
+ SystemTime.wMilliseconds,
+ Config.DemoServer.RxBytes,
+ Config.DemoServer.TxBytes);
+ } while (WaitForSingleObject(QuitEvent, 1000) == WAIT_TIMEOUT);
+
+cleanupAdapter:
+ WireGuardDeleteAdapter(Adapter, NULL);
+ WireGuardFreeAdapter(Adapter);
+cleanupQuit:
+ SetConsoleCtrlHandler(CtrlHandler, FALSE);
+ CloseHandle(QuitEvent);
+cleanupWireGuard:
+ FreeLibrary(WireGuard);
+cleanupWinsock:
+ WSACleanup();
+ return LastError;
+}
diff --git a/example/example.vcxproj b/example/example.vcxproj new file mode 100644 index 0000000..dbf4f1f --- /dev/null +++ b/example/example.vcxproj @@ -0,0 +1,42 @@ +<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{5B234798-88FB-49CE-A583-79CA31C8F801}</ProjectGuid>
+ <RootNamespace>example</RootNamespace>
+ <ProjectName>example</ProjectName>
+ </PropertyGroup>
+ <PropertyGroup Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <PlatformToolset>WindowsApplicationForDrivers10.0</PlatformToolset>
+ <ForcedTargetVersion>Windows10</ForcedTargetVersion>
+ </PropertyGroup>
+ <Import Project="..\wireguard-nt.props" />
+ <ItemDefinitionGroup>
+ <ClCompile>
+ <PreprocessorDefinitions>_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <DisableSpecificWarnings>4100;$(DisableSpecificWarnings)</DisableSpecificWarnings>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup>
+ <ClCompile>
+ <AdditionalIncludeDirectories>..\api</AdditionalIncludeDirectories>
+ </ClCompile>
+ <Link>
+ <AdditionalDependencies>bcrypt.lib;crypt32.lib;iphlpapi.lib;kernel32.lib;ntdll.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClCompile Include="example.c" />
+ </ItemGroup>
+ <ItemGroup>
+ <ProjectReference Include="..\api\api.vcxproj">
+ <Project>{99648503-7DFB-4C06-A87A-E7B66E93FF84}</Project>
+ </ProjectReference>
+ </ItemGroup>
+ <Import Project="..\wireguard-nt.props.user" Condition="exists('..\wireguard-nt.props.user')" />
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets" />
+</Project>
diff --git a/example/example.vcxproj.filters b/example/example.vcxproj.filters new file mode 100644 index 0000000..eb29fb9 --- /dev/null +++ b/example/example.vcxproj.filters @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup> + <Filter Include="Source Files"> + <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier> + <Extensions>cpp;c;cc;cxx;c++;cppm;ixx;def;odl;idl;hpj;bat;asm;asmx</Extensions> + </Filter> + </ItemGroup> + <ItemGroup> + <ClCompile Include="example.c"> + <Filter>Source Files</Filter> + </ClCompile> + </ItemGroup> +</Project>
\ No newline at end of file diff --git a/prebuilt-binaries-license.txt b/prebuilt-binaries-license.txt new file mode 100644 index 0000000..5f83775 --- /dev/null +++ b/prebuilt-binaries-license.txt @@ -0,0 +1,84 @@ +Prebuilt Binaries License +------------------------- + +1. DEFINITIONS. "Software" means the precise contents of the "wireguard.dll" + files that are included in the .zip file that contains this document as + downloaded from download.wireguard.com. + +2. LICENSE GRANT. WireGuard LLC grants to you a non-exclusive and + non-transferable right to use Software for lawful purposes under certain + obligations and limited rights as set forth in this agreement. + +3. RESTRICTIONS. Software is owned and copyrighted by WireGuard LLC. It is + licensed, not sold. Title to Software and all associated intellectual + property rights are retained by WireGuard. You must not: + a. reverse engineer, decompile, disassemble, extract from, or otherwise + modify the Software; + b. modify or create derivative work based upon Software in whole or in + parts, except insofar as only the API interfaces of the "wireguard.h" file + distributed alongside the Software (the "Permitted API") are used; + c. remove any proprietary notices, labels, or copyrights from the Software; + d. resell, redistribute, lease, rent, transfer, sublicense, or otherwise + transfer rights of the Software without the prior written consent of + WireGuard LLC, except insofar as the Software is distributed alongside + other software that uses the Software only via the Permitted API; + e. use the name of WireGuard LLC, the WireGuard project, the WireGuard + project, or the names of its contributors to endorse or promote products + derived from the Software without specific prior written consent. + +4. LIMITED WARRANTY. THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTY OF + ANY KIND. WIREGUARD LLC HEREBY EXCLUDES AND DISCLAIMS ALL IMPLIED OR + STATUTORY WARRANTIES, INCLUDING ANY WARRANTIES OF MERCHANTABILITY, FITNESS + FOR A PARTICULAR PURPOSE, QUALITY, NON-INFRINGEMENT, TITLE, RESULTS, + EFFORTS, OR QUIET ENJOYMENT. THERE IS NO WARRANTY THAT THE PRODUCT WILL BE + ERROR-FREE OR WILL FUNCTION WITHOUT INTERRUPTION. YOU ASSUME THE ENTIRE + RISK FOR THE RESULTS OBTAINED USING THE PRODUCT. TO THE EXTENT THAT + WIREGUARD LLC MAY NOT DISCLAIM ANY WARRANTY AS A MATTER OF APPLICABLE LAW, + THE SCOPE AND DURATION OF SUCH WARRANTY WILL BE THE MINIMUM PERMITTED UNDER + SUCH LAW. ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND + WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR + A PARTICULAR PURPOSE OR NON-INFRINGEMENT ARE DISCLAIMED, EXCEPT TO THE + EXTENT THAT THESE DISCLAIMERS ARE HELD TO BE LEGALLY INVALID. + +5. LIMITATION OF LIABILITY. To the extent not prohibited by law, in no event + WireGuard LLC or any third-party-developer will be liable for any lost + revenue, profit or data or for special, indirect, consequential, incidental + or punitive damages, however caused regardless of the theory of liability, + arising out of or related to the use of or inability to use Software, even + if WireGuard LLC has been advised of the possibility of such damages. + Solely you are responsible for determining the appropriateness of using + Software and accept full responsibility for all risks associated with its + exercise of rights under this agreement, including but not limited to the + risks and costs of program errors, compliance with applicable laws, damage + to or loss of data, programs or equipment, and unavailability or + interruption of operations. The foregoing limitations will apply even if + the above stated warranty fails of its essential purpose. You acknowledge, + that it is in the nature of software that software is complex and not + completely free of errors. In no event shall WireGuard LLC or any + third-party-developer be liable to you under any theory for any damages + suffered by you or any user of Software or for any special, incidental, + indirect, consequential or similar damages (including without limitation + damages for loss of business profits, business interruption, loss of + business information or any other pecuniary loss) arising out of the use or + inability to use Software, even if WireGuard LLC has been advised of the + possibility of such damages and regardless of the legal or quitable theory + (contract, tort, or otherwise) upon which the claim is based. + +6. TERMINATION. This agreement is affected until terminated. You may + terminate this agreement at any time. This agreement will terminate + immediately without notice from WireGuard LLC if you fail to comply with + the terms and conditions of this agreement. Upon termination, you must + delete Software and all copies of Software and cease all forms of + distribution of Software. + +7. SEVERABILITY. If any provision of this agreement is held to be + unenforceable, this agreement will remain in effect with the provision + omitted, unless omission would frustrate the intent of the parties, in + which case this agreement will immediately terminate. + +8. RESERVATION OF RIGHTS. All rights not expressly granted in this agreement + are reserved by WireGuard LLC. For example, WireGuard LLC reserves the + right at any time to cease development of Software, to alter distribution + details, features, specifications, capabilities, functions, licensing + terms, release dates, APIs, ABIs, general availability, or other + characteristics of the Software. diff --git a/wireguard-nt.proj b/wireguard-nt.proj new file mode 100644 index 0000000..77041d9 --- /dev/null +++ b/wireguard-nt.proj @@ -0,0 +1,118 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- + SPDX-License-Identifier: GPL-2.0 + + Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. +--> +<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003" DefaultTargets="Build"> + <PropertyGroup> + <Configuration Condition="'$(Configuration)'==''">Release</Configuration> + </PropertyGroup> + + <!-- + General Properties + --> + <Import Project="wireguard-nt.props" /> + <Import Project="wireguard-nt.props.user" Condition="exists('wireguard-nt.props.user')" /> + + <Target Name="Driver" DependsOnTargets="Driver-x86;Driver-amd64;Driver-arm;Driver-arm64" /> + <Target Name="Dll" DependsOnTargets="Dll-x86;Dll-amd64;Dll-arm;Dll-arm64" /> + <Target Name="Clean"> + <RemoveDir Directories="$(Configuration)\amd64\" /> + <RemoveDir Directories="$(Configuration)\arm\" /> + <RemoveDir Directories="$(Configuration)\arm64\" /> + <RemoveDir Directories="$(Configuration)\x86\" /> + <RemoveDir Directories="driver\sdv\;dist\" /> + <Delete Files="driver\smvbuild.log;driver\smvstats.txt;driver\wireguard.DVL.XML" /> + </Target> + + <Target Name="Driver-x86" + Outputs="$(Configuration)\x86\driver\wireguard.sys;$(Configuration)\x86\driver\wireguard.inf;$(Configuration)\x86\driver\wireguard.cat"> + <MSBuild Projects="driver\driver.vcxproj" Targets="Build" Properties="Configuration=$(Configuration);Platform=Win32" /> + </Target> + <Target Name="Driver-amd64" + Outputs="$(Configuration)\amd64\driver\wireguard.sys;$(Configuration)\amd64\driver\wireguard.inf;$(Configuration)\amd64\driver\wireguard.cat"> + <MSBuild Projects="driver\driver.vcxproj" Targets="Build" Properties="Configuration=$(Configuration);Platform=x64" /> + </Target> + <Target Name="Driver-arm" + Outputs="$(Configuration)\arm\driver\wireguard.sys;$(Configuration)\arm\driver\wireguard.inf;$(Configuration)\arm\driver\wireguard.cat"> + <MSBuild Projects="driver\driver.vcxproj" Targets="Build" Properties="Configuration=$(Configuration);Platform=ARM" /> + </Target> + <Target Name="Driver-arm64" + Outputs="$(Configuration)\arm64\driver\wireguard.sys;$(Configuration)\arm64\driver\wireguard.inf;$(Configuration)\arm64\driver\wireguard.cat"> + <MSBuild Projects="driver\driver.vcxproj" Targets="Build" Properties="Configuration=$(Configuration);Platform=ARM64" /> + </Target> + + <!-- + Static Driver Verifier + --> + <Target Name="SDV" Outputs="driver\sdv\SDV.DVL.xml"> + <Exec WorkingDirectory="driver" Command="msbuild.exe driver.vcxproj /t:sdv /p:Inputs=/check:*;Configuration=$(Configuration);Platform=x64;SDVHacks=true /nologo" /> + </Target> + <Target Name="SDVView"> + <Exec WorkingDirectory="driver" Command="msbuild.exe driver.vcxproj /t:sdv /p:Inputs=/view;Configuration=$(Configuration);Platform=x64;SDVHacks=true /nologo" /> + </Target> + + <!-- + Driver Verification Log + --> + <Target Name="DVL" DependsOnTargets="SDV" + Outputs="driver\wireguard.DVL.XML" + Inputs="driver\sdv\SDV.DVL.xml"> + <MSBuild Projects="driver\driver.vcxproj" Targets="Build" Properties="Configuration=$(Configuration);Platform=x64;RunCodeAnalysis=true" /> + <Exec WorkingDirectory="driver" Command="msbuild.exe driver.vcxproj /t:dvl /p:Configuration=$(Configuration);Platform=x64 /nologo" /> + </Target> + + <!-- + wireguard.dll Building + --> + <Target Name="Dll-x86" + Outputs="$(Configuration)\x86\wireguard.dll" + DependsOnTargets="Dll-amd64;Dll-arm64"> + <MSBuild Projects="api\api.vcxproj" Targets="Build" Properties="Configuration=$(Configuration);Platform=Win32" /> + </Target> + <Target Name="Dll-amd64" + Outputs="$(Configuration)\amd64\wireguard.dll" + DependsOnTargets="Dll-arm64"> + <MSBuild Projects="api\api.vcxproj" Targets="Build" Properties="Configuration=$(Configuration);Platform=x64" /> + </Target> + <Target Name="Dll-arm" + Outputs="$(Configuration)\arm\wireguard.dll" + DependsOnTargets="Dll-arm64"> + <MSBuild Projects="api\api.vcxproj" Targets="Build" Properties="Configuration=$(Configuration);Platform=ARM" /> + </Target> + <Target Name="Dll-arm64" + Outputs="$(Configuration)\arm64\wireguard.dll"> + <MSBuild Projects="api\api.vcxproj" Targets="Build" Properties="Configuration=$(Configuration);Platform=ARM64" /> + </Target> + + <!-- + Zip Building + --> + <PropertyGroup> + <ZipTargetPath>dist\wireguard-nt-$(WireGuardVersion).zip</ZipTargetPath> + <ZipIntDir>dist\zip-intermediate\</ZipIntDir> + </PropertyGroup> + <ItemGroup> + <ZipFilesSrc Include="prebuilt-binaries-license.txt" /><ZipFilesDst Include="$(ZipIntDir)wireguard-nt\LICENSE.txt" /> + <ZipFilesSrc Include="README.md" /><ZipFilesDst Include="$(ZipIntDir)wireguard-nt\README.md" /> + <ZipFilesSrc Include="api\wireguard.h" /><ZipFilesDst Include="$(ZipIntDir)wireguard-nt\include\wireguard.h" /> + <ZipFilesSrc Include="$(Configuration)\amd64\wireguard.dll" /><ZipFilesDst Include="$(ZipIntDir)wireguard-nt\bin\amd64\wireguard.dll" /> + <ZipFilesSrc Include="$(Configuration)\arm\wireguard.dll" /><ZipFilesDst Include="$(ZipIntDir)wireguard-nt\bin\arm\wireguard.dll" /> + <ZipFilesSrc Include="$(Configuration)\arm64\wireguard.dll" /><ZipFilesDst Include="$(ZipIntDir)wireguard-nt\bin\arm64\wireguard.dll" /> + <ZipFilesSrc Include="$(Configuration)\x86\wireguard.dll" /><ZipFilesDst Include="$(ZipIntDir)wireguard-nt\bin\x86\wireguard.dll" /> + </ItemGroup> + <Target Name="Zip" + Inputs="@(ZipFilesSrc)" + Outputs="$(ZipTargetPath)" + DependsOnTargets="Dll"> + <RemoveDir Directories="$(ZipIntDir)" /> + <Copy SourceFiles="@(ZipFilesSrc)" DestinationFiles="@(ZipFilesDst)" /> + <ZipDirectory DestinationFile="$(ZipTargetPath)" Overwrite="true" SourceDirectory="$(ZipIntDir)" /> + <RemoveDir Directories="$(ZipIntDir)" /> + <GetFileHash Files="$(ZipTargetPath)" Algorithm="SHA256" HashEncoding="hex"> + <Output TaskParameter="Items" ItemName="InstallerLibraryHash" /> + </GetFileHash> + <Message Text="SHA256("$(ZipTargetPath)") = @(InstallerLibraryHash->Metadata('FileHash')->ToLower())"/> + </Target> +</Project> diff --git a/wireguard-nt.props b/wireguard-nt.props new file mode 100644 index 0000000..dfe919a --- /dev/null +++ b/wireguard-nt.props @@ -0,0 +1,159 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- + SPDX-License-Identifier: GPL-2.0 + + Copyright (C) 2018-2021 WireGuard LLC. All Rights Reserved. +--> +<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup Label="ProjectConfigurations"> + <ProjectConfiguration Include="Debug|ARM"> + <Configuration>Debug</Configuration> + <Platform>ARM</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Debug|ARM64"> + <Configuration>Debug</Configuration> + <Platform>ARM64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Debug|Win32"> + <Configuration>Debug</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Debug|x64"> + <Configuration>Debug</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|ARM"> + <Configuration>Release</Configuration> + <Platform>ARM</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|ARM64"> + <Configuration>Release</Configuration> + <Platform>ARM64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|Win32"> + <Configuration>Release</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|x64"> + <Configuration>Release</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + </ItemGroup> + <PropertyGroup Label="Globals"> + <VCProjectVersion>16.0</VCProjectVersion> + <WindowsTargetPlatformVersion>$(LatestTargetPlatformVersion)</WindowsTargetPlatformVersion> + </PropertyGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> + <PropertyGroup Label="Configuration"> + <CharacterSet>Unicode</CharacterSet> + <Driver_SpectreMitigation>false</Driver_SpectreMitigation> + <Inf2CatUseLocalTime Condition="'$(ConfigurationType)'=='Driver'">true</Inf2CatUseLocalTime> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration"> + <WholeProgramOptimization>true</WholeProgramOptimization> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration"> + <UseDebugLibraries>true</UseDebugLibraries> + </PropertyGroup> + <PropertyGroup Condition="'$(Platform)'=='Win32'" Label="Configuration"> + <TargetVersion>Windows7</TargetVersion> + </PropertyGroup> + <PropertyGroup Condition="'$(Platform)'=='ARM'" Label="Configuration"> + <TargetVersion>Windows8</TargetVersion> + </PropertyGroup> + <PropertyGroup Condition="'$(Platform)'=='x64'" Label="Configuration"> + <TargetVersion>Windows7</TargetVersion> + </PropertyGroup> + <PropertyGroup Condition="'$(Platform)'=='ARM64'" Label="Configuration"> + <TargetVersion>Windows10</TargetVersion> + </PropertyGroup> + <PropertyGroup Condition="'$(ForcedTargetVersion)'!=''"> + <TargetVersion>$(ForcedTargetVersion)</TargetVersion> + </PropertyGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> + <ImportGroup Label="ExtensionSettings" /> + <ImportGroup Label="PropertySheets"> + <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> + </ImportGroup> + <PropertyGroup Label="UserMacros" /> + <PropertyGroup> + <WireGuardVersionMaj>0</WireGuardVersionMaj> + <WireGuardVersionMin>1</WireGuardVersionMin> + <WireGuardVersionRel>0</WireGuardVersionRel> + + <!-- Used for versioning, must be n.n[.n[.n]]. --> + <WireGuardVersion>$(WireGuardVersionMaj).$(WireGuardVersionMin)</WireGuardVersion> + <WireGuardVersion Condition="'$(WireGuardVersionRel)'!='0'">$(WireGuardVersion).$(WireGuardVersionRel)</WireGuardVersion> + + <!-- .vcxproj are using different platform names. --> + <WireGuardPlatform Condition="'$(Platform)'=='ARM'">arm</WireGuardPlatform> + <WireGuardPlatform Condition="'$(Platform)'=='ARM64'">arm64</WireGuardPlatform> + <WireGuardPlatform Condition="'$(Platform)'=='Win32'">x86</WireGuardPlatform> + <WireGuardPlatform Condition="'$(Platform)'=='x64'">amd64</WireGuardPlatform> + + <IntDir>..\$(Configuration)\$(WireGuardPlatform)\$(ProjectName)-intermediate\</IntDir> + <OutDir>..\$(Configuration)\$(WireGuardPlatform)\</OutDir> + <OutDir Condition="'$(ConfigurationType)'=='Driver'">$(IntDir)</OutDir> + <PackageDir>..\$(Configuration)\$(WireGuardPlatform)\$(ProjectName)\</PackageDir> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)'=='Debug'"> + <RunCodeAnalysis>true</RunCodeAnalysis> + </PropertyGroup> + <PropertyGroup Condition="'$(RunCodeAnalysis)'=='true'"> + <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet> + <ClCompile> + <EnablePREfast>true</EnablePREfast> + </ClCompile> + </PropertyGroup> + <ItemDefinitionGroup> + <ClCompile> + <PreprocessorDefinitions>WIREGUARD_VERSION_MAJ=$(WireGuardVersionMaj);WIREGUARD_VERSION_MIN=$(WireGuardVersionMin);WIREGUARD_VERSION_REL=$(WireGuardVersionRel);WIREGUARD_VERSION="$(WireGuardVersion)";%(PreprocessorDefinitions)</PreprocessorDefinitions> + <WarningLevel>Level4</WarningLevel> + </ClCompile> + <ResourceCompile> + <PreprocessorDefinitions>WIREGUARD_VERSION_MAJ=$(WireGuardVersionMaj);WIREGUARD_VERSION_MIN=$(WireGuardVersionMin);WIREGUARD_VERSION_REL=$(WireGuardVersionRel);WIREGUARD_VERSION="$(WireGuardVersion)";%(PreprocessorDefinitions)</PreprocessorDefinitions> + </ResourceCompile> + <Link> + <ProgramDatabaseFile>$(OutDir)$(TargetName).pdb</ProgramDatabaseFile> + <ProgramDatabaseFile Condition="'$(ConfigurationType)'=='Driver'">$(PackageDir)$(TargetName).pdb</ProgramDatabaseFile> + </Link> + <DriverSign> + <FileDigestAlgorithm>sha256</FileDigestAlgorithm> + </DriverSign> + </ItemDefinitionGroup> + <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'"> + <ClCompile> + <Optimization>MaxSpeed</Optimization> + <IntrinsicFunctions>true</IntrinsicFunctions> + <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed> + </ClCompile> + <Link> + <EnableCOMDATFolding>true</EnableCOMDATFolding> + <OptimizeReferences>true</OptimizeReferences> + <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration> + </Link> + </ItemDefinitionGroup> + <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'"> + <ClCompile> + <Optimization>Disabled</Optimization> + </ClCompile> + </ItemDefinitionGroup> + <ItemDefinitionGroup Condition="'$(PlatformToolset)|$(Configuration)'=='WindowsApplicationForDrivers10.0|Release'"> + <ClCompile> + <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <RuntimeLibrary>MultiThreaded</RuntimeLibrary> + </ClCompile> + <ResourceCompile> + <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions> + </ResourceCompile> + </ItemDefinitionGroup> + <ItemDefinitionGroup Condition="'$(PlatformToolset)|$(Configuration)'=='WindowsApplicationForDrivers10.0|Debug'"> + <ClCompile> + <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> + </ClCompile> + <ResourceCompile> + <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions> + </ResourceCompile> + </ItemDefinitionGroup> +</Project> diff --git a/wireguard-nt.sln b/wireguard-nt.sln new file mode 100644 index 0000000..99b0600 --- /dev/null +++ b/wireguard-nt.sln @@ -0,0 +1,89 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.28922.388 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "example", "example\example.vcxproj", "{5B234798-88FB-49CE-A583-79CA31C8F801}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "api", "api\api.vcxproj", "{99648503-7DFB-4C06-A87A-E7B66E93FF84}" + ProjectSection(ProjectDependencies) = postProject + {8B282C8F-5870-44C3-9A2A-B9091F4E9F68} = {8B282C8F-5870-44C3-9A2A-B9091F4E9F68} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "driver", "driver\driver.vcxproj", "{8B282C8F-5870-44C3-9A2A-B9091F4E9F68}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{3A98F138-EE02-4488-B856-B3C48500BEA8}" + ProjectSection(SolutionItems) = preProject + README.md = README.md + TODO.md = TODO.md + wireguard-nt.proj = wireguard-nt.proj + wireguard-nt.props = wireguard-nt.props + EndProjectSection +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|amd64 = Debug|amd64 + Debug|arm = Debug|arm + Debug|arm64 = Debug|arm64 + Debug|x86 = Debug|x86 + Release|amd64 = Release|amd64 + Release|arm = Release|arm + Release|arm64 = Release|arm64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {5B234798-88FB-49CE-A583-79CA31C8F801}.Debug|amd64.ActiveCfg = Debug|x64 + {5B234798-88FB-49CE-A583-79CA31C8F801}.Debug|amd64.Build.0 = Debug|x64 + {5B234798-88FB-49CE-A583-79CA31C8F801}.Debug|arm.ActiveCfg = Debug|ARM + {5B234798-88FB-49CE-A583-79CA31C8F801}.Debug|arm.Build.0 = Debug|ARM + {5B234798-88FB-49CE-A583-79CA31C8F801}.Debug|arm64.ActiveCfg = Debug|ARM64 + {5B234798-88FB-49CE-A583-79CA31C8F801}.Debug|arm64.Build.0 = Debug|ARM64 + {5B234798-88FB-49CE-A583-79CA31C8F801}.Debug|x86.ActiveCfg = Debug|Win32 + {5B234798-88FB-49CE-A583-79CA31C8F801}.Debug|x86.Build.0 = Debug|Win32 + {5B234798-88FB-49CE-A583-79CA31C8F801}.Release|amd64.ActiveCfg = Release|x64 + {5B234798-88FB-49CE-A583-79CA31C8F801}.Release|amd64.Build.0 = Release|x64 + {5B234798-88FB-49CE-A583-79CA31C8F801}.Release|arm.ActiveCfg = Release|ARM + {5B234798-88FB-49CE-A583-79CA31C8F801}.Release|arm.Build.0 = Release|ARM + {5B234798-88FB-49CE-A583-79CA31C8F801}.Release|arm64.ActiveCfg = Release|ARM64 + {5B234798-88FB-49CE-A583-79CA31C8F801}.Release|arm64.Build.0 = Release|ARM64 + {5B234798-88FB-49CE-A583-79CA31C8F801}.Release|x86.ActiveCfg = Release|Win32 + {5B234798-88FB-49CE-A583-79CA31C8F801}.Release|x86.Build.0 = Release|Win32 + {99648503-7DFB-4C06-A87A-E7B66E93FF84}.Debug|amd64.ActiveCfg = Debug|x64 + {99648503-7DFB-4C06-A87A-E7B66E93FF84}.Debug|amd64.Build.0 = Debug|x64 + {99648503-7DFB-4C06-A87A-E7B66E93FF84}.Debug|arm.ActiveCfg = Debug|ARM + {99648503-7DFB-4C06-A87A-E7B66E93FF84}.Debug|arm.Build.0 = Debug|ARM + {99648503-7DFB-4C06-A87A-E7B66E93FF84}.Debug|arm64.ActiveCfg = Debug|ARM64 + {99648503-7DFB-4C06-A87A-E7B66E93FF84}.Debug|arm64.Build.0 = Debug|ARM64 + {99648503-7DFB-4C06-A87A-E7B66E93FF84}.Debug|x86.ActiveCfg = Debug|Win32 + {99648503-7DFB-4C06-A87A-E7B66E93FF84}.Debug|x86.Build.0 = Debug|Win32 + {99648503-7DFB-4C06-A87A-E7B66E93FF84}.Release|amd64.ActiveCfg = Release|x64 + {99648503-7DFB-4C06-A87A-E7B66E93FF84}.Release|amd64.Build.0 = Release|x64 + {99648503-7DFB-4C06-A87A-E7B66E93FF84}.Release|arm.ActiveCfg = Release|ARM + {99648503-7DFB-4C06-A87A-E7B66E93FF84}.Release|arm.Build.0 = Release|ARM + {99648503-7DFB-4C06-A87A-E7B66E93FF84}.Release|arm64.ActiveCfg = Release|ARM64 + {99648503-7DFB-4C06-A87A-E7B66E93FF84}.Release|arm64.Build.0 = Release|ARM64 + {99648503-7DFB-4C06-A87A-E7B66E93FF84}.Release|x86.ActiveCfg = Release|Win32 + {99648503-7DFB-4C06-A87A-E7B66E93FF84}.Release|x86.Build.0 = Release|Win32 + {8B282C8F-5870-44C3-9A2A-B9091F4E9F68}.Debug|amd64.ActiveCfg = Debug|x64 + {8B282C8F-5870-44C3-9A2A-B9091F4E9F68}.Debug|amd64.Build.0 = Debug|x64 + {8B282C8F-5870-44C3-9A2A-B9091F4E9F68}.Debug|arm.ActiveCfg = Debug|ARM + {8B282C8F-5870-44C3-9A2A-B9091F4E9F68}.Debug|arm.Build.0 = Debug|ARM + {8B282C8F-5870-44C3-9A2A-B9091F4E9F68}.Debug|arm64.ActiveCfg = Debug|ARM64 + {8B282C8F-5870-44C3-9A2A-B9091F4E9F68}.Debug|arm64.Build.0 = Debug|ARM64 + {8B282C8F-5870-44C3-9A2A-B9091F4E9F68}.Debug|x86.ActiveCfg = Debug|Win32 + {8B282C8F-5870-44C3-9A2A-B9091F4E9F68}.Debug|x86.Build.0 = Debug|Win32 + {8B282C8F-5870-44C3-9A2A-B9091F4E9F68}.Release|amd64.ActiveCfg = Release|x64 + {8B282C8F-5870-44C3-9A2A-B9091F4E9F68}.Release|amd64.Build.0 = Release|x64 + {8B282C8F-5870-44C3-9A2A-B9091F4E9F68}.Release|arm.ActiveCfg = Release|ARM + {8B282C8F-5870-44C3-9A2A-B9091F4E9F68}.Release|arm.Build.0 = Release|ARM + {8B282C8F-5870-44C3-9A2A-B9091F4E9F68}.Release|arm64.ActiveCfg = Release|ARM64 + {8B282C8F-5870-44C3-9A2A-B9091F4E9F68}.Release|arm64.Build.0 = Release|ARM64 + {8B282C8F-5870-44C3-9A2A-B9091F4E9F68}.Release|x86.ActiveCfg = Release|Win32 + {8B282C8F-5870-44C3-9A2A-B9091F4E9F68}.Release|x86.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {254563AF-E0B3-48F0-B564-A45AEC95591F} + EndGlobalSection +EndGlobal |