diff --git a/.vs/cmake.db b/.vs/cmake.db index a159c02..4fbcad1 100644 Binary files a/.vs/cmake.db and b/.vs/cmake.db differ diff --git a/.vs/sishu-yolo-sdk/FileContentIndex/0b356054-dc3e-4eb7-ab9d-8fd60b6d8421.vsidx b/.vs/sishu-yolo-sdk/FileContentIndex/0b356054-dc3e-4eb7-ab9d-8fd60b6d8421.vsidx deleted file mode 100644 index 36636bd..0000000 Binary files a/.vs/sishu-yolo-sdk/FileContentIndex/0b356054-dc3e-4eb7-ab9d-8fd60b6d8421.vsidx and /dev/null differ diff --git a/.vs/sishu-yolo-sdk/FileContentIndex/d32ecd9e-418f-4d7a-aa90-55b1f7915ca8.vsidx b/.vs/sishu-yolo-sdk/FileContentIndex/d32ecd9e-418f-4d7a-aa90-55b1f7915ca8.vsidx deleted file mode 100644 index 4c7cc84..0000000 Binary files a/.vs/sishu-yolo-sdk/FileContentIndex/d32ecd9e-418f-4d7a-aa90-55b1f7915ca8.vsidx and /dev/null differ diff --git a/.vs/sishu-yolo-sdk/FileContentIndex/fe252d52-eb72-45ab-b4ef-cdf95b2a7fdb.vsidx b/.vs/sishu-yolo-sdk/FileContentIndex/fe252d52-eb72-45ab-b4ef-cdf95b2a7fdb.vsidx deleted file mode 100644 index 150a5e2..0000000 Binary files a/.vs/sishu-yolo-sdk/FileContentIndex/fe252d52-eb72-45ab-b4ef-cdf95b2a7fdb.vsidx and /dev/null differ diff --git a/.vs/sishu-yolo-sdk/v17/.wsuo b/.vs/sishu-yolo-sdk/v17/.wsuo index 2d8e2a8..3c5d7dc 100644 Binary files a/.vs/sishu-yolo-sdk/v17/.wsuo and b/.vs/sishu-yolo-sdk/v17/.wsuo differ diff --git a/.vs/sishu-yolo-sdk/v17/Browse.VC.db b/.vs/sishu-yolo-sdk/v17/Browse.VC.db index 0b70e3d..6cf627d 100644 Binary files a/.vs/sishu-yolo-sdk/v17/Browse.VC.db and b/.vs/sishu-yolo-sdk/v17/Browse.VC.db differ diff --git a/.vs/sishu-yolo-sdk/v17/ipch/AutoPCH/316448d41650dc7c/YOLOSDK_JNI.ipch b/.vs/sishu-yolo-sdk/v17/ipch/AutoPCH/316448d41650dc7c/YOLOSDK_JNI.ipch index bd54f53..522f9dc 100644 Binary files a/.vs/sishu-yolo-sdk/v17/ipch/AutoPCH/316448d41650dc7c/YOLOSDK_JNI.ipch and b/.vs/sishu-yolo-sdk/v17/ipch/AutoPCH/316448d41650dc7c/YOLOSDK_JNI.ipch differ diff --git a/.vs/sishu-yolo-sdk/v17/ipch/AutoPCH/64df2d83173fa675/YOLOCORE.ipch b/.vs/sishu-yolo-sdk/v17/ipch/AutoPCH/64df2d83173fa675/YOLOCORE.ipch index 974c737..50e1d0e 100644 Binary files a/.vs/sishu-yolo-sdk/v17/ipch/AutoPCH/64df2d83173fa675/YOLOCORE.ipch and b/.vs/sishu-yolo-sdk/v17/ipch/AutoPCH/64df2d83173fa675/YOLOCORE.ipch differ diff --git a/.vs/slnx.sqlite b/.vs/slnx.sqlite index f5dc46b..3c9303c 100644 Binary files a/.vs/slnx.sqlite and b/.vs/slnx.sqlite differ diff --git a/README.md b/README.md index f6909a1..c508290 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,8 @@ javac -h cpp/include src/main/java/com/bonus/sdk/YoloSdk.java src/main/java/com/bonus/sdk/Detection.java windows -cmake -S cpp -B build/windows -G "Visual Studio 17 2022" -A x64 -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\scripts\buildsystems\vcpkg.cmake - +# (ÔÚ E:\sishu-yolo-sdk Ŀ¼ÏÂÔËÐÐ) +cmake -S cpp -B build/windows -G "Visual Studio 17 2022" -A x64 -DCMAKE_TOOLCHAIN_FILE="E:/vcpkg/scripts/buildsystems/vcpkg.cmake" cmake --build build/windows --config Release linux diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 0b8a9e0..098840b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -4,14 +4,18 @@ set(CMAKE_CXX_STANDARD 17) if(WIN32) - find_package(JNI COMPONENTS JNI REQUIRED) + set(JNI_INCLUDE_DIRS + "C:/Program Files/Microsoft/jdk-11.0.29.7-hotspot/include" + "C:/Program Files/Microsoft/jdk-11.0.29.7-hotspot/include/win32" + ) + set(JNI_LIBRARIES "C:/Program Files/Microsoft/jdk-11.0.29.7-hotspot/lib/jvm.lib") message(STATUS "Configuring for Windows (using vcpkg OpenCV + manual ONNX)") # find_package(OpenCV REQUIRED) # - set(ORT_MANUAL_PATH ${CMAKE_SOURCE_DIR}/../prebuilt_libs/onnxruntime-win-x64-1.23.2) + set(ORT_MANUAL_PATH ${CMAKE_SOURCE_DIR}/../prebuilt_libs/onnxruntime-win-x64-1.18.0) elseif(UNIX) set(JNI_INCLUDE_DIRS diff --git a/cpp/src/YoloCore.cpp b/cpp/src/YoloCore.cpp index af5c1c8..2b2f75e 100644 --- a/cpp/src/YoloCore.cpp +++ b/cpp/src/YoloCore.cpp @@ -96,7 +96,7 @@ YoloDetector::YoloDetector(const ORTCHAR_T* model_path, int in_width, int in_hei Ort::SessionOptions session_options; - // session_options.AppendExecutionProvider_CPU(0); + //session_options.AppendExecutionProvider("XnnpackExecutionProvider"); session = std::make_unique(env, model_path, session_options); diff --git a/pom.xml b/pom.xml index 822b084..63850ce 100644 --- a/pom.xml +++ b/pom.xml @@ -50,6 +50,9 @@ false my_yolo_sdk.dll + abseil_dll.dll + libprotobuf.dll + zlib1.dll onnxruntime.dll opencv_core4.dll opencv_dnn4.dll diff --git a/prebuilt_libs/onnxruntime-win-x64-1.18.0/CodeSignSummary-33acd3d3-018c-4973-802b-e92fbf2c79a1.md b/prebuilt_libs/onnxruntime-win-x64-1.18.0/CodeSignSummary-33acd3d3-018c-4973-802b-e92fbf2c79a1.md new file mode 100644 index 0000000..6ef1fff --- /dev/null +++ b/prebuilt_libs/onnxruntime-win-x64-1.18.0/CodeSignSummary-33acd3d3-018c-4973-802b-e92fbf2c79a1.md @@ -0,0 +1,4 @@ + | FileName | Status | FileSize | TotalTime(sec) | Upload(sec) | Submit(sec) | SignWait(sec) | Retry Count | + |---------------------------|--------|----------|----------------|-------------|-------------|---------------|-------------| + | onnxruntime.dll | Pass | 10.3MB | 64.24 | 1.63 | 0.49 | 62.12 | 0 | + | time_providers_shared.dll | Pass | 11.5KB | 74.9 | 0.97 | 0.43 | 72.78 | 0 | diff --git a/prebuilt_libs/onnxruntime-win-x64-1.18.0/GIT_COMMIT_ID b/prebuilt_libs/onnxruntime-win-x64-1.18.0/GIT_COMMIT_ID new file mode 100644 index 0000000..d8661a5 --- /dev/null +++ b/prebuilt_libs/onnxruntime-win-x64-1.18.0/GIT_COMMIT_ID @@ -0,0 +1 @@ +45737400a2f3015c11f005ed7603611eaed306a6 diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/LICENSE b/prebuilt_libs/onnxruntime-win-x64-1.18.0/LICENSE similarity index 100% rename from prebuilt_libs/onnxruntime-win-x64-1.23.2/LICENSE rename to prebuilt_libs/onnxruntime-win-x64-1.18.0/LICENSE diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/Privacy.md b/prebuilt_libs/onnxruntime-win-x64-1.18.0/Privacy.md similarity index 100% rename from prebuilt_libs/onnxruntime-win-x64-1.23.2/Privacy.md rename to prebuilt_libs/onnxruntime-win-x64-1.18.0/Privacy.md diff --git a/prebuilt_libs/onnxruntime-win-x64-1.18.0/README.md b/prebuilt_libs/onnxruntime-win-x64-1.18.0/README.md new file mode 100644 index 0000000..24c3e19 --- /dev/null +++ b/prebuilt_libs/onnxruntime-win-x64-1.18.0/README.md @@ -0,0 +1,61 @@ +

+ +**ONNX Runtime is a cross-platform inference and training machine-learning accelerator**. + +**ONNX Runtime inference** can enable faster customer experiences and lower costs, supporting models from deep learning frameworks such as PyTorch and TensorFlow/Keras as well as classical machine learning libraries such as scikit-learn, LightGBM, XGBoost, etc. ONNX Runtime is compatible with different hardware, drivers, and operating systems, and provides optimal performance by leveraging hardware accelerators where applicable alongside graph optimizations and transforms. [Learn more →](https://www.onnxruntime.ai/docs/#onnx-runtime-for-inferencing) + +**ONNX Runtime training** can accelerate the model training time on multi-node NVIDIA GPUs for transformer models with a one-line addition for existing PyTorch training scripts. [Learn more →](https://www.onnxruntime.ai/docs/#onnx-runtime-for-training) + +## Get Started & Resources + +* **General Information**: [onnxruntime.ai](https://onnxruntime.ai) + +* **Usage documentation and tutorials**: [onnxruntime.ai/docs](https://onnxruntime.ai/docs) + +* **YouTube video tutorials**: [youtube.com/@ONNXRuntime](https://www.youtube.com/@ONNXRuntime) + +* [**Upcoming Release Roadmap**](https://github.com/microsoft/onnxruntime/wiki/Upcoming-Release-Roadmap) + +* **Companion sample repositories**: + - ONNX Runtime Inferencing: [microsoft/onnxruntime-inference-examples](https://github.com/microsoft/onnxruntime-inference-examples) + - ONNX Runtime Training: [microsoft/onnxruntime-training-examples](https://github.com/microsoft/onnxruntime-training-examples) + +## Builtin Pipeline Status + +|System|Inference|Training| +|---|---|---| +|Windows|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Windows%20CPU%20CI%20Pipeline?label=Windows+CPU)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=9)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Windows%20GPU%20CI%20Pipeline?label=Windows+GPU)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=10)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Windows%20GPU%20TensorRT%20CI%20Pipeline?label=Windows+GPU+TensorRT)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=47)|| +|Linux|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Linux%20CPU%20CI%20Pipeline?label=Linux+CPU)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=11)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Linux%20CPU%20Minimal%20Build%20E2E%20CI%20Pipeline?label=Linux+CPU+Minimal+Build)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=64)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Linux%20GPU%20CI%20Pipeline?label=Linux+GPU)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=12)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Linux%20GPU%20TensorRT%20CI%20Pipeline?label=Linux+GPU+TensorRT)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=45)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Linux%20OpenVINO%20CI%20Pipeline?label=Linux+OpenVINO)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=55)|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/orttraining-linux-ci-pipeline?label=Linux+CPU+Training)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=86)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/orttraining-linux-gpu-ci-pipeline?label=Linux+GPU+Training)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=84)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/orttraining/orttraining-ortmodule-distributed?label=Training+Distributed)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=148)| +|Mac|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/MacOS%20CI%20Pipeline?label=MacOS+CPU)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=13)|| +|Android|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Android%20CI%20Pipeline?label=Android)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=53)|| +|iOS|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/iOS%20CI%20Pipeline?label=iOS)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=134)|| +|Web|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/ONNX%20Runtime%20Web%20CI%20Pipeline?label=Web)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=161)|| +|Other|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/onnxruntime-binary-size-checks-ci-pipeline?repoName=microsoft%2Fonnxruntime&label=Binary+Size+Check)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=187&repoName=microsoft%2Fonnxruntime)|| + +## Third-party Pipeline Status + +|System|Inference|Training| +|---|---|---| +|Linux|[![Build Status](https://github.com/Ascend/onnxruntime/actions/workflows/build-and-test.yaml/badge.svg)](https://github.com/Ascend/onnxruntime/actions/workflows/build-and-test.yaml)|| + +## Data/Telemetry + +Windows distributions of this project may collect usage data and send it to Microsoft to help improve our products and services. See the [privacy statement](docs/Privacy.md) for more details. + +## Contributions and Feedback + +We welcome contributions! Please see the [contribution guidelines](CONTRIBUTING.md). + +For feature requests or bug reports, please file a [GitHub Issue](https://github.com/Microsoft/onnxruntime/issues). + +For general discussion or questions, please use [GitHub Discussions](https://github.com/microsoft/onnxruntime/discussions). + +## Code of Conduct + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) +or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. + +## License + +This project is licensed under the [MIT License](LICENSE). diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/ThirdPartyNotices.txt b/prebuilt_libs/onnxruntime-win-x64-1.18.0/ThirdPartyNotices.txt similarity index 92% rename from prebuilt_libs/onnxruntime-win-x64-1.23.2/ThirdPartyNotices.txt rename to prebuilt_libs/onnxruntime-win-x64-1.18.0/ThirdPartyNotices.txt index 7b2bbdd..8ec770d 100644 --- a/prebuilt_libs/onnxruntime-win-x64-1.23.2/ThirdPartyNotices.txt +++ b/prebuilt_libs/onnxruntime-win-x64-1.18.0/ThirdPartyNotices.txt @@ -2108,6 +2108,261 @@ SOFTWARE. _____ +TVM Open Deep Learning Compiler Stack + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +CONTRIBUTORS + +TVM Contributors +================ +TVM adopts the Apache style model and governs by merit. We believe that it is important to create an inclusive community where everyone can use, +contribute to, and influence the direction of the project. We actively invite contributors who have earned the merit to be part of the development community. + +See the [community structure document](http://docs.tvm.ai/contribute/community.html) for the explanation of community structure and contribution guidelines. + +## Committers +- [Tianqi Chen](https://github.com/tqchen) (PMC) +- [Thierry Moreau](http://homes.cs.washington.edu/~moreau/) +- [Ziheng Jiang](https://github.com/ZihengJiang) +- [Haichen Shen](http://homes.cs.washington.edu/~haichen/) +- [Yizhi Liu](https://github.com/yzhliu) + +## Code Owners +- [Aditya Atluri](https://github.com/adityaatluri) ROCM +- [Leyuan Wang](https://github.com/Laurawly) TOPI +- [Yuwei Hu](https://github.com/Huyuwei) TOPI +- [Zhixun Tan](https://github.com/phisiart) OpenGL/WebGL backend +- [Nick Hynes](https://github.com/nhynes) SGX and secured computing +- [Lianmin Zheng](https://github.com/merrymercy) AutoTVM + +## Reviewers +- [Zhi Chen](https://github.com/zhiics) +- [Xiaoqiang Dan](https://github.com/xqdan) +- [Liangfu Chen](https://github.com/liangfu) +- [Masahiro Masuda](https://github.com/masahi) +- [Kazutaka Morita](https://github.com/kazum) +- [Tatsuya Nishiyama](https://github.com/nishi-t) +- [Pariksheet Pinjari](https://github.com/PariksheetPinjari909) +- [Jared Roesch](https://github.com/jroesch) +- [Siva](https://github.com/srkreddy1238) +- [Siju Samuel](https://github.com/siju-samuel) +- [Alex Weaver](https://github.com/alex-weaver) +- [Yao Wang](https://github.com/kevinthesun) +- [Jian Weng](https://github.com/were) +- [Eddie Yan](https://github.com/eqy) +- [Joshua Z. Zhang](https://github.com/zhreshold) + +## List of Contributors +- [Full List of Contributors](https://github.com/dmlc/tvm/graphs/contributors) + - To contributors: please add your name to the list. +- [Qiao Zhang](https://github.com/zhangqiaorjc) +- [Haolong Zhang](https://github.com/haolongzhangm) +- [Cody Hao Yu](https://github.com/comaniac) +- [Chris Nuernberger](https://github.com/cnuernber) + +_____ + FreeBSD: getopt.c file Copyright (c) 1987, 1993, 1994 @@ -2237,6 +2492,212 @@ DAMAGE. _____ +google/nsync + +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +_____ + google/re2 Copyright (c) 2009 The RE2 Authors. All rights reserved. @@ -4359,7 +4820,7 @@ SOFTWARE. ---------------------------------------------------------------------------- -This is the MIT/Expat License. For more information see: +This is the MIT/Expat Licence. For more information see: 1. http://www.opensource.org/licenses/mit-license.php @@ -6045,112 +6506,3 @@ https://github.com/intel/neural-speed terms, and open source software license terms. These separate license terms govern your use of the third party programs as set forth in the "THIRD-PARTY-PROGRAMS" file. - -_____ - -dawn - -https://dawn.googlesource.com/dawn - - BSD 3-Clause License - - Copyright 2017-2023 The Dawn & Tint Authors - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - 3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -_____ - -KleidiAI - -https://gitlab.arm.com/kleidi/kleidiai - -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. - -"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. - -2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: - - (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. - - You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. - -5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. - -6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -Copyright 2024-2025 Arm Limited and/or its affiliates - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/prebuilt_libs/onnxruntime-win-x64-1.18.0/VERSION_NUMBER b/prebuilt_libs/onnxruntime-win-x64-1.18.0/VERSION_NUMBER new file mode 100644 index 0000000..84cc529 --- /dev/null +++ b/prebuilt_libs/onnxruntime-win-x64-1.18.0/VERSION_NUMBER @@ -0,0 +1 @@ +1.18.0 diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/cpu_provider_factory.h b/prebuilt_libs/onnxruntime-win-x64-1.18.0/include/cpu_provider_factory.h similarity index 100% rename from prebuilt_libs/onnxruntime-win-x64-1.23.2/include/cpu_provider_factory.h rename to prebuilt_libs/onnxruntime-win-x64-1.18.0/include/cpu_provider_factory.h diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_c_api.h b/prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_c_api.h similarity index 62% rename from prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_c_api.h rename to prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_c_api.h index 8561de9..52f496c 100644 --- a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_c_api.h +++ b/prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_c_api.h @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. // See docs\c_cxx\README.md on generating the Doxygen documentation from this file @@ -38,7 +38,7 @@ * * This value is used by some API functions to behave as this version of the header expects. */ -#define ORT_API_VERSION 23 +#define ORT_API_VERSION 18 #ifdef __cplusplus extern "C" { @@ -46,15 +46,14 @@ extern "C" { //! @} // SAL2 Definitions -#ifndef _MSC_VER +#ifndef _WIN32 #define _In_ #define _In_z_ #define _In_opt_ #define _In_opt_z_ #define _Out_ -#define _Out_opt_ #define _Outptr_ -#define _Outptr_opt_ +#define _Out_opt_ #define _Inout_ #define _Inout_opt_ #define _Frees_ptr_opt_ @@ -62,12 +61,9 @@ extern "C" { #define _Ret_notnull_ #define _Check_return_ #define _Outptr_result_maybenull_ -#define _Outptr_result_maybenull_z_ #define _In_reads_(X) -#define _In_reads_opt_ #define _Inout_updates_(X) #define _Out_writes_(X) -#define _Out_writes_opt_(X) #define _Inout_updates_all_(X) #define _Out_writes_bytes_all_(X) #define _Out_writes_all_(X) @@ -146,9 +142,6 @@ extern "C" { // __VA_ARGS__ on Windows and Linux are different #define ORT_API(RETURN_TYPE, NAME, ...) RETURN_TYPE ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION -#define ORT_API_T(RETURN_TYPE, NAME, ...) \ - RETURN_TYPE(ORT_API_CALL* NAME)(__VA_ARGS__) NO_EXCEPTION - #define ORT_API_STATUS(NAME, ...) \ _Success_(return == 0) _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) \ NO_EXCEPTION ORT_MUST_USE_RESULT @@ -203,10 +196,7 @@ typedef enum ONNXTensorElementDataType { ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FN, // Non-IEEE floating-point format based on IEEE754 single-precision ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FNUZ, // Non-IEEE floating-point format based on IEEE754 single-precision ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2, // Non-IEEE floating-point format based on IEEE754 single-precision - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2FNUZ, // Non-IEEE floating-point format based on IEEE754 single-precision - // Int4 types were introduced in ONNX 1.16. See https://onnx.ai/onnx/technical/int4.html - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT4, // maps to a pair of packed uint4 values (size == 1 byte) - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT4 // maps to a pair of packed int4 values (size == 1 byte) + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2FNUZ // Non-IEEE floating-point format based on IEEE754 single-precision } ONNXTensorElementDataType; // Synced with onnx TypeProto oneof @@ -262,9 +252,6 @@ typedef enum OrtErrorCode { ORT_NOT_IMPLEMENTED, ORT_INVALID_GRAPH, ORT_EP_FAIL, - ORT_MODEL_LOAD_CANCELED, - ORT_MODEL_REQUIRES_COMPILATION, - ORT_NOT_FOUND, } OrtErrorCode; typedef enum OrtOpAttrType { @@ -275,8 +262,6 @@ typedef enum OrtOpAttrType { ORT_OP_ATTR_FLOATS, ORT_OP_ATTR_STRING, ORT_OP_ATTR_STRINGS, - ORT_OP_ATTR_GRAPH, - ORT_OP_ATTR_TENSOR, } OrtOpAttrType; //! @} @@ -309,7 +294,6 @@ ORT_RUNTIME_CLASS(ThreadingOptions); ORT_RUNTIME_CLASS(ArenaCfg); ORT_RUNTIME_CLASS(PrepackedWeightsContainer); ORT_RUNTIME_CLASS(TensorRTProviderOptionsV2); -ORT_RUNTIME_CLASS(NvTensorRtRtxProviderOptions); ORT_RUNTIME_CLASS(CUDAProviderOptionsV2); ORT_RUNTIME_CLASS(CANNProviderOptions); ORT_RUNTIME_CLASS(DnnlProviderOptions); @@ -317,19 +301,8 @@ ORT_RUNTIME_CLASS(Op); ORT_RUNTIME_CLASS(OpAttr); ORT_RUNTIME_CLASS(Logger); ORT_RUNTIME_CLASS(ShapeInferContext); -ORT_RUNTIME_CLASS(LoraAdapter); -ORT_RUNTIME_CLASS(ValueInfo); -ORT_RUNTIME_CLASS(Node); -ORT_RUNTIME_CLASS(Graph); -ORT_RUNTIME_CLASS(Model); -ORT_RUNTIME_CLASS(ModelCompilationOptions); -ORT_RUNTIME_CLASS(HardwareDevice); -ORT_RUNTIME_CLASS(EpDevice); -ORT_RUNTIME_CLASS(KeyValuePairs); -ORT_RUNTIME_CLASS(SyncStream); // Opaque class to create an onnxruntime::Stream. -ORT_RUNTIME_CLASS(ExternalInitializerInfo); -#ifdef _MSC_VER +#ifdef _WIN32 typedef _Return_type_success_(return == 0) OrtStatus* OrtStatusPtr; #else typedef OrtStatus* OrtStatusPtr; @@ -342,71 +315,16 @@ typedef OrtStatus* OrtStatusPtr; * When an allocator is passed to any function, be sure that the allocator object is not destroyed until the last allocated object using it is freed. */ typedef struct OrtAllocator { - uint32_t version; ///< Must be initialized to ORT_API_VERSION - - /// Returns a pointer to an allocated block of `size` bytes - void*(ORT_API_CALL* Alloc)(struct OrtAllocator* this_, size_t size); - - /// Free a block of memory previously allocated with OrtAllocator::Alloc - void(ORT_API_CALL* Free)(struct OrtAllocator* this_, void* p); - - /// Return a pointer to an ::OrtMemoryInfo that describes this allocator - const struct OrtMemoryInfo*(ORT_API_CALL* Info)(const struct OrtAllocator* this_); + uint32_t version; ///< Must be initialized to ORT_API_VERSION + void*(ORT_API_CALL* Alloc)(struct OrtAllocator* this_, size_t size); ///< Returns a pointer to an allocated block of `size` bytes + void(ORT_API_CALL* Free)(struct OrtAllocator* this_, void* p); ///< Free a block of memory previously allocated with OrtAllocator::Alloc + const struct OrtMemoryInfo*(ORT_API_CALL* Info)(const struct OrtAllocator* this_); ///< Return a pointer to an ::OrtMemoryInfo that describes this allocator /** * @brief Optional allocation function to use for memory allocations made during session initialization. * Use this function if you want to separate allocations made by ORT during Run() calls from - * those made during session initialization. This allows for separate memory management strategies for these - * allocations. - * - * \return pointer to an allocated block of `size` bytes. nullptr if size was 0 or allocation failed. - * - * \since 1.18 + * those made during session initialization. This allows for separate memory management strategies for these allocations. */ - void*(ORT_API_CALL* Reserve)(struct OrtAllocator* this_, size_t size); - - /** - * @brief Function used to get the statistics of the allocator. - * - * Return a pointer to the OrtKeyValuePairs structure that contains the statistics of the allocator. - * The user should call OrtApi::ReleaseKeyValuePairs when done. - * - * Current known keys are: - * - Limit: Bytes limit of the allocator. -1 if no limit is set. - * - InUse: Number of bytes in use. - * - TotalAllocated: The total number of allocated bytes by the allocator. - * - MaxInUse: The maximum bytes in use. - * - NumAllocs: Number of allocations. - * - NumReserves: Number of reserves. (Number of calls to Reserve() in arena-based allocators) - * - NumArenaExtensions: Number of arena extensions (Relevant only for arena based allocators) - * - NumArenaShrinkages: Number of arena shrinkages (Relevant only for arena based allocators) - * - MaxAllocSize: The max single allocation seen. - * - * The allocator is free to add other entries as appropriate. - * - * \note Implementation of this function is optional and GetStats may be set to a nullptr. - * If the OrtAllocator is wrapping an internal ORT allocator that does not implement GetStats - * the returned OrtKeyValuePairs instance will be empty. - * - * \since 1.23 - */ - ORT_API2_STATUS(GetStats, _In_ const struct OrtAllocator* this_, _Outptr_ OrtKeyValuePairs** out); - - /** \brief Allocate using a stream. - * - * If the allocator is stream aware this performs allocation using a stream. - * - * Alloc will be used if this is nullptr. - * - * \param[in] this_ OrtAllocator instance - * \param[in] size Size of the allocation in bytes. nullptr if size was 0 or allocation failed. - * \param[in] stream The stream to allocate on. - * - * \return pointer to an allocated block of `size` bytes - * - * \note Implementation of this function is optional and AllocOnStream may be set to a nullptr. - * \since 1.23 - */ - void*(ORT_API_CALL* AllocOnStream)(struct OrtAllocator* this_, size_t size, OrtSyncStream* stream); + void*(ORT_API_CALL* Reserve)(struct OrtAllocator* this_, size_t size); ///< Returns a pointer to an allocated block of `size` bytes } OrtAllocator; typedef void(ORT_API_CALL* OrtLoggingFunction)( @@ -422,7 +340,6 @@ typedef enum GraphOptimizationLevel { ORT_DISABLE_ALL = 0, ORT_ENABLE_BASIC = 1, ORT_ENABLE_EXTENDED = 2, - ORT_ENABLE_LAYOUT = 3, ORT_ENABLE_ALL = 99 } GraphOptimizationLevel; @@ -454,136 +371,27 @@ typedef struct OrtCustomOp OrtCustomOp; typedef enum OrtAllocatorType { OrtInvalidAllocator = -1, OrtDeviceAllocator = 0, - OrtArenaAllocator = 1, - OrtReadOnlyAllocator = 2, + OrtArenaAllocator = 1 } OrtAllocatorType; /** \brief Memory types for allocated memory, execution provider specific types should be extended in each provider. */ // Whenever this struct is updated, please also update the MakeKey function in onnxruntime / core / framework / execution_provider.cc typedef enum OrtMemType { - /// Any CPU memory used by non-CPU execution provider - OrtMemTypeCPUInput = -2, - /// CPU accessible memory outputted by non-CPU execution provider, i.e. HOST_ACCESSIBLE - OrtMemTypeCPUOutput = -1, - /// CPU accessible memory allocated by non-CPU execution provider, i.e. HOST_ACCESSIBLE - OrtMemTypeCPU = OrtMemTypeCPUOutput, - /// The default allocator for execution provider - OrtMemTypeDefault = 0, + OrtMemTypeCPUInput = -2, ///< Any CPU memory used by non-CPU execution provider + OrtMemTypeCPUOutput = -1, ///< CPU accessible memory outputted by non-CPU execution provider, i.e. CUDA_PINNED + OrtMemTypeCPU = OrtMemTypeCPUOutput, ///< Temporary CPU accessible memory allocated by non-CPU execution provider, i.e. CUDA_PINNED + OrtMemTypeDefault = 0, ///< The default allocator for execution provider } OrtMemType; -/** \brief This matches OrtDevice::MemoryType values */ -typedef enum OrtDeviceMemoryType { - OrtDeviceMemoryType_DEFAULT = 0, ///< Device memory - OrtDeviceMemoryType_HOST_ACCESSIBLE = 5, ///< Shared/pinned memory for transferring between CPU and the device -} OrtDeviceMemoryType; - /** \brief This mimics OrtDevice type constants so they can be returned in the API */ typedef enum OrtMemoryInfoDeviceType { OrtMemoryInfoDeviceType_CPU = 0, OrtMemoryInfoDeviceType_GPU = 1, - OrtMemoryInfoDeviceType_FPGA = 2, - OrtMemoryInfoDeviceType_NPU = 3, + OrtMemoryInfoDeviceType_FPGA = 2 } OrtMemoryInfoDeviceType; -typedef enum OrtHardwareDeviceType { - OrtHardwareDeviceType_CPU, - OrtHardwareDeviceType_GPU, - OrtHardwareDeviceType_NPU -} OrtHardwareDeviceType; - -/** \brief These are the default EP selection policies used by ORT when doing automatic EP selection. - */ -typedef enum OrtExecutionProviderDevicePolicy { - OrtExecutionProviderDevicePolicy_DEFAULT, - OrtExecutionProviderDevicePolicy_PREFER_CPU, - OrtExecutionProviderDevicePolicy_PREFER_NPU, - OrtExecutionProviderDevicePolicy_PREFER_GPU, - OrtExecutionProviderDevicePolicy_MAX_PERFORMANCE, - OrtExecutionProviderDevicePolicy_MAX_EFFICIENCY, - OrtExecutionProviderDevicePolicy_MIN_OVERALL_POWER, -} OrtExecutionProviderDevicePolicy; - -/** \brief Delegate to allow providing custom OrtEpDevice selection logic - * - * This delegate is called by the EP selection code to allow the user to provide custom device selection logic. - * The user can use this to select OrtEpDevice instances from the list of available devices. - * - * \param ep_devices The list of available devices. - * \param num_devices The number of available devices. - * \param model_metadata The model metadata. - * \param runtime_metadata The runtime metadata. May be nullptr. - * \param selected Pre-allocated array to populate with selected OrtEpDevice pointers from ep_devices. - * \param max_selected The maximum number of devices that can be selected in the pre-allocated array. - Currently the maximum is 8. - * \param num_selected The number of selected devices. - * \param state Opaque pointer. Required to use the delegate from other languages like C# and python. - * - * \return OrtStatus* Selection status. Return nullptr on success. - * Use CreateStatus to provide error info. Use ORT_FAIL as the error code. - * ORT will release the OrtStatus* if not null. - */ -typedef OrtStatus*(ORT_API_CALL* EpSelectionDelegate)(_In_ const OrtEpDevice** ep_devices, - _In_ size_t num_devices, - _In_ const OrtKeyValuePairs* model_metadata, - _In_opt_ const OrtKeyValuePairs* runtime_metadata, - _Inout_ const OrtEpDevice** selected, - _In_ size_t max_selected, - _Out_ size_t* num_selected, - _In_ void* state); - -/** \brief Function called by ORT to write a buffer to a custom destination (e.g., file, stream, etc.). - * - * \param state Opaque pointer holding the user's state. - * \param buffer The buffer to write. - * \param buffer_num_bytes The size of the buffer in bytes. - * - * \return OrtStatus* Write status. Return nullptr on success. - * Use CreateStatus to provide error info. Use ORT_FAIL as the error code. - * ORT will release the OrtStatus* if not null. - */ -typedef OrtStatus*(ORT_API_CALL* OrtWriteBufferFunc)(_In_ void* state, - _In_ const void* buffer, - _In_ size_t buffer_num_bytes); - -/** \brief Function called by ORT to allow user to specify how an initializer should be saved, that is, either - * written to an external file or stored within the model. ORT calls this function for every initializer when - * generating a model. - * - * If the function implementation sets the `new_external_info` output parameter to NULL, ORT stores the initializer data - * within the generated model. - * - * Otherwise, if the function implementation sets `new_external_info` to a valid OrtExternalInitializerInfo instance, - * ORT assumes that this function stores the initializer data in a file. In this case, ORT configures the model's - * initializer to point to the location specified by the `new_external_info` output parameter. - * - * \param[in] state Opaque pointer holding the user's state. - * \param[in] initializer_name The initializer's name as a null-terminated string. - * \param[in] initializer_value OrtValue containing the initializer's data, type, and shape. - * \param[in] external_info If the initializer is originally stored in an external file, `external_info` contains - * the file path, file offset, and the data's byte size within the file. Otherwise, - * `external_info` is NULL if the initializer is not originally stored in a file. - * \param[out] new_external_info Output parameter set to a new OrtExternalInitializerInfo instance indicating the - * location where the function implementation stored the initializer data. - * The function implementation must use `OrtApi::CreateExternalInitializerInfo()` to - * create the instance. - * If the function implementation sets `new_external_info` to NULL, - * ORT stores the initializers within the model. - * - * \note ORT takes ownership of the `new_external_info` output parameter. - * - * \return OrtStatus* Write status. Return nullptr on success. - * Use CreateStatus to provide error info. Use ORT_FAIL as the error code. - * ORT will release the OrtStatus* if not null. - */ -typedef OrtStatus*(ORT_API_CALL* OrtGetInitializerLocationFunc)( - _In_ void* state, - _In_ const char* initializer_name, - _In_ const OrtValue* initializer_value, - _In_opt_ const OrtExternalInitializerInfo* external_info, - _Outptr_result_maybenull_ OrtExternalInitializerInfo** new_external_info); - /** \brief Algorithm to use for cuDNN Convolution Op */ typedef enum OrtCudnnConvAlgoSearch { @@ -662,13 +470,13 @@ typedef struct OrtCUDAProviderOptions { /** \brief Enable TunableOp for using. * Set it to 1/0 to enable/disable TunableOp. Otherwise, it is disabled by default. - * This option can be overridden by environment variable ORT_CUDA_TUNABLE_OP_ENABLE. + * This option can be overriden by environment variable ORT_CUDA_TUNABLE_OP_ENABLE. */ int tunable_op_enable; /** \brief Enable TunableOp for tuning. * Set it to 1/0 to enable/disable TunableOp tuning. Otherwise, it is disabled by default. - * This option can be overridden by environment variable ORT_CUDA_TUNABLE_OP_TUNING_ENABLE. + * This option can be overriden by environment variable ORT_CUDA_TUNABLE_OP_TUNING_ENABLE. */ int tunable_op_tuning_enable; @@ -705,7 +513,7 @@ typedef struct OrtROCMProviderOptions { */ int device_id; - /** \brief ROCM MIOpen Convolution algorithm exhaustive search option. + /** \brief ROCM MIOpen Convolution algorithm exaustive search option. * Defaults to 0 (false). */ int miopen_conv_exhaustive_search; @@ -751,13 +559,13 @@ typedef struct OrtROCMProviderOptions { /** \brief Enable TunableOp for using. * Set it to 1/0 to enable/disable TunableOp. Otherwise, it is disabled by default. - * This option can be overridden by environment variable ORT_ROCM_TUNABLE_OP_ENABLE. + * This option can be overriden by environment variable ORT_ROCM_TUNABLE_OP_ENABLE. */ int tunable_op_enable; /** \brief Enable TunableOp for tuning. * Set it to 1/0 to enable/disable TunableOp tuning. Otherwise, it is disabled by default. - * This option can be overridden by environment variable ORT_ROCM_TUNABLE_OP_TUNING_ENABLE. + * This option can be overriden by environment variable ORT_ROCM_TUNABLE_OP_TUNING_ENABLE. */ int tunable_op_tuning_enable; @@ -803,41 +611,14 @@ typedef struct OrtTensorRTProviderOptions { typedef struct OrtMIGraphXProviderOptions { int device_id; // hip device id. int migraphx_fp16_enable; // MIGraphX FP16 precision. Default 0 = false, nonzero = true - int migraphx_fp8_enable; // MIGraphX FP8 precision. Default 0 = false, nonzero = true int migraphx_int8_enable; // MIGraphX INT8 precision. Default 0 = false, nonzero = true - int migraphx_use_native_calibration_table; // MIGraphx INT8 cal table. Default 0 = false, nonzero = true + int migraphx_use_native_calibration_table; // MIGraphx INT8 cal table. Default 0 = false, noznero = true const char* migraphx_int8_calibration_table_name; // MIGraphx INT8 calibration table name - int migraphx_save_compiled_model; // migraphx save compiled model. Default 0 = false, nonzero = true - const char* migraphx_save_model_path; // migraphx model path name - int migraphx_load_compiled_model; // migraphx int8 cal table. Default 0 = false, nonzero = true - const char* migraphx_load_model_path; // migraphx model path name - bool migraphx_exhaustive_tune; // MIGraphX tuned compile. Default = false, nonzero = true - - /** \brief MIGraphX memory limit (To use all possible memory pass in maximum size_t) - * Defaults to SIZE_MAX. - * \note If a ::OrtArenaCfg has been applied, it will override this field - */ - size_t migraphx_mem_limit; - - /** \brief Strategy used to grow the memory arena - * 0 = kNextPowerOfTwo
- * 1 = kSameAsRequested
- * Defaults to 0. - * \note If a ::OrtArenaCfg has been applied, it will override this field - */ - int migraphx_arena_extend_strategy; - - // This is the legacy struct and don't add new fields here. } OrtMIGraphXProviderOptions; /** \brief OpenVINO Provider Options - * \brief This Struct is frozen since ORT 1.13.0. Its maintained part of Legacy API for compatibility. - * \brief For latest OpenVINO Provider Options update to the ProviderOptions map. - * \brief Latest OpenVINO Provider Options are listed in the - * \htmlonly - * onnxruntime document. - * \endhtmlonly - * \see OrtApi::SessionOptionsAppendExecutionProvider() + * + * \see OrtApi::SessionOptionsAppendExecutionProvider_OpenVINO */ typedef struct OrtOpenVINOProviderOptions { #ifdef __cplusplus @@ -870,15 +651,6 @@ typedef struct OrtApi OrtApi; struct OrtTrainingApi; typedef struct OrtTrainingApi OrtTrainingApi; -struct OrtModelEditorApi; -typedef struct OrtModelEditorApi OrtModelEditorApi; - -struct OrtCompileApi; -typedef struct OrtCompileApi OrtCompileApi; - -struct OrtEpApi; -typedef struct OrtEpApi OrtEpApi; - /** \brief The helper interface to get the right version of OrtApi * * Get a pointer to this structure through ::OrtGetApiBase @@ -953,16 +725,6 @@ typedef void (*RunAsyncCallbackFn)(void* user_data, OrtValue** outputs, size_t n * * \nosubgrouping */ -/* - * Public enum for compiled model compatibility across EPs. - */ -typedef enum OrtCompiledModelCompatibility { - OrtCompiledModelCompatibility_EP_NOT_APPLICABLE = 0, - OrtCompiledModelCompatibility_EP_SUPPORTED_OPTIMAL, - OrtCompiledModelCompatibility_EP_SUPPORTED_PREFER_RECOMPILATION, - OrtCompiledModelCompatibility_EP_UNSUPPORTED, -} OrtCompiledModelCompatibility; - struct OrtApi { /// \name OrtStatus /// @{ @@ -1071,8 +833,7 @@ struct OrtApi { * * \snippet{doc} snippets.dox OrtStatus Return Value */ - ORT_API2_STATUS(CreateSessionFromArray, _In_ const OrtEnv* env, - _In_ const void* model_data, size_t model_data_length, + ORT_API2_STATUS(CreateSessionFromArray, _In_ const OrtEnv* env, _In_ const void* model_data, size_t model_data_length, _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out); /** \brief Run the model in an ::OrtSession @@ -1565,8 +1326,6 @@ struct OrtApi { * Create a tensor with user's buffer. You can fill the buffer either before calling this function or after. * p_data is owned by caller. ReleaseValue won't release p_data. * - * If you wish to transfer ownership of p_data to ORT use CreateTensorWithDataAndDeleterAsOrtValue. - * * \param[in] info Memory description of where the p_data buffer resides (CPU vs GPU etc). * \param[in] p_data Pointer to the data buffer. * \param[in] p_data_len The number of bytes in the data buffer. @@ -1845,7 +1604,7 @@ struct OrtApi { */ ORT_API2_STATUS(MemoryInfoGetName, _In_ const OrtMemoryInfo* ptr, _Out_ const char** out); - /** \brief Get the device id from ::OrtMemoryInfo + /** \brief Get the id from ::OrtMemoryInfo */ ORT_API2_STATUS(MemoryInfoGetId, _In_ const OrtMemoryInfo* ptr, _Out_ int* out); @@ -2088,8 +1847,8 @@ struct OrtApi { * and not present, the function returns success and out is set to nullptr. * * \param[in] context ::OrtKernelContext instance - * \param[in] index See KernelContext_GetInputCount for boundaries check. - * \param[out] out OrtValue if the input is present otherwise is set nullptr + * \param[in] input index. See KernelContext_GetInputCount for boundaries check. + * \param[in, out] returns a ptr to OrtValue if the input is present * * \snippet{doc} snippets.dox OrtStatus Return Value */ @@ -2102,10 +1861,8 @@ struct OrtApi { * and not present, the function returns success and out is set to nullptr. * * \param[in] context ::OrtKernelContext instance - * \param[in] index See KernelContext_GetOutputCount for boundaries check. - * \param[in] dim_values output dimensions - * \param[in] dim_count number of dimensions - * \param[out] out a ptr to OrtValue to output otherwise set to nullptr + * \param[in] output index. See KernelContext_GetOutputCount for boundaries check. + * \param[in, out] returns a ptr to OrtValue if the output is present * * \snippet{doc} snippets.dox OrtStatus Return Value */ @@ -2224,8 +1981,7 @@ struct OrtApi { /** \brief Get the value type from an ::OrtMapTypeInfo * * \param[in] map_type_info - * \param[out] type_info A copy of the OrtTypeInfo for the map value type. - * The user must free this value with ReleaseTypeInfo. + * \param[out] type_info * * \snippet{doc} snippets.dox OrtStatus Return Value */ @@ -2240,8 +1996,7 @@ struct OrtApi { * This is used by WinML to support model reflection APIs. * * \param[in] sequence_type_info - * \param[out] type_info A copy of the OrtTypeInfo for the sequence element type. - * The user must free this value with ReleaseTypeInfo. + * \param[out] type_info * * \snippet{doc} snippets.dox OrtStatus Return Value */ @@ -2522,8 +2277,6 @@ struct OrtApi { /// @{ /** \brief Create an allocator for an ::OrtSession following an ::OrtMemoryInfo - * - * The allocator wraps the internal allocator from the OrtSession and becomes invalid when the session does. * * \param[in] session * \param[in] mem_info valid ::OrtMemoryInfo instance @@ -3035,8 +2788,8 @@ struct OrtApi { * crossing which the current chunk is chunked into 2. * "initial_growth_chunk_size_bytes": (Possible) Size of the second allocation in the arena. * Only relevant if arena strategy is `kNextPowerOfTwo`. Use -1 to allow ORT to choose the default. - * "max_power_of_two_extend_bytes": The maximum extend size if arena strategy is `kNextPowerOfTwo`. - * It is not an allocation limit, it is only a limit for extension when requested byte is less than the limit. + * "max_power_of_two_extend_bytes": The maximum enxtend size if arena strategy is `kNextPowerOfTwo`. + * It is not an allocation limit, it is only a limit for extention when requested byte is less than the limit. * When requested bytes is more than the limit, allocator will still return as requested. * Use -1 to allow ORT to choose the default 1GB for max_power_of_two_extend_bytes. * Ultimately, the allocation size is determined by the allocation memory request. @@ -3118,8 +2871,7 @@ struct OrtApi { * \snippet{doc} snippets.dox OrtStatus Return Value */ ORT_API2_STATUS(CreateSessionWithPrepackedWeightsContainer, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path, - _In_ const OrtSessionOptions* options, - _Inout_ OrtPrepackedWeightsContainer* prepacked_weights_container, + _In_ const OrtSessionOptions* options, _Inout_ OrtPrepackedWeightsContainer* prepacked_weights_container, _Outptr_ OrtSession** out); /** \brief Create session from memory with prepacked weights container @@ -3142,8 +2894,7 @@ struct OrtApi { */ ORT_API2_STATUS(CreateSessionFromArrayWithPrepackedWeightsContainer, _In_ const OrtEnv* env, _In_ const void* model_data, size_t model_data_length, - _In_ const OrtSessionOptions* options, - _Inout_ OrtPrepackedWeightsContainer* prepacked_weights_container, + _In_ const OrtSessionOptions* options, _Inout_ OrtPrepackedWeightsContainer* prepacked_weights_container, _Outptr_ OrtSession** out); /// @} @@ -3769,8 +3520,7 @@ struct OrtApi { * * \param[in] name Name of the attribute * \param[in] data Data content of the attribute - * \param[in] len Number of bytes stored in data for ORT_OP_ATTR_STRING. - Number of elements if data represents an array (e.g., ORT_OP_ATTR_INTS). Otherwise, set to 1. + * \param[in] len Number of bytes stored in data * \param[in] type Data type * \param[out] op_attr Attribute that has been created, which must be released by OrtApi::ReleaseOpAttr * @@ -3797,9 +3547,9 @@ struct OrtApi { * \param[in] op_name Operator name * \param[in] domain Operator domain * \param[in] version Operator opset version - * \param[in] type_constraint_names Name of the type constraints, such as "T" or "T1" - * \param[in] type_constraint_values Type of each constraints - * \param[in] type_constraint_count Number of constraints + * \param[in] type_constraint_names Name of the type contraints, such as "T" or "T1" + * \param[in] type_constraint_values Type of each contraints + * \param[in] type_constraint_count Number of contraints * \param[in] attr_values Attributes used to initialize the operator * \param[in] attr_count Number of the attributes * \param[in] input_count Number of inputs @@ -3857,110 +3607,58 @@ struct OrtApi { * \param[in] provider_options_values - values to configure the provider options * \param[in] num_keys - number of keys passed in * - * Currently supported provider names: - * QNNExecutionProvider (or QNN) - * OpenVINOExecutionProvider (or OpenVINO) - * XnnpackExecutionProvider (or XNNPACK) - * WebNNExecutionProvider (or WEBNN) - * WebGpuExecutionProvider (or WebGPU) - * AzureExecutionProvider (or AZURE) - * JsExecutionProvider (or JS) - * VitisAIExecutionProvider (or VitisAI) - * CoreMLExecutionProvider (or CoreML) + * Currently supported providers: + * QNN + * SNPE + * XNNPACK * * Note: If an execution provider has a dedicated SessionOptionsAppendExecutionProvider_ function * that should be used to add it. * * QNN supported keys: - * "backend_type": Type of QNN backend. Specifies a backend path that is the associated QNN backend library file - * name. E.g., given backend type "htp", on Windows, the backend path would be "QnnHtp.dll", and on other - * platforms, it would be "libQnnHtp.so". Mutually exclusive with "backend_path". - * Available options: - * -# "cpu" - * -# "gpu" - * -# "htp": Default. - * -# "saver" - * -# "ir" - * "backend_path": File path to QNN backend library. Mutually exclusive with "backend_type". - * "profiling_level": QNN profiling level. - * Available options: - * -# "off": Default. - * -# "basic" - * -# "detailed" + * "backend_path": file path to QNN backend library. + * "profiling_level": QNN profiling level, options: "off", "basic", "detailed". Default to off. * "profiling_file_path": QNN profiling file path if ETW not enabled. * "rpc_control_latency": QNN RPC control latency. * "vtcm_mb": QNN VTCM size in MB. default to 0(not set). - * "htp_performance_mode": QNN performance mode. - * Available options: - * -# "burst" - * -# "balanced" - * -# "default": Default. - * -# "high_performance" - * -# "high_power_saver" - * -# "low_balanced" - * -# "extreme_power_saver" - * -# "low_power_saver" - * -# "power_saver" - * -# "sustained_high_performance" - * "dump_qnn_ir_dlc": Use the QnnIr backend library to write .dlc files for each subgraph dispatched to QNN. When - * enabled, inference results will be incorrect. Use only for debugging. - * -# "0": Default: disabled - * -# "1": enabled - * "dump_qnn_ir_dlc_dir": Set the directory into which QnnIr will be configured to write QNN graphs as .dlc files. - * Default is current working directory. - * "qnn_ir_backend_path": File path to the QnnIr backend library. If "dump_qnn_ir_dlc" is enabled, use this path - * instead of looking for the Ir backend in the standard location. + * "htp_performance_mode": QNN performance mode, options: "burst", "balanced", "default", "high_performance", + * "high_power_saver", "low_balanced", "extreme_power_saver", "low_power_saver", "power_saver", "sustained_high_performance". Default to "default". * "qnn_saver_path": File path to the QNN Saver backend library. If specified, QNN Saver will be enabled and will - * dump QNN API calls to disk for replay/debugging. QNN Saver produces incorrect model inference results and - * may alter model/EP partitioning. Use only for debugging. - * "qnn_context_priority": QNN context priority. - * Available options: - * -# "low" - * -# "normal": Default. - * -# "normal_high" - * -# "high" - * "htp_graph_finalization_optimization_mode": Set the optimization mode for graph finalization on the HTP backend. - * Available options: - * -# "0": Default. - * -# "1": Faster preparation time, less optimal graph. - * -# "2": Longer preparation time, more optimal graph. - * -# "3": Longest preparation time, most likely even more optimal graph. See QNN SDK documentation for specific - * details. - * "soc_model": The SoC model number. Refer to the QNN SDK documentation for valid values. - * Defaults to "0" (unknown). - * "htp_arch": The minimum HTP architecture the driver will use to select compatible QNN operators. - * Available options: - * -# "0": Default (none). - * -# "68" - * -# "69" - * -# "73" - * -# "75" + * dump QNN API calls to disk for replay/debugging. QNN Saver produces incorrect model inference results and + * may alter model/EP partitioning. Use only for debugging. + * "qnn_context_priority": QNN context priority, options: "low", "normal", "normal_high", "high". Default to "normal". + * "htp_graph_finalization_optimization_mode": Set the optimization mode for graph finalization on the HTP backend. Available options: + * - "0": Default. + * - "1": Faster preparation time, less optimal graph. + * - "2": Longer preparation time, more optimal graph. + * - "3": Longest preparation time, most likely even more optimal graph. See QNN SDK documentation for specific details. + * "soc_model": The SoC model number. Refer to the QNN SDK documentation for valid values. Defaults to "0" (unknown). + * "htp_arch": The minimum HTP architecture the driver will use to select compatible QNN operators. Available options: + * - "0": Default (none). + * - "68" + * - "69" + * - "73" + * - "75" * "device_id": The ID of the device to use when setting 'htp_arch'. Defaults to "0" (for single device). - * "enable_htp_fp16_precision": Used for float32 model for HTP backend. - * Enable the float32 model to be inferenced with fp16 precision. Otherwise, it will be fp32 precision. - * -# "0": With fp32 precision. - * -# "1": Default. With fp16 precision. - * "offload_graph_io_quantization": Offload graph input quantization and graph output dequantization to another - * execution provider (typically CPU EP). - * -# "0": Disabled. QNN EP will handle quantization and dequantization of graph I/O. - * -# "1": Enabled. This is the default value. - * "enable_htp_spill_fill_buffer": Enable HTP spill fill buffer setting. The flag is used while generating context - * binary. - * -# "0": Default. Disabled. - * -# "1": Enabled. - * "enable_htp_shared_memory_allocator": Enable the QNN HTP shared memory allocator. Requires libcdsprpc.so/dll to - * be available. - * -# "0": Default. Disabled. - * -# "1": Enabled. - * "dump_json_qnn_graph": Set to "1" to dump QNN graphs generated by QNN EP as JSON files. Each graph partition - * assigned to QNN EP is dumped to a separate file. - * "json_qnn_graph_dir": Directory in which to dump QNN JSON graphs. If not specified, QNN graphs are dumped in the - * program's current working directory. Ignored if "dump_json_qnn_graph" is not set. - * "op_packages": QNN UDO op_package for QNN EP, allowed format: - *   "::[:],::[:]", - *   where op_type is the name of the operation, op_package_path is the path to the op package shared library, - * interface is the symbol name to register the op life cycle functions, and target is the backend type. For more - * details, refer to: https://docs.qualcomm.com/bundle/publicresource/topics/80-63442-50/op_packages.html + "enable_htp_fp16_precision": Only used for float32 model. + Enable the float32 model to be inferenced with fp16 precision. Otherwise, it will be fp32 precision. + - "0": Default. With fp32 precision. + - "1": With fp16 precision. + * + * SNPE supported keys: + * "runtime": SNPE runtime engine, options: "CPU", "CPU_FLOAT32", "GPU", "GPU_FLOAT32_16_HYBRID", "GPU_FLOAT16", + * "DSP", "DSP_FIXED8_TF", "AIP_FIXED_TF", "AIP_FIXED8_TF". + * Mapping to SNPE Runtime_t definition: CPU, CPU_FLOAT32 => zdl::DlSystem::Runtime_t::CPU; + * GPU, GPU_FLOAT32_16_HYBRID => zdl::DlSystem::Runtime_t::GPU; + * GPU_FLOAT16 => zdl::DlSystem::Runtime_t::GPU_FLOAT16; + * DSP, DSP_FIXED8_TF => zdl::DlSystem::Runtime_t::DSP. + * AIP_FIXED_TF, AIP_FIXED8_TF => zdl::DlSystem::Runtime_t::AIP_FIXED_TF. + * "priority": execution priority, options: "low", "normal". + * "buffer_type": ITensor or user buffers, options: "ITENSOR", user buffer with different types - "TF8", "TF16", "UINT8", "FLOAT". + * "ITENSOR" -- default, ITensor which is float only. + * "TF8" -- quantized model required, "FLOAT" -- for both quantized or non-quantized model + * "enable_init_cache": enable SNPE init caching feature, set to 1 to enabled it. Disabled by default. + * If SNPE is not available (due to a non Snpe enabled build or its dependencies not being installed), this function will fail. * * XNNPACK supported keys: * "intra_op_num_threads": number of thread-pool size to use for XNNPACK execution provider. @@ -4071,7 +3769,7 @@ struct OrtApi { /** \brief Release an OrtCANNProviderOptions * - * \param[in] input The pointer of OrtCANNProviderOptions which will been deleted + * \param[in] the pointer of OrtCANNProviderOptions which will been deleted * * \since Version 1.13. */ @@ -4501,7 +4199,7 @@ struct OrtApi { /** \brief Get the logging severity level of the ::OrtLogger. * - * Can be used in a custom operator to get the logging severity level of the ::OrtLogger associated with + * Can be used in a custom operator to get the logging serverity level of the ::OrtLogger associated with * the ::OrtKernelInfo. * * \param[in] logger The ::OrtLogger instance. @@ -4561,8 +4259,8 @@ struct OrtApi { * specific type that is described by the returned ::OrtTypeInfo. * * \param[in] optional_type_info - * \param[out] out A copy of ::OrtTypeInfo for what the optional value could be. - * The user must free this value with ReleaseTypeInfo. + * \param[out] out A pointer to the ::OrtTypeInfo for what the optional value could be. + * it is owned by OrtOptionalTypeInfo instance. * * \snippet{doc} snippets.dox OrtStatus Return Value * @@ -4675,8 +4373,7 @@ struct OrtApi { * \param[in] provider_options_values value of the provider options map * \param[in] num_keys Length of the provider options map */ - ORT_API2_STATUS(CreateAndRegisterAllocatorV2, _Inout_ OrtEnv* env, _In_ const char* provider_type, - _In_ const OrtMemoryInfo* mem_info, _In_ const OrtArenaCfg* arena_cfg, + ORT_API2_STATUS(CreateAndRegisterAllocatorV2, _Inout_ OrtEnv* env, _In_ const char* provider_type, _In_ const OrtMemoryInfo* mem_info, _In_ const OrtArenaCfg* arena_cfg, _In_reads_(num_keys) const char* const* provider_options_keys, _In_reads_(num_keys) const char* const* provider_options_values, _In_ size_t num_keys); /** \brief Run the model asynchronously in a thread owned by intra op thread pool @@ -4761,14 +4458,13 @@ struct OrtApi { * E.g. a cuda stream or a cublas handle * * \param context - Kernel context - * \param resource_version - Version of the resource + * \param resouce_version - Version of the resource * \param resource_id - Type of resource * \param resource - A pointer to returned resource * * \since Version 1.16. */ - ORT_API2_STATUS(KernelContext_GetResource, _In_ const OrtKernelContext* context, _In_ int resource_version, - _In_ int resource_id, _Outptr_ void** resource); + ORT_API2_STATUS(KernelContext_GetResource, _In_ const OrtKernelContext* context, _In_ int resouce_version, _In_ int resource_id, _Outptr_ void** resource); /** \brief Set user logging function * @@ -4823,10 +4519,10 @@ struct OrtApi { ORT_API2_STATUS(ShapeInferContext_GetAttribute, _In_ const OrtShapeInferContext* context, _In_ const char* attr_name, _Outptr_ const OrtOpAttr** attr); /** - * Set type and shape info of an output + * Set type and shape info of an ouput * * \param[in] context - * \param[in] index The index of the output + * \param[in] index The index of the ouput * \param[out] info Type shape info of the output * * \since Version 1.17. @@ -4853,8 +4549,6 @@ struct OrtApi { * \param[in] len Number of bytes allowed to store in data * \param[out] out Number of bytes required to save the data when the call failed, or the real number of bytes saved to data on success * - * \note Does not support reading graph attributes. Refer to Node_GetSubgraphs. - * * \since Version 1.17. */ ORT_API2_STATUS(ReadOpAttr, _In_ const OrtOpAttr* op_attr, _In_ OrtOpAttrType type, _Inout_ void* data, _In_ size_t len, _Out_ size_t* out); @@ -4894,8 +4588,6 @@ struct OrtApi { * \param[in] num_keys * * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.17. */ ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_OpenVINO_V2, _In_ OrtSessionOptions* options, @@ -4913,8 +4605,6 @@ struct OrtApi { * \param[in] num_keys * * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.18. */ ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_VitisAI, _In_ OrtSessionOptions* options, @@ -4922,16 +4612,13 @@ struct OrtApi { _In_reads_(num_keys) const char* const* provider_options_values, _In_ size_t num_keys); - /** \brief Get scratch buffer from the corresponding allocator under the specific OrtMemoryInfo object. + /** \brief Get scratch buffer from the corresponding allocator under the sepcific OrtMemoryInfo object. * NOTE: callers are responsible to release this scratch buffer from the corresponding allocator * \param[in] context OrtKernelContext instance * \param[in] mem_info OrtMemoryInfo instance * \param[in] count_or_bytes How many bytes is this scratch buffer - * \param[out] out A pointer to the scratch buffer - * + * \param[out] out A pointer to the scrach buffer * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.18. */ ORT_API2_STATUS(KernelContext_GetScratchBuffer, _In_ const OrtKernelContext* context, _In_ const OrtMemoryInfo* mem_info, _In_ size_t count_or_bytes, _Outptr_ void** out); @@ -4942,8 +4629,6 @@ struct OrtApi { * \param[out] out A pointer to OrtAllocator * * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.18. */ ORT_API2_STATUS(KernelInfoGetAllocator, _In_ const OrtKernelInfo* info, _In_ OrtMemType mem_type, _Outptr_ OrtAllocator** out); @@ -4956,7 +4641,7 @@ struct OrtApi { * will occur before any of the optimizations take place. The data will be copied into the graph * since TensorProto can't refer to the user provided buffers. * - * \param[in] options + * \param[in] session options * \param[in] external_initializer_file_names Array of null terminated UTF-8 encoded strings of the file names * which holds the external initializers. * \param[in] external_initializer_file_buffer_array Array of pointers to the buffer of the file content. @@ -4965,1619 +4650,12 @@ struct OrtApi { * \param[in] num_external_initializer_files Number of external files * * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.18. */ ORT_API2_STATUS(AddExternalInitializersFromFilesInMemory, _In_ OrtSessionOptions* options, _In_reads_(num_external_initializer_files) const ORTCHAR_T* const* external_initializer_file_names, _In_reads_(num_external_initializer_files) char* const* external_initializer_file_buffer_array, _In_reads_(num_external_initializer_files) const size_t* external_initializer_file_lengths, size_t num_external_initializer_files); - - /** \brief Create an OrtLoraAdapter - * - * The function attempts to locate file specified by adapter_file_path, read it and create an OrtLoraAdapter - * instance. The adapter_file_path should be a valid path to a file that contains a valid Lora Adapter - * format. The function attempts to validate the format at load time. The file will always be memory mapped, unless - * the platform does not support memory mapping, in which case the file will be read into memory. - * - * \param[in] adapter_file_path adapter file path. - * \param[in] allocator optional pointer to a device allocator. If specified - * data is copied to the device at some point before Run() is invoked. If nullptr, data stays on CPU. - * The data would still be copied to device if required by the model at inference time. - * \param[out] out A pointer to a newly created OrtLoraAdapter instance. Must be released with - * OrtApi::ReleaseLoraAdapter. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.20. - */ - ORT_API2_STATUS(CreateLoraAdapter, const ORTCHAR_T* adapter_file_path, _In_ OrtAllocator* allocator, - _Outptr_ OrtLoraAdapter** out); - - /** \brief Create an OrtLoraAdapter - * - * The function copies the bytes from the array and creates an OrtLoraAdapter instance. - * - * - * \param[in] bytes pointer to a valid Lora Adapter format buffer. - * \param[in] num_bytes length of bytes buffer. - * \param[in] allocator optional pointer to a device allocator. If specified - * data is copied to the device at some point before Run() is invoked. If nullptr, data stays on CPU. - * The data would still be copied to device if required by the model at inference time. - * \param[out] out A pointer to a newly created OrtLoraAdapter instance. Must be released with - * OrtApi::ReleaseLoraAdapter. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.20. - */ - ORT_API2_STATUS(CreateLoraAdapterFromArray, _In_ const void* bytes, size_t num_bytes, _In_ OrtAllocator* allocator, - _Outptr_ OrtLoraAdapter** out); - - /** \brief Release an ::OrtLoraAdapter obtained from OrtApi::CreateLoraAdapter - */ - ORT_CLASS_RELEASE(LoraAdapter); - - /** \brief Add the Lora Adapter to the list of active adapters. - * - * The function adds the Lora Adapter to the list of active adapters. The Lora Adapter must be created with - * OrtApi::CreateLoraAdapter or FromArray. The Lora Adapter will be used by the session to run the model. - * The instance of the OrtRunOptions can then be used to customize the Run() calls. - * More than one OrtLoraAdapter can be active at the same time. Lora Parameters that belong to different - * Lora adapters that will be active at the same time must not overlap. - * This setting does not affect RunWithBinding. - * - * \param[in] options OrtRunOptions instance - * \param[in] adapter OrtLoraAdapter instance - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.20. - */ - ORT_API2_STATUS(RunOptionsAddActiveLoraAdapter, _Inout_ OrtRunOptions* options, _In_ const OrtLoraAdapter* adapter); - - /// @} - /// \name OrtEpDynamicOptions - /// @{ - - /** \brief Set DynamicOptions for EPs (Execution Providers) - * - * Valid options can be found in `include\onnxruntime\core\session\onnxruntime_session_options_config_keys.h` - * Look for `kOrtEpDynamicOptions` - * - * \param[in] sess OrtSession - * \param[in] keys Array of null terminated UTF8 encoded strings of EP dynamic option keys - * \param[in] values Array of null terminated UTF8 encoded string of EP dynamic option values - * \param[in] kv_len Number of elements in the keys and values arrays - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.20. - */ - ORT_API2_STATUS(SetEpDynamicOptions, _Inout_ OrtSession* sess, _In_reads_(kv_len) const char* const* keys, - _In_reads_(kv_len) const char* const* values, _In_ size_t kv_len); - - /// @} - - /** \brief Release an OrtValueInfo instance if it was not added to an OrtGraph. - * \since Version 1.22. - */ - ORT_CLASS_RELEASE(ValueInfo); - - /** \brief Release an OrtNode if it was not added to an OrtGraph. - * \since Version 1.22. - */ - ORT_CLASS_RELEASE(Node); - - /** \brief Release an OrtGraph. - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.22. - */ - ORT_CLASS_RELEASE(Graph); - - /** \brief Release an OrtModel. - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.22. - */ - ORT_CLASS_RELEASE(Model); - - /** \brief Get the value name from an OrtValueInfo instance. - * \param[in] value_info The OrtValueInfo instance. - * \param[out] name The name of the OrtValueInfo - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.22. - */ - ORT_API2_STATUS(GetValueInfoName, _In_ const OrtValueInfo* value_info, _Out_ const char** name); - - /** \brief Get the type information from an OrtValueInfo instance. - * \param[in] value_info The OrtValueInfo instance. - * \param[out] type_info The type info of the OrtValueInfo - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.22. - */ - ORT_API2_STATUS(GetValueInfoTypeInfo, _In_ const OrtValueInfo* value_info, _Outptr_ const OrtTypeInfo** type_info); - - /** \brief Get the Model Editor API instance - * - * Get the Model Editor API instance to create a new model or augment an existing model. - * - * \return Model Editor API struct - * - * \since Version 1.22. - */ - const OrtModelEditorApi*(ORT_API_CALL* GetModelEditorApi)(); - - /** \brief Create an OrtValue for a Tensor that uses pre-existing memory. - * - * ORT will take ownership of the memory and free it using the provided deleter when no longer in use. - * - * \param[in] deleter OrtAllocator instance that will be used to free the memory. - * Only the OrtAllocator:Info and OrtAllocator::Release functions are required. - * The OrtMemoryInfo returned by OrtAllocator::Info must match the location of p_data. - * \param[in] p_data Pointer to the memory that will be used by the Tensor. ORT will take ownership of the memory. - * \param[in] p_data_len Length of the memory in bytes. - * \param[in] shape Dimensions of the Tensor. All values should be > 0. - * \param[in] shape_len Number of dimensions in the shape array. - * \param[in] type Data type of the Tensor. - * \param[out] out Newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(CreateTensorWithDataAndDeleterAsOrtValue, _In_ OrtAllocator* deleter, - _In_ void* p_data, size_t p_data_len, - _In_ const int64_t* shape, size_t shape_len, - ONNXTensorElementDataType type, - _Outptr_ OrtValue** out); - - /** \brief sets load cancellation flag to abort session loading process. - * - * \param[in] options instance that was passed to the session at creation time. - * \param[in] cancel setting this to true after model loading process was initiated will - * attempt to cancel the loading process. If cancellation is successful, CreateSession() - * CreateSessionFromArray() or any other session creation API that take session options as an - * argument will return an OrtStatus indicating that session loading was canceled at user request, - * error code ORT_MODEL_LOAD_CANCELED. - * The APIs above would not return any valid Session instance. This is the best case effort and the result - * is not guaranteed. The session may have already been created and initialized - * before the cancellation request was issued. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(SessionOptionsSetLoadCancellationFlag, _Inout_ OrtSessionOptions* options, - _In_ bool cancel); - - /** \brief Get the Compile API instance. - * - * Get the Compile API instance to compile ONNX models. Execution providers that support compilation fuse a subgraph - * into an EPContext node that wraps a provider-specific binary representation of the subgraph. - * For more details about the EPContext design, refer to: - * \htmlonly - * EPContext design document. - * \endhtmlonly - * - * \return Compile API struct instance. - * - * \since Version 1.22. - */ - const OrtCompileApi*(ORT_API_CALL* GetCompileApi)(); - - // - // OrtKeyValuePairs - // - - /** \brief Create an OrtKeyValuePairs instance. - * - * \param[out] out A pointer to a newly created OrtKeyValuePairs instance. - * - * \note Must be released by calling ReleaseKeyValuePairs. - * - * \since Version 1.22. - */ - void(ORT_API_CALL* CreateKeyValuePairs)(_Outptr_ OrtKeyValuePairs** out); - - /** \brief Add a key-value pair to the OrtKeyValuePairs instance. - * - * If a pair with the same key already exists, it is overwritten. - * - * \param[in] kvps OrtKeyValuePairs instance. - * \param[in] key Key to be added. - * \param[in] value Value to be added. - * - * \note The `key` and `value` are copied internally. - * - * \since Version 1.22. - */ - - void(ORT_API_CALL* AddKeyValuePair)(_In_ OrtKeyValuePairs* kvps, _In_ const char* key, _In_ const char* value); - - /** \brief Get the value associated with a key in the OrtKeyValuePairs instance. - * - * \param[in] kvps OrtKeyValuePairs instance. - * \param[in] key Key to be searched. - * - * \return The value associated with the key, or nullptr if the key does not exist. - * - * \since Version 1.22. - */ - const char*(ORT_API_CALL* GetKeyValue)(_In_ const OrtKeyValuePairs* kvps, _In_ const char* key); - - /** \brief Get all the key-value pairs from the OrtKeyValuePairs instance. - * - * \param[in] kvps OrtKeyValuePairs instance. - * \param[out] keys Array of keys from `kvps`. - * \param[out] values Array of values from `kvps`. - * \param[out] num_entries Number of entries in `keys` and `values`. - * - * \since Version 1.22. - */ - void(ORT_API_CALL* GetKeyValuePairs)(_In_ const OrtKeyValuePairs* kvps, - _Outptr_ const char* const** keys, _Outptr_ const char* const** values, - _Out_ size_t* num_entries); - - /** \brief Remove a key-value pair from the OrtKeyValuePairs instance. - * - * \param[in] kvps OrtKeyValuePairs instance. - * \param[in] key Key to be removed. No error if not found. - * - * \since Version 1.22. - */ - void(ORT_API_CALL* RemoveKeyValuePair)(_In_ OrtKeyValuePairs* kvps, _In_ const char* key); - - /** \brief Release an OrtKeyValuePairs instance. - * - * \param[in] input OrtKeyValuePairs instance to be released. - * - * \since Version 1.22. - */ - ORT_CLASS_RELEASE(KeyValuePairs); - - /** \brief Register an execution provider library with ORT. - * - * The library must export 'CreateEpFactories' and 'ReleaseEpFactory' functions. - * See OrtEpApi for more details. - * - * \param[in] env The OrtEnv instance to register the library in. - * \param[in] registration_name The name to register the execution provider library under. - * \param[in] path The path to the execution provider library. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(RegisterExecutionProviderLibrary, _In_ OrtEnv* env, _In_ const char* registration_name, - _In_ const ORTCHAR_T* path); - - /** \brief Unregister an execution provider library with ORT. - * - * ORT will call ReleaseEpFactory for all factories created by the library, and unload the library. - * - * You MUST ensure there are no Session instances using execution providers created by the library - * before calling this function. - * - * \param[in] env The OrtEnv instance to unregister the library from. - * \param[in] registration_name The name the execution provider library was registered under. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(UnregisterExecutionProviderLibrary, _In_ OrtEnv* env, _In_ const char* registration_name); - - /** \brief Get the list of available OrtEpDevice instances. - * - * Each OrtEpDevice instance contains details of the execution provider and the device it will use. - * - * \param[in] env The OrtEnv instance to query. - * \param[out] ep_devices The OrtEpDevice instances that the execution provider will use. - * \param[out] num_ep_devices The number of OrtEpDevice instances returned. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(GetEpDevices, _In_ const OrtEnv* env, - _Outptr_ const OrtEpDevice* const** ep_devices, _Out_ size_t* num_ep_devices); - - /** \brief Append the execution provider that is responsible for the selected OrtEpDevice instances - * to the session options. - * - * \param[in] session_options Session options to add execution provider to. - * \param[in] env Environment that execution providers were registered with. - * \param[in] ep_devices One or more OrtEpDevice instances to create an execution provider for. - * Obtain from GetEpDevices. All OrtEpDevice instances must be from the same execution - * provider. It is only necessary to provide multiple OrtEpDevices if you want to use the - * same execution provider for multiple devices. - * e.g. the EP is capable of running on GPU and NPU. - * \param[in] num_ep_devices Number of OrtEpDevice instances. - * \param[in] ep_option_keys Optional keys to configure the execution provider. - * \param[in] ep_option_vals Optional values to configure the execution provider. - * \param[in] num_ep_options Number of execution provide options to add. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_V2, _In_ OrtSessionOptions* session_options, - _In_ OrtEnv* env, - _In_reads_(num_ep_devices) const OrtEpDevice* const* ep_devices, _In_ size_t num_ep_devices, - _In_reads_(num_op_options) const char* const* ep_option_keys, - _In_reads_(num_op_options) const char* const* ep_option_vals, - size_t num_ep_options); - - /** \brief Set the execution provider selection policy for the session. - * - * Allows users to specify a device selection policy for automatic execution provider (EP) selection. - * If custom selection is required please use SessionOptionsSetEpSelectionPolicyDelegate instead. - * - * \param[in] session_options The OrtSessionOptions instance. - * \param[in] policy The device selection policy to use (see OrtExecutionProviderDevicePolicy). - * - * \since Version 1.22 - */ - ORT_API2_STATUS(SessionOptionsSetEpSelectionPolicy, _In_ OrtSessionOptions* session_options, - _In_ OrtExecutionProviderDevicePolicy policy); - - /** \brief Set the execution provider selection policy delegate for the session. - * - * Allows users to provide a custom device selection policy for automatic execution provider (EP) selection. - * - * \param[in] session_options The OrtSessionOptions instance. - * \param[in] delegate Delegate callback for custom selection. - * \param[in] delegate_state Optional state that will be passed to the delegate callback. nullptr if not required. - * - * \since Version 1.22 - */ - ORT_API2_STATUS(SessionOptionsSetEpSelectionPolicyDelegate, _In_ OrtSessionOptions* session_options, - _In_ EpSelectionDelegate delegate, - _In_opt_ void* delegate_state); - - /** \brief Get the hardware device type. - * - * \param[in] device The OrtHardwareDevice instance to query. - * \return The hardware device type. - * - * \since Version 1.22. - */ - OrtHardwareDeviceType(ORT_API_CALL* HardwareDevice_Type)(_In_ const OrtHardwareDevice* device); - - /** \brief Get the hardware device's vendor identifier. - * - * \param[in] device The OrtHardwareDevice instance to query. - * \return The hardware device vendor identifier. - * - * \since Version 1.22. - */ - uint32_t(ORT_API_CALL* HardwareDevice_VendorId)(_In_ const OrtHardwareDevice* device); - - /** \brief Get the hardware device's vendor name. - * - * \param[in] device The OrtHardwareDevice instance to query. - * \return The hardware device's vendor name. - * - * \since Version 1.22. - */ - const char*(ORT_API_CALL* HardwareDevice_Vendor)(_In_ const OrtHardwareDevice* device); - - /** \brief Get the hardware device's unique identifier. - * - * \param[in] device The OrtHardwareDevice instance to query. - * \return The device id. - * - * \note This is not a unique identifier. It identifies the hardware type when combined with vendor id. - * \since Version 1.22. - */ - uint32_t(ORT_API_CALL* HardwareDevice_DeviceId)(_In_ const OrtHardwareDevice* device); - - /** \brief Get hardware device metadata. - * - * \param[in] device The OrtHardwareDevice instance to query. - * \return An OrtKeyValuePairs instance containing the metadata for the device. - * Note: ORT owns the instance so the user must not call ReleaseKeyValuePairs with it. - * - * \since Version 1.22. - */ - const OrtKeyValuePairs*(ORT_API_CALL* HardwareDevice_Metadata)(_In_ const OrtHardwareDevice* device); - - /** \brief Get the execution provider name. - * - * \param[in] ep_device The OrtEpDevice instance to query. - * \return The execution provider name. - * - * \since Version 1.22. - */ - const char*(ORT_API_CALL* EpDevice_EpName)(_In_ const OrtEpDevice* ep_device); - - /** \brief Get the execution provider's vendor name. - * - * \param[in] ep_device The OrtEpDevice instance to query. - * \return The execution provider's vendor name. - * - * \since Version 1.22. - */ - const char*(ORT_API_CALL* EpDevice_EpVendor)(_In_ const OrtEpDevice* ep_device); - - /** \brief Get the metadata for the OrtEpDevice. - * - * \param[in] ep_device The OrtEpDevice instance to query. - * \return An OrtKeyValuePairs instance containing the metadata for the device. - * - * \since Version 1.22. - */ - const OrtKeyValuePairs*(ORT_API_CALL* EpDevice_EpMetadata)(_In_ const OrtEpDevice* ep_device); - - /** \brief Get the execution provider options for the OrtEpDevice. - * - * \param[in] ep_device The OrtEpDevice instance to query. - * \return An OrtKeyValuePairs instance containing the execution provider options for the device. - * - * \since Version 1.22. - */ - const OrtKeyValuePairs*(ORT_API_CALL* EpDevice_EpOptions)(_In_ const OrtEpDevice* ep_device); - - /** \brief Get the OrtHardwareDevice instance for the OrtEpDevice. - * - * \param[in] ep_device The OrtEpDevice instance to query. - * \return The OrtHardwareDevice instance for the device. - * - * \since Version 1.22. - */ - const OrtHardwareDevice*(ORT_API_CALL* EpDevice_Device)(_In_ const OrtEpDevice* ep_device); - - /** \brief Get the OrtEpApi instance for implementing an execution provider. - * - * \since Version 1.22. - */ - const OrtEpApi*(ORT_API_CALL* GetEpApi)(); - - /** \brief Compute total size in bytes of the tensor data contained in an OrtValue. - * - * Returns the total number of bytes used to store the tensor data. For numeric tensors, - * this is sizeof(element_type) * total_element_count. OrtValues that are not tensors or - * that are tensors that contain strings will cause an error to be returned. - * - * \param[in] ort_value OrtValue instance containing a tensor - * \param[out] size The total size of the tensor data in bytes - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23 - */ - ORT_API2_STATUS(GetTensorSizeInBytes, _In_ const OrtValue* ort_value, _Out_ size_t* size); - - /** \brief Calls OrtAllocator::GetStats function - * - * Return a pointer to the OrtKeyValuePairs structure that contains the statistics of the allocator - * and the user should call OrtApi::ReleaseKeyValuePairs. - * - * NOTE: If the allocator does not implement this function, the OrtKeyValuePairs instance will be empty. - * - * \param[in] ort_allocator The allocator to get stats from - * \param[out] out A pointer to the OrtKeyValuePairs instance that contains the stats - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(AllocatorGetStats, _In_ const OrtAllocator* ort_allocator, _Outptr_ OrtKeyValuePairs** out); - - /** \brief Create an ::OrtMemoryInfo - * - * \param[in] name Arbitrary name. - * \param[in] device_type Device type. - * \param[in] vendor_id PCI Vendor ID. Use 0 for a generic allocator (e.g. WebGPU). - * \param[in] device_id Device ID if there are multiple devices of the same type. e.g. 2 GPU devices. - * \param[in] mem_type Memory type. Use OrtDeviceMemoryType_DEFAULT for device memory, and - * OrtDeviceMemoryType_HOST_ACCESSIBLE (if applicable) for memory used to transfer between the - * device and the CPU. Use the device_type and device_id of the GPU/NPU that the memory is also - * accessible to. - * \param[in] alignment Alignment of the memory if required. Pass 0 for default alignment. - * \param[in] allocator_type Allocator type. If OrtAllocatorType::OrtArenaAllocator, the ORT arena will be used. - * Caveat: Support for OrtArenaAllocator is currently limited to usage of internal ORT - * allocators via CreateAllocator/CreateAndRegisterAllocator/CreateAndRegisterAllocatorV2. - * \param[out] out Newly created ::OrtMemoryInfo. Must be freed with OrtApi::ReleaseMemoryInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23 - */ - ORT_API2_STATUS(CreateMemoryInfo_V2, _In_ const char* name, _In_ enum OrtMemoryInfoDeviceType device_type, - _In_ uint32_t vendor_id, _In_ int32_t device_id, _In_ enum OrtDeviceMemoryType mem_type, - _In_ size_t alignment, enum OrtAllocatorType allocator_type, - _Outptr_ OrtMemoryInfo** out); - - /** \brief Get the device memory type from ::OrtMemoryInfo - * - * \param[in] ptr The OrtMemoryInfo instance to query. - * \return The device memory type. - * - * \since Version 1.23 - */ - ORT_API_T(OrtDeviceMemoryType, MemoryInfoGetDeviceMemType, _In_ const OrtMemoryInfo* ptr); - - /** \brief Get the vendor id from ::OrtMemoryInfo - * - * \param[in] ptr The OrtMemoryInfo instance to query. - * \return The vendor id. - * - * \since Version 1.23 - */ - ORT_API_T(uint32_t, MemoryInfoGetVendorId, _In_ const OrtMemoryInfo* ptr); - - /// \name OrtValueInfo - /// @{ - - /** \brief Get the OrtNode that produces the value represented by the given OrtValueInfo. - * Optionally returns the associated output index. - * - * \param[in] value_info The OrtValueInfo instance. - * \param[out] producer_node Output parameter set to the OrtNode that produces the OrtValueInfo. - * \param[out] producer_output_index Optional output parameter set to the OrtNode instance's output index - * that produces the value. Ignored if set to NULL. - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.23. - */ - ORT_API2_STATUS(ValueInfo_GetValueProducer, _In_ const OrtValueInfo* value_info, - _Outptr_ const OrtNode** producer_node, _Out_opt_ size_t* producer_output_index); - - /** \brief Get the number of consumers of a value as a node input. - * - * Only nodes are considered "consumers" by this function. To check if an OrtValueInfo is a graph output, - * call ValueInfo_IsGraphOutput(). - * - * A single OrtNode may use a single value for more than one input (e.g., Mul(x, x)), so the returned - * `num_consumers` may be larger than the number of unique OrtNode instances that consume the value. - * - * \param[in] value_info The OrtValueInfo instance. - * \param[out] num_consumers Output parameter set to the number of consumers of the value. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(ValueInfo_GetValueNumConsumers, _In_ const OrtValueInfo* value_info, _Out_ size_t* num_consumers); - - /** \brief Returns information (OrtNode and input index) for all consumer nodes that use the value as an input. - * - * Only nodes are considered "consumers" by this function. - * - * Caller provides 2 pre-allocated arrays that will be filled with the OrtNode and input index values. - * Use ValueInfo_GetValueNumConsumers() to get the number of consumers of the value. - * - * An OrtNode instance may appear multiple times if it uses the given value more than once. - * Example: For a node MulNode(x, x) that consumes the value 'x' twice, the following is returned: - * - nodes: [MulNode, MulNode] - * - input_indices: [0, 1] - * - * \param[in] value_info The OrtValueInfo instance. - * \param[out] nodes Pre-allocated array of size `num_consumers` that is filled with OrtNode instances. - * \param[out] input_indices Pre-allocated array of `num_consumers` elements that is filled - * with input indices. Index is set to -1 for an "implicit" input to a consumer node - * that contains a subgraph (e.g., If, Loop) with nodes that use the value internally. - * \param[in] num_consumers The size of the `consumer_nodes` and `consumer_input_indices` arrays. - * Typical usage sets this to the value of ValueInfo_GetValueNumConsumers(). - * An error status is returned if `num_consumers` is less than the number of actual - * consumers. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(ValueInfo_GetValueConsumers, _In_ const OrtValueInfo* value_info, - _Out_writes_all_(num_consumers) const OrtNode** nodes, - _Out_writes_all_(num_consumers) int64_t* input_indices, - _In_ size_t num_consumers); - - /** \brief Get the underlying initializer value, as an OrtValue, from the given OrtValueInfo. - * - * Sets the output parameter to NULL if the given OrtValueInfo does not represent an initializer. - * Does not return an error status in this case. - * - * Supports initializers defined in an outer scope (i.e., a parent graph). - * - * Supports initializers stored in an external file. For external initializers, ORT memory maps - * the initializer data on the first call to this function. If caller needs custom memory mapping, - * use ValueInfo_GetExternalInitializerInfo to get the location of the initializer data. - * - * \param[in] value_info The OrtValueInfo instance. - * \param[out] initializer_value Output parameter set to the initializer value or NULL. Do not cache the OrtValue - * as it is released when the owning OrtGraph is released. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(ValueInfo_GetInitializerValue, _In_ const OrtValueInfo* value_info, - _Outptr_ const OrtValue** initializer_value); - - /** \brief Get information about an external initializer (e.g., filepath, file offset, byte size). - * - * Sets the output parameter `info` to NULL if the given OrtValueInfo does not represent an initializer - * with external data. In this case, a NULL status (non-error) is returned. - * - * \param[in] value_info The OrtValueInfo instance. - * \param[out] info Output parameter set to an OrtExternalInitializerInfo instance that can be used to query - * file path, file offset, etc. ORT sets this to NULL if the OrtValueInfo does not represent - * an external initializer. - * Must release with ReleaseExternalInitializerInfo. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(ValueInfo_GetExternalInitializerInfo, _In_ const OrtValueInfo* value_info, - _Outptr_result_maybenull_ OrtExternalInitializerInfo** info); - - /** \brief Returns a boolean indicating if the given value is a required graph input. - * - * For ONNX IR version < 4, all graph inputs without a matching initializer are required. - * - * For ONNX IR version >=4, a graph input with a matching initializer is an optional graph input - * with the initializer serving as the default value. - * - * \param[in] value_info The OrtValueInfo instance representing the graph value. - * \param[out] is_required_graph_input Output parameter set to true if the graph value is a required graph input. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(ValueInfo_IsRequiredGraphInput, _In_ const OrtValueInfo* value_info, - _Out_ bool* is_required_graph_input); - - /** \brief Returns a boolean indicating if the given value is an optional graph input. - * - * Optional graph inputs were introduced in ONNX IR version 4. For ONNX IR version >=4, a graph input with a - * matching initializer is an optional graph input with the initializer serving as the default value. - * The matching initializer is also known as a non-constant initializer. - * - * \param[in] value_info The OrtValueInfo instance representing the graph value. - * \param[out] is_optional_graph_input Output parameter set to true if the graph value is an optional graph input. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(ValueInfo_IsOptionalGraphInput, _In_ const OrtValueInfo* value_info, - _Out_ bool* is_optional_graph_input); - - /** \brief Returns a boolean indicating if the given value is a graph output. - * - * \param[in] value_info The OrtValueInfo instance representing the graph value. - * \param[out] is_graph_output Output parameter set to true if the graph value is a graph output. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(ValueInfo_IsGraphOutput, _In_ const OrtValueInfo* value_info, _Out_ bool* is_graph_output); - - /** \brief Returns a boolean indicating if the given value is a constant initializer. - * - * For ONNX IR version < 4, all initializers are constant. - * - * For ONNX IR version >=4, an initializer that serves as the default value for a matching graph input is not a - * constant initializer. - * - * \param[in] value_info The OrtValueInfo instance representing the graph value. - * \param[out] is_constant_initializer Output parameter set to true if the graph value is a constant initializer. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(ValueInfo_IsConstantInitializer, _In_ const OrtValueInfo* value_info, - _Out_ bool* is_constant_initializer); - - /** \brief Returns a boolean indicating if the given value is defined in an outer scope. - * - * Certain operator types (e.g., If and Loop) contain nested subgraphs. This function enables - * determining whether a value is defined in a parent node's graph. - * - * \param[in] value_info The OrtValueInfo instance representing the graph value. - * \param[out] is_from_outer_scope Output parameter set to true if the value is defined in an outer - * scope (i.e., a parent graph). - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(ValueInfo_IsFromOuterScope, _In_ const OrtValueInfo* value_info, - _Out_ bool* is_from_outer_scope); - - /// @} - - /// \name OrtGraph - /// @{ - - /** \brief Returns a graph's name. - * - * \param[in] graph The OrtGraph instance. - * \param[out] graph_name Output parameter set to the graph's name. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Graph_GetName, _In_ const OrtGraph* graph, _Outptr_ const char** graph_name); - - /** \brief Get the filepath to the model from which an OrtGraph is constructed. - * - * \note The model's filepath is empty if the filepath is unknown, such as when the model is loaded from bytes - * via CreateSessionFromArray. - * - * \param[in] graph The OrtGraph instance. - * \param[out] model_path Output parameter set to the model's null-terminated filepath. - * Set to an empty path string if unknown. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Graph_GetModelPath, _In_ const OrtGraph* graph, _Outptr_ const ORTCHAR_T** model_path); - - /** \brief Returns the ONNX IR version. - * - * \param[in] graph The OrtGraph instance. - * \param[out] onnx_ir_version Output parameter set to the ONNX IR version. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Graph_GetOnnxIRVersion, _In_ const OrtGraph* graph, _Out_ int64_t* onnx_ir_version); - - /** \brief Returns the number of operator sets that the graph's model uses. - * - * \note An operator set is uniquely identified by the (domain, opset_version) pair. All models must have at - * least one entry that specifies which entry of the ONNX operator set is used. The ONNX domain is represented by - * an empty string. - * - * \param[in] graph The OrtGraph instance. - * \param[out] num_operator_sets Output parameter set to the number of operator sets that the graph's model uses. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Graph_GetNumOperatorSets, _In_ const OrtGraph* graph, _Out_ size_t* num_operator_sets); - - /** \brief Returns the operator sets that the graph's model uses. - * - * \note An operator set is uniquely identified by the (domain, opset_version) pair. All models must have at - * least one entry that specifies which entry of the ONNX operator set is used. The ONNX domain is represented by - * an empty string. - * - * \param[in] graph The OrtGraph instance. - * \param[out] domains Pre-allocated array of `num_operator_sets` elements that is filled with - * null-terminated domain names. - * \param[out] opset_versions Pre-allocated array of `num_operator_sets` elements that is filled with - * the opset version of the corresponding domain in the `domains` array. - * \param[in] num_operator_sets The size of the `domains` and `opset_versions` arrays. - * Typical usage sets this to the result of Graph_GetNumOperatorSets(). - * An error status is returned if `num_operator_sets` is less than the actual number - * of operator sets. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Graph_GetOperatorSets, _In_ const OrtGraph* graph, - _Out_writes_(num_operator_sets) const char** domains, - _Out_writes_(num_operator_sets) int64_t* opset_versions, _In_ size_t num_operator_sets); - - /** \brief Returns the number of graph inputs. - * - * \note The count includes initializers that are included in the list of graph inputs. - * - * \param[in] graph The OrtGraph instance. - * \param[out] num_inputs Output parameter set to the number of graph inputs. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Graph_GetNumInputs, _In_ const OrtGraph* graph, _Out_ size_t* num_inputs); - - /** \brief Returns the graph's inputs as OrtValueInfo instances. - * - * \note The result includes initializers that are included in the list of graph inputs. - * - * \param[in] graph The OrtGraph instance. - * \param[out] inputs Pre-allocated array of `num_inputs` elements that is filled with the graph's inputs. - * \param[in] num_inputs The size of the `inputs` array. - * Typical usage sets this to the result of Graph_GetNumInputs(). An error status is - * returned if `num_inputs` is less than the number of graph inputs. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Graph_GetInputs, _In_ const OrtGraph* graph, - _Out_writes_(num_inputs) const OrtValueInfo** inputs, _In_ size_t num_inputs); - - /** \brief Returns the number of graph outputs. - * - * \param[in] graph The OrtGraph instance. - * \param[out] num_outputs Output parameter set to the number of graph outputs. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Graph_GetNumOutputs, _In_ const OrtGraph* graph, _Out_ size_t* num_outputs); - - /** \brief Returns the graph's outputs as OrtValueInfo instances. - * - * \param[in] graph The OrtGraph instance. - * \param[out] outputs Pre-allocated array of `num_outputs` elements that is filled with the graph's outputs. - * \param[in] num_outputs The size of the `outputs` array. - * Typical usage sets this to the result of Graph_GetNumOutputs(). An error status is - * returned if `num_outputs` is less than the number of graph outputs. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Graph_GetOutputs, _In_ const OrtGraph* graph, - _Out_writes_(num_outputs) const OrtValueInfo** outputs, _In_ size_t num_outputs); - - /** \brief Returns the number of graph initializers. - * - * Counts constant and non-constant initializers. - * - * \param[in] graph The OrtGraph instance. - * \param[out] num_initializers Output parameter set to the number of graph initializers. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Graph_GetNumInitializers, _In_ const OrtGraph* graph, _Out_ size_t* num_initializers); - - /** \brief Returns the graph's initializers as OrtValueInfo instances. - * - * Includes constant and non-constant initializers. - * - * For ONNX IR version < 4, all initializers are constant. - * - * For ONNX IR version >= 4, an initializer with a name that matches a graph input is considered a - * non-constant initializer. - * - * Call ValueInfo_GetInitializerValue to get the initializer's data. - * - * \param[in] graph The OrtGraph instance. - * \param[out] initializers Pre-allocated array of `num_outputs` elements that is filled with the initializers. - * \param[in] num_initializers The size of the `initializers` array. Typical usage sets this to the - * result of Graph_GetNumInitializers(). An error status is returned if - * `num_initializers` is less than the number of graph initializers. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Graph_GetInitializers, _In_ const OrtGraph* graph, - _Out_writes_(num_initializers) const OrtValueInfo** initializers, - _In_ size_t num_initializers); - - /** \brief Returns the number of graph nodes. - * - * \param[in] graph The OrtGraph instance. - * \param[out] num_nodes Output parameter set to the number of graph nodes. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Graph_GetNumNodes, _In_ const OrtGraph* graph, _Out_ size_t* num_nodes); - - /** \brief Returns the graph's nodes as OrtNode instances. - * - * The nodes are sorted using a stable topological ordering. Callers are responsible for maintaining their - * own node ordering if a different order is required. - * - * \param[in] graph The OrtGraph instance. - * \param[out] nodes Pre-allocated array of `num_nodes` elements that is filled with the graph's nodes. - * \param[in] num_nodes The size of the `nodes` array. Typical usage sets this to the - * result of Graph_GetNumNodes(). An error status is returned if - * `num_nodes` is less than the number of graph nodes. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Graph_GetNodes, _In_ const OrtGraph* graph, - _Out_writes_(num_nodes) const OrtNode** nodes, _In_ size_t num_nodes); - - /** \brief Get the parent node for the given graph, if any exists. - * - * Certain operator types (e.g., If and Loop) contain nested subgraphs. This function enables - * access to the parent node (e.g., the If and Loop node) from a nested subgraph. - * - * \param[in] graph The OrtGraph instance. - * \param[out] node Output parameter that is set to the graph's parent node. - * Set to NULL if a parent node does not exist (e.g., for a top-level graph). - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Graph_GetParentNode, _In_ const OrtGraph* graph, _Outptr_result_maybenull_ const OrtNode** node); - - /** \brief Returns an OrtGraph that contains a subset of nodes in the source OrtGraph. - * - * \note The lifetime of "dst_graph" is tied to that of "src_graph", as they both internally reference - * the same underlying graph. - * - * \param[in] src_graph The source OrtGraph instance. - * \param[in] nodes A subset of the nodes/OrtNodes in 'graph'. - * \param[in] num_nodes Number of nodes. - * \param[out] dst_graph An OrtGraph created from a given set of nodes. Must be released by calling ReleaseGraph. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Graph_GetGraphView, _In_ const OrtGraph* src_graph, _In_ const OrtNode** nodes, - _In_ size_t num_nodes, _Outptr_ OrtGraph** dst_graph); - - /// @} - - /// \name OrtNode - /// @{ - - /** \brief Returns a node's identifier. - * - * The node's identifier is only unique in the node's parent graph. Different nested subgraphs - * (e.g., subgraphs contained by If and Loop nodes) may reuse identifiers. - * - * \param[in] node The OrtNode instance. - * \param[out] node_id Output parameter set to the node's identifier. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Node_GetId, _In_ const OrtNode* node, _Out_ size_t* node_id); - - /** \brief Returns a node's name. Can be an empty string. - * - * \param[in] node The OrtNode instance. - * \param[out] node_name Output parameter set to the node's name. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Node_GetName, _In_ const OrtNode* node, _Outptr_ const char** node_name); - - /** \brief Returns a node's operator type (e.g., "Conv"). - * - * \param[in] node The OrtNode instance. - * \param[out] operator_type Output parameter set to the name of the node's operator type. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Node_GetOperatorType, _In_ const OrtNode* node, _Outptr_ const char** operator_type); - - /** \brief Returns a node's domain name. - * - * \param[in] node The OrtNode instance. - * \param[out] domain_name Output parameter set to the node's domain name. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Node_GetDomain, _In_ const OrtNode* node, _Outptr_ const char** domain_name); - - /** \brief Get the opset version in which the given node's operator type was first defined. - * - * \param[in] node The OrtNode instance. - * \param[out] since_version The opset version in which the node's operator type was first defined. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Node_GetSinceVersion, _In_ const OrtNode* node, _Out_ int* since_version); - - /** \brief Returns the number of node inputs. - * - * \param[in] node The OrtNode instance. - * \param[out] num_inputs Output parameter set to the number of node inputs. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Node_GetNumInputs, _In_ const OrtNode* node, _Out_ size_t* num_inputs); - - /** \brief Returns the node's inputs as OrtValueInfo instances. - * - * \param[in] node The OrtNode instance. - * \param[out] inputs Pre-allocated array of `num_inputs` elements that is filled with the node's inputs. - * \param[in] num_inputs The size of the `inputs` array. - * Typical usage sets this to the result of Node_GetNumInputs(). An error status is - * returned if `num_inputs` is less than the number of node inputs. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Node_GetInputs, _In_ const OrtNode* node, - _Out_writes_(num_inputs) const OrtValueInfo** inputs, _In_ size_t num_inputs); - - /** \brief Returns the number of node outputs. - * - * \param[in] node The OrtNode instance. - * \param[out] num_outputs Output parameter set to the number of node outputs. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Node_GetNumOutputs, _In_ const OrtNode* node, _Out_ size_t* num_outputs); - - /** \brief Returns the node's outputs as OrtValueInfo instances. - * - * \param[in] node The OrtNode instance. - * \param[out] outputs Pre-allocated array of `num_outputs` elements that is filled with the node's outputs. - * \param[in] num_outputs The size of the `outputs` array. - * Typical usage sets this to the result of Node_GetNumOutputs(). An error status is - * returned if `num_outputs` is less than the number of node outputs. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Node_GetOutputs, _In_ const OrtNode* node, - _Out_writes_(num_outputs) const OrtValueInfo** outputs, _In_ size_t num_outputs); - - /** \brief Returns the number of node implicit inputs. - * - * Certain operator types (e.g., If and Loop) contain nested subgraphs. The internal nodes within the nested subgraphs - * may use values from the outer scope. Those "outer scope" values are considered implicit inputs to the node that - * contains the subgraphs (e.g., the If or Loop node). - * - * \param[in] node The OrtNode instance. - * \param[out] num_implicit_inputs Output parameter set to the number of node implicit inputs. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Node_GetNumImplicitInputs, _In_ const OrtNode* node, _Out_ size_t* num_implicit_inputs); - - /** \brief Get the implicit inputs, as OrtValueInfo instances, that are used within the given node's subgraphs. - * - * \note Only certain operator types (e.g., If and Loop) contain nested subgraphs. - * The internal nodes within the nested subgraphs may use values from the outer scope. Those "outer scope" values - * are considered implicit inputs to the node that contains the subgraphs (e.g., the If or Loop node). - * - * \param[in] node The OrtNode instance. - * \param[out] implicit_inputs Pre-allocated array of `num_implicit_inputs` elements that is filled the node's - * implicit inputs. - * \param[in] num_implicit_inputs The size of the `implicit_inputs` array. Typical usage sets this to the result - * of Node_GetNumImplicitInputs(). An error status is returned if - * `num_implicit_inputs` is less than the number of node implicit inputs. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Node_GetImplicitInputs, _In_ const OrtNode* node, - _Out_writes_(num_implicit_inputs) const OrtValueInfo** implicit_inputs, - _In_ size_t num_implicit_inputs); - - /** \brief Returns the number of node attributes. - * - * \param[in] node The OrtNode instance. - * \param[out] num_attributes Output parameter set to the number of node attributes. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Node_GetNumAttributes, _In_ const OrtNode* node, _Out_ size_t* num_attributes); - - /** \brief Returns a node's attributes as OrtOpAttr instances. - * - * \param[in] node The OrtNode instance. - * \param[out] attributes Pre-allocated array of `num_attributes` elements that is filled with the node's attributes. - * \param[in] num_attributes The size of the `num_attributes` array. - * Typical usage sets this to the result of Node_GetNumAttributes(). An error status is - * returned if `num_attributes` is less than the number of node attributes. - * - * \note ONNX Runtime automatically sets optional (unset) attributes to their default values if the default value - * is a constant expression that does not depend on other tensor/model characteristics. Conv's 'kernel_shape' - * attribute is an example of an optional attribute that does not have a constant default value. This function - * does not provide any unset optional attributes without a constant default value. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Node_GetAttributes, _In_ const OrtNode* node, - _Out_writes_(num_attributes) const OrtOpAttr** attributes, _In_ size_t num_attributes); - - /** \brief Gets the OrtNode's attribute as OrtOpAttr by name. - * - * \param[in] node The OrtNode instance. - * \param[in] attribute_name The name of the attribute - * \param[out] attribute Output parameter set to the OrtOpAttr instance if an attribute by the given name exists. - * For an unset optional attribute, `attribute` is set to NULL and a non-error status is - * returned. For an invalid attribute name, `attribute` is set to NULL and an error status with - * code ORT_NOT_FOUND is returned. - * - * \note ONNX Runtime automatically sets optional (unset) attributes to their default values if the default value - * is a constant expression that does not depend on other tensor/model characteristics. Conv's 'kernel_shape' - * attribute is an example of an optional attribute that does not have a constant default value. This function - * does not provide any unset optional attributes without a constant default value. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Node_GetAttributeByName, _In_ const OrtNode* node, _In_ const char* attribute_name, - _Outptr_result_maybenull_ const OrtOpAttr** attribute); - - /** \brief Get the OrtNode's 'TENSOR' attribute as an OrtValue. - * - * \param[in] attribute The OrtOpAttr instance. - * \param[out] attr_tensor If successful, contains the 'TENSOR' attribute as a newly created OrtValue. - Must be freed with OrtApi::ReleaseValue. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(OpAttr_GetTensorAttributeAsOrtValue, _In_ const OrtOpAttr* attribute, - _Outptr_result_maybenull_ OrtValue** attr_tensor); - - /** \brief Get the attribute type as OrtOpAttrType from an OrtOpAttr. - * - * \param[in] attribute The OrtOpAttr instance. - * \param[out] type Output the attribute type as OrtOpAttrType. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(OpAttr_GetType, _In_ const OrtOpAttr* attribute, _Out_ OrtOpAttrType* type); - - /** \brief Get the attribute name from an OrtOpAttr. - * - * \param[in] attribute The OrtOpAttr instance. - * \param[out] name Output parameter set to the attribute's name. The name is a null-terminated string. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(OpAttr_GetName, _In_ const OrtOpAttr* attribute, _Outptr_ const char** name); - - /** \brief Returns the number of subgraphs contained by the given node. - * - * \note Only certain operator types (e.g., If and Loop) contain nested subgraphs. - * - * \param[in] node The OrtNode instance. - * \param[out] num_subgraphs Output parameter set to the number of node subgraphs. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Node_GetNumSubgraphs, _In_ const OrtNode* node, _Out_ size_t* num_subgraphs); - - /** \brief Get the subgraphs, as OrtGraph instances, contained by the given node. - * - * \note Only certain operator types (e.g., If and Loop) contain nested subgraphs. ONNX nodes store subgraphs in - * their attributes, however, this function must be used to obtain subgraphs from an OrtNode. - * - * \param[in] node The OrtNode instance. - * \param[out] subgraphs Pre-allocated array of `num_subgraphs` elements that is filled with the node's subgraphs. - * \param[in] num_subgraphs The size of the `num_subgraphs` array. - * Typical usage sets this to the result of Node_GetNumSubgraphs(). An error status is - * returned if `num_subgraphs` is less than the number of node subgraphs. - * \param[out] attribute_names Optional pre-allocated array of `num_subgraphs` elements that is filled with the - * attribute names that correspond to the subgraphs. Ignored if set to NULL. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Node_GetSubgraphs, _In_ const OrtNode* node, - _Out_writes_(num_subgraphs) const OrtGraph** subgraphs, _In_ size_t num_subgraphs, - _Out_writes_opt_(num_subgraphs) const char** attribute_names); - - /** \brief Get the node's parent OrtGraph instance. - * - * Can return NULL if the OrtNode was created without an owning graph. - * - * \param[in] node The OrtNode instance. - * \param[out] graph Output parameter set to the node's OrtGraph. Can be set to NULL - * if the node is not currently contained by a graph. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Node_GetGraph, _In_ const OrtNode* node, _Outptr_result_maybenull_ const OrtGraph** graph); - - /** \brief Returns the execution provider name that this node is assigned to run on. - * Returns NULL if the node has not been assigned to any execution provider yet. - * For plugin execution providers, the name is the one returned by OrtEp::GetName. - * - * \param[in] node The OrtNode instance. - * \param[out] out Output execution provider type and can be NULL if node has not been assigned. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Node_GetEpName, _In_ const OrtNode* node, _Outptr_result_maybenull_ const char** out); - - /// @} - - /// \name OrtExternalInitializerInfo - /// @{ - - /** \brief Release an OrtExternalInitializerInfo instance. - * - * \param[in] input OrtExternalInitializerInfo instance to be released. - * - * \since Version 1.23. - */ - ORT_CLASS_RELEASE(ExternalInitializerInfo); - - /** \brief Get the relative path to the file that stores the initializer's data. - * - * \note The path is relative to the filesystem directory where the ONNX model was stored. - * Caller can use Graph_GetModelPath to get the model's full path and construct the absolute path to the - * external initializer file if necessary. - * - * \param[in] info The OrtExternalInitializerInfo instance. - * \return The relative path to the file that stores the initializer's data. Do NOT free this pointer. - * - * \since Version 1.23. - */ - ORT_API_T(const ORTCHAR_T*, ExternalInitializerInfo_GetFilePath, _In_ const OrtExternalInitializerInfo* info); - - /** \brief Get the byte offset within the file where the initializer's data is stored. - * - * \param[in] info The OrtExternalInitializerInfo instance. - * \return The byte offset where the initializer's data is stored within the file. - * - * \since Version 1.23. - */ - ORT_API_T(int64_t, ExternalInitializerInfo_GetFileOffset, _In_ const OrtExternalInitializerInfo* info); - - /** \brief Get the size in bytes of the initializer's data within the file. - * - * \param[in] info The OrtExternalInitializerInfo instance. - * \return The size in bytes of the initializer's data within the file. - * - * \since Version 1.23. - */ - ORT_API_T(size_t, ExternalInitializerInfo_GetByteSize, _In_ const OrtExternalInitializerInfo* info); - - /// @} - - /// \name OrtRunOptions - /// @{ - - /** \brief Get a run configuration entry. - * - * If a run configuration entry with key `config_key` doesn't exist, `config_value` will be set to NULL. - * - * `config_key`s are defined in onnxruntime_run_options_config_keys.h. - * - * \param[in] options The OrtRunOptions instance. - * \param[in] config_key The configuration entry key. A null-terminated string. - * \return The configuration entry value. Either a null-terminated string if the entry was found. nullptr otherwise. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23 - */ - ORT_API_T(const char*, GetRunConfigEntry, _In_ const OrtRunOptions* options, - _In_z_ const char* config_key); - - /// @} - - /** \brief Get the OrtMemoryInfo for the device. - * - * \param[in] ep_device The OrtEpDevice instance to query. - * \param[in] memory_type The memory type to return. - * \return A pointer to the OrtMemoryInfo for the device. This may be nullptr if not set. - * If memory_type is OrtDeviceMemoryType_DEFAULT and nullptr is returned the EP uses CPU memory. - * - * \since Version 1.23 - */ - ORT_API_T(const OrtMemoryInfo*, EpDevice_MemoryInfo, _In_ const OrtEpDevice* ep_device, - _In_ OrtDeviceMemoryType memory_type); - - /** \brief Create/replace a shared allocator for the OrtEpDevice in the OrtEnv. - * - * OrtEpDevice maps to the EP factory, and the factory provides the allocator implementation. - * - * Both OrtDeviceMemoryType_DEFAULT and OrtDeviceMemoryType_HOST_ACCESSIBLE are optional for an EP to provide. - * It is EP implementation dependent as to what is available. - * - * If a shared allocator already exists for the OrtEpDevice and OrtDeviceMemoryType, it is replaced. This allows - * changing the shared allocator configuration from the default. e.g. adding an arena. - * - * \param[in] env The OrtEnv instance to create the shared allocator in. - * \param[in] ep_device The OrtEpDevice instance to create the shared allocator for. - * \param[in] mem_type The memory type to use for the shared allocator. - * \param[in] allocator_type The type of allocator to create. Only OrtDeviceAllocator is valid currently. - * \param[in] allocator_options Optional key-value pairs to configure the allocator. If arena based, see - * include/onnxruntime/core/framework/allocator.h for the keys and values that can be - * used. - * \param[out] allocator A pointer to the created shared allocator. Owned by the OrtEnv instance. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23 - */ - ORT_API2_STATUS(CreateSharedAllocator, _In_ OrtEnv* env, _In_ const OrtEpDevice* ep_device, - _In_ OrtDeviceMemoryType mem_type, _In_ OrtAllocatorType allocator_type, - _In_opt_ const OrtKeyValuePairs* allocator_options, - _Outptr_opt_ OrtAllocator** allocator); - - /** \brief Get a shared allocator from the OrtEnv. - * - * By default there is a shared allocator created for all OrtEpDevice instances, so if you get the OrtMemoryInfo - * from the OrtEpDevice using EpDevice_MemoryInfo a shared allocator is guaranteed to exist. - * - * This will also match and return custom allocators added with RegisterAllocator. - * - * It is not an error to not find a matching allocator. - * - * \param[in] env The OrtEnv instance to get the shared allocator from. - * \param[in] mem_info The OrtMemoryInfo instance to get the shared allocator for. - * \param[out] allocator A pointer to the shared allocator, or nullptr if no shared allocator exists for - * the given memory info. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23 - */ - ORT_API2_STATUS(GetSharedAllocator, _In_ OrtEnv* env, _In_ const OrtMemoryInfo* mem_info, - _Outptr_result_maybenull_ OrtAllocator** allocator); - - /** \brief Release a shared allocator from the OrtEnv for the OrtEpDevice and memory type. - * - * This will release the shared allocator for the given OrtEpDevice and memory type. - * If no shared allocator exists, this is a no-op. - * - * \param[in] env The OrtEnv instance to release the shared allocator from. - * \param[in] ep_device The OrtEpDevice instance to release the shared allocator for. - * \param[in] mem_type The memory type of the shared allocator to release. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23 - */ - ORT_API2_STATUS(ReleaseSharedAllocator, _In_ OrtEnv* env, _In_ const OrtEpDevice* ep_device, - _In_ OrtDeviceMemoryType mem_type); - - /** \brief Get a const pointer to the raw data inside a tensor - * - * Used to read the internal tensor data directly. - * \note The returned pointer is valid until the OrtValue is destroyed. - * - * \param[in] value A tensor type (string tensors are not supported) - * \param[out] out Filled in with a pointer to the internal storage - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23 - */ - ORT_API2_STATUS(GetTensorData, _In_ const OrtValue* value, _Outptr_ const void** out); - - /** \brief Get Session configuration entries. - * - * \param[in] options The session options. - * \param[out] out A pointer to a newly created OrtKeyValuePairs instance. - * - * An OrtKeyValuePairs instance containing all session configuration entries. - * Note: the user should call OrtApi::ReleaseKeyValuePairs. - * - * \since Version 1.23. - */ - ORT_API2_STATUS(GetSessionOptionsConfigEntries, _In_ const OrtSessionOptions* options, _Outptr_ OrtKeyValuePairs** out); - - /** \brief Get the OrtMemoryInfo for each input of the session. - * - * The memory info can be used to determine where the input tensors are required. - * - * The session must be fully initialized before calling this function as the input locations are not known until - * this has occurred. - * - * \param[in] session The OrtSession instance. - * \param[out] inputs_memory_info Pre-allocated array of size `num_inputs` that will be filled with the - * OrtMemoryInfo* value for each input. - * The order is the same as returned by SessionGetInputName. - * \param[in] num_inputs The number of inputs in the session. Must match SessionGetInputCount. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23 - */ - ORT_API2_STATUS(SessionGetMemoryInfoForInputs, _In_ const OrtSession* session, - _Out_writes_(num_inputs) const OrtMemoryInfo** inputs_memory_info, - _In_ size_t num_inputs); - - /** \brief Get the OrtMemoryInfo for each output of the session. - * - * The memory info can be used to determine the device the output tensors are produced on. - * The user can pre-allocate an OrtValue using this information or use IOBinding to keep the data on the device. - * ORT will copy the output to CPU otherwise. - * - * The session must be fully initialized before calling this function as the output locations are not known until - * this has occurred. - * - * \param[in] session The OrtSession instance. - * \param[out] outputs_memory_info Pre-allocated array of size `num_outputs` that will be filled with - * OrtMemoryInfo* values for each output. - * The order is the same as returned by SessionGetOutputName. - * \param[in] num_outputs The number of outputs in the session. Must match SessionGetOutputCount. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23 - */ - ORT_API2_STATUS(SessionGetMemoryInfoForOutputs, _In_ const OrtSession* session, - _Out_writes_(num_outputs) const OrtMemoryInfo** outputs_memory_info, - _In_ size_t num_outputs); - - /** \brief Get the OrtEpDevice (if available) for each input of the session. - * - * An OrtEpDevice will be available if auto EP selection is enabled by calling - * SessionOptionsSetEpSelectionPolicy or SessionOptionsSetEpSelectionPolicyDelegate, - * or if the OrtEpDevice was manually added to the session using SessionOptionsAppendExecutionProvider_V2. - * - * If an OrtEpDevice is not available for the input a nullptr is returned. - * - * The returned OrtEpDevice can be used to create an OrtSyncStream via CreateSyncStreamForEpDevice to asynchronously - * provide input to the inference session Run. - * - * The session must be fully initialized before calling this function as the assigned EPs are not known until - * this has occurred. - * - * \param[in] session The OrtSession instance. - * \param[out] inputs_ep_devices Pre-allocated array of size `num_inputs` that will be filled with - * OrtEpDevice* values for each input. - * The order is the same as returned by SessionGetInputName. - * \param[in] num_inputs The number of inputs in the session. Must match SessionGetInputCount. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23 - */ - ORT_API2_STATUS(SessionGetEpDeviceForInputs, _In_ const OrtSession* session, - _Out_writes_(num_inputs) const OrtEpDevice** inputs_ep_devices, - _In_ size_t num_inputs); - - /** \brief Create an OrtSyncStream for the given OrtEpDevice. - * - * The OrtSyncStream can be used to enable asynchronous operations. - * e.g. async usage of CopyTensors to provide input to an OrtSession Run call. - * - * An error code of ORT_NOT_IMPLEMENTED will be returned if the EP does not support OrtSyncStream. - * - * \param[in] ep_device The OrtEpDevice instance to create the sync stream for. - * \param[in] stream_options Options for OrtSyncStream creation. May be nullptr. - * \param[out] stream Output parameter set to the created OrtSyncStream instance. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23 - */ - ORT_API2_STATUS(CreateSyncStreamForEpDevice, _In_ const OrtEpDevice* ep_device, - _In_opt_ const OrtKeyValuePairs* stream_options, - _Outptr_ OrtSyncStream** stream); - - /** \brief Get the native handle of the sync stream. - * - * This returns the native handle for the stream. e.g. cudaStream_t for CUDA streams. - * - * \param[in] stream The OrtSyncStream instance to get the handle from. - * - * \returns The native handle of the stream. - * - * \since Version 1.23 - */ - ORT_API_T(void*, SyncStream_GetHandle, _In_ OrtSyncStream* stream); - - ORT_CLASS_RELEASE(SyncStream); - - /** \brief Copy OrtValue instances containing Tensors between devices. - * - * The overall copy must be between a single source device and a single destination device. i.e. - * - all src_tensors must have matching OrtMemoryInfo, - * - all dst_tensors must have matching OrtMemoryInfo. - * - * OrtValue instances can be created by: - * - Use GetSharedAllocator to get the shared allocator for the OrtMemoryInfo if you need to allocate memory - * on the device. - * - Use CreateTensorAsOrtValue, CreateTensorWithDataAsOrtValue or CreateTensorWithDataAndDeleterAsOrtValue - * to create an OrtValue containing a tensor depending on whether you have existing data or not, and whether - * you want ORT to free the existing data once it is done with the OrtValue. - * - * \param[in] env The OrtEnv instance to use. The data transfer implementation is provided by an execution provider - * that is registered in this OrtEnv. - * \param[in] src_tensors Array of OrtValue instances containing the source tensors to copy. - * \param[in] dst_tensors Array of OrtValue instances to copy the source tensors to. - * \param[in] stream Optional OrtSyncStream that can be used to perform the copy asynchronously. May be nullptr. - * \param[in] num_tensors The number of tensors to copy. The size of `src_tensors` and `dst_tensors` must match. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23 - */ - ORT_API2_STATUS(CopyTensors, _In_ const OrtEnv* env, - _In_reads_(num_tensors) const OrtValue* const* src_tensors, - _In_reads_(num_tensors) OrtValue* const* dst_tensors, - _In_opt_ OrtSyncStream* stream, - _In_ size_t num_tensors); - - /** \brief Get ::OrtModelMetadata from an ::OrtGraph - * - * \param[in] graph The OrtGraph instance. - * \param[out] out Newly created ::OrtModelMetadata. Must be freed using OrtApi::ReleaseModelMetadata. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Graph_GetModelMetadata, _In_ const OrtGraph* graph, _Outptr_ OrtModelMetadata** out); - - /** \brief Validate a compiled model's compatibility information for one or more EP devices. - * - * \param[in] ep_devices The EP devices to validate against (e.g., from GetEpDevices). - * All devices must belong to the same execution provider. - * \param[in] num_ep_devices The number of EP devices provided. - * \param[in] compatibility_info The compatibility info string produced when the model was compiled. - * \param[out] out_status The resulting compatibility status for the EP devices. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(GetModelCompatibilityForEpDevices, - _In_reads_(num_ep_devices) const OrtEpDevice* const* ep_devices, - _In_ size_t num_ep_devices, - _In_ const char* compatibility_info, - _Out_ OrtCompiledModelCompatibility* out_status); - - /// \name OrtExternalInitializerInfo - /// @{ - - /** \brief Creates an OrtExternalInitializerInfo instance. - * - * \param[in] filepath The relative path to the file that stores the initializer's data. ORT copies this path string. - * \param[in] file_offset The byte offset where the initializer's data is stored within the file. - * \param[in] byte_size The size in bytes of the initializer's data within the file. - * \param[out] out Output parameter set to the new OrtExternalInitializerInfo instance. - * Must be released by calling ReleaseExternalInitializerInfo(). - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(CreateExternalInitializerInfo, _In_ const ORTCHAR_T* filepath, _In_ int64_t file_offset, - _In_ size_t byte_size, _Outptr_ OrtExternalInitializerInfo** out); - - /// @} }; /* @@ -6640,7 +4718,7 @@ struct OrtCustomOp { // Returns the memory type of the input tensors. This API allows the custom op // to place the inputs on specific devices. By default, it returns // OrtMemTypeDefault, which means the input is placed on the default device for - // the execution provider. If the inputs need to be with different memory types, + // the execution provider. If the inputs need to be with different memory tyeps, // this function can be overridden to return the specific memory types. OrtMemType(ORT_API_CALL* GetInputMemoryType)(_In_ const struct OrtCustomOp* op, _In_ size_t index); @@ -6692,689 +4770,6 @@ struct OrtCustomOp { void(ORT_API_CALL* ReleaseAliasMap)(_Frees_ptr_opt_ int* input_index, _Frees_ptr_opt_ int* output_index); }; -/** - * ORT Model Editor API - */ - -/** - * \brief The OrtModelEditorApi struct provides functions to create or edit an ONNX model. - * - * See onnxruntime/test/shared_lib/test_model_editor_api.cc for example usage. - * - * \since Version 1.22. - */ -struct OrtModelEditorApi { - // Model building/editing requires a full build. We return nullptr from GetModelEditorApi if this is a minimal - // build, so it doesn't matter if there are no function pointers in this struct as a user will never get an - // OrtModelEditorApi instance. We do however need a dummy field to avoid empty struct warning. -#if defined(ORT_MINIMAL_BUILD) - const bool not_defined_in_this_build; -#else - /** \brief Create an OrtTypeInfo instance for a Tensor. - * - * Create an OrtTypeInfo instance for a Tensor to use as graph inputs/outputs with the Model Editor API. - * - * User can release `tensor_info` after creating the OrtTypeInfo. - * - * \param[in] tensor_info Tensor type and shape information. - * \param[out] type_info TypeInfo instance for the tensor. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(CreateTensorTypeInfo, _In_ const OrtTensorTypeAndShapeInfo* tensor_info, - _Outptr_ OrtTypeInfo** type_info); - - /** \brief Create an OrtTypeInfo instance for a SparseTensor. - * - * Create an OrtTypeInfo instance for a SparseTensor to use as graph inputs/outputs with the Model Editor API. - * - * User can release `tensor_info` after creating the OrtTypeInfo. - * - * \param[in] tensor_info SparseTensor type and shape information. - * \param[out] type_info TypeInfo instance for the tensor. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(CreateSparseTensorTypeInfo, _In_ const OrtTensorTypeAndShapeInfo* tensor_info, - _Outptr_ OrtTypeInfo** type_info); - - /** \brief Create an OrtTypeInfo instance for a Map. - * - * Create an OrtTypeInfo instance for a Map to use as graph inputs/outputs with the Model Editor API. - * - * User can release `map_value_type` after creating the OrtTypeInfo. - * - * \param[in] map_key_type Key type for the map. - * \param[in] map_value_type Value type for the map. - * \param[out] type_info TypeInfo instance for the map. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(CreateMapTypeInfo, ONNXTensorElementDataType map_key_type, _In_ const OrtTypeInfo* map_value_type, - _Outptr_ OrtTypeInfo** type_info); - - /** \brief Create an OrtTypeInfo instance for a Sequence. - * - * Create an OrtTypeInfo instance for a Sequence to use as graph inputs/outputs with the Model Editor API. - * - * User can release `sequence_type` after creating the OrtTypeInfo. - * - * \param[in] sequence_type Sequence type and shape information. - * \param[out] type_info TypeInfo instance for the sequence. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(CreateSequenceTypeInfo, _In_ const OrtTypeInfo* sequence_type, _Outptr_ OrtTypeInfo** type_info); - - /** \brief Create an OrtTypeInfo instance for an Optional. - * - * Create an OrtTypeInfo instance for an Optional to use as graph inputs/outputs with the Model Editor API. - * - * User can release `contained_type` after creating the OrtTypeInfo. - * - * \param[in] contained_type Tensor type and shape information. - * \param[out] type_info TypeInfo instance for the tensor. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(CreateOptionalTypeInfo, _In_ const OrtTypeInfo* contained_type, _Outptr_ OrtTypeInfo** type_info); - - /** \brief Create an OrtValueInfo for use as an OrtGraph input or output. - * - * \param[in] name The name of the input or output. - * \param[in] type_info The type information for the input or output. The provided value is copied. - * \param[out] value_info The OrtValueInfo instance. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(CreateValueInfo, _In_ const char* name, _In_ const OrtTypeInfo* type_info, - _Outptr_ OrtValueInfo** value_info); - - /** \brief Create an OrtNode to add to an OrtGraph. - * - * Create an OrtNode. - * - * Create attributes with CreateOpAttr. OrtOpAttr instances are copied. - * - * \param[in] operator_name The name of the operator. - * \param[in] domain_name The domain of the operator. Use an empty string for ONNX operators. - * \param[in] node_name The name of the node. - * \param[in] input_names The names of the inputs. - * \param[in] input_names_len The number of input names. - * \param[in] output_names The names of the outputs. - * \param[in] output_names_len The number of output names. - * \param[in] attributes The optional attributes of the node. - * \param[in] attribs_len The number of attributes. May be zero. - * \param[out] node The OrtNode instance. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(CreateNode, _In_ const char* operator_name, _In_ const char* domain_name, _In_ const char* node_name, - _In_reads_(input_names_len) const char* const* input_names, size_t input_names_len, - _In_reads_(output_names_len) const char* const* output_names, size_t output_names_len, - _In_reads_(attribs_len) _In_opt_ OrtOpAttr** attributes, _In_ size_t attribs_len, - _Outptr_ OrtNode** node); - - /** \brief Create an OrtGraph - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.22. - */ - ORT_API2_STATUS(CreateGraph, _Outptr_ OrtGraph** graph); - - /** \brief Set the inputs for the OrtGraph. - * - * Set the graph inputs. This will replace any existing inputs with the new values. - * The OrtGraph takes ownership of the OrtValueInfo instances and you should NOT call ReleaseOrtValueInfo. - * - * \param[in] graph The OrtGraph instance to update. - * \param[in] inputs The input OrtValueInfo instances. - * \param[in] inputs_len The number of input OrtValueInfo instances. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(SetGraphInputs, _Inout_ OrtGraph* graph, - _In_reads_(inputs_len) _In_ OrtValueInfo** inputs, _In_ size_t inputs_len); - - /** \brief Set the outputs for the OrtGraph. - * - * Set the graph outputs. This will replace any existing outputs with the new values. - * The OrtGraph takes ownership of the OrtValueInfo instances provided and you should NOT call ReleaseOrtValueInfo. - * - * \param[in] graph The OrtGraph instance to update. - * \param[in] outputs The output OrtValueInfo instances. - * \param[in] outputs_len The number of output OrtValueInfo instances. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(SetGraphOutputs, _Inout_ OrtGraph* graph, - _In_reads_(outputs_len) _In_ OrtValueInfo** outputs, _In_ size_t outputs_len); - - /** \brief Add an initializer to the OrtGraph - * - * ORT will take ownership of the OrtValue and you should NOT call ReleaseOrtValue. - * - * Two options: - * - * Allocated memory: - * Use CreateTensorAsOrtValue (allocates memory) and populate the tensor with the data. - * Set `data_is_external` to false. - * - * Pre-existing memory: - * Use CreateTensorWithDataAsOrtValue or CreateTensorWithDataAndDeleterAsOrtValue to create an OrtValue - * with a tensor that contains a pointer to the existing data. - * Set `data_is_external` to true. - * - * The pointer must remain valid for the duration of the inference session. - * If using CreateTensorWithDataAsOrtValue you are responsible for freeing the memory after the inference session - * is released. - * If using CreateTensorWithDataAndDeleterAsOrtValue, ORT will free the memory using the provided deleter as - * soon as the OrtValue is no longer in use. - * - * NOTE: A tensor containing pre-existing memory MUST have 128 bytes of data or more. - * For smaller tensors use CreateTensorAsOrtValue. - * - * ONNX shape inferencing does not support external data. An initializer involved in shape inferencing is - * typically small (a single value or limited by the rank of a tensor) and uses less than 128 bytes of - * memory, so this limit acts as a simple catch-all rule to avoid issues. - * e.g. Reshape's `shape`, Clip's `min` and `max`, various ops `axes`. - * - * \param[in] graph The OrtGraph instance to update. - * \param[in] name The value name for the initializer. - * \param[in] tensor The OrtValue instance containing the tensor data. - * \param[in] data_is_external Set to true if the data is external and should not be copied. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(AddInitializerToGraph, _Inout_ OrtGraph* graph, _In_ const char* name, _In_ OrtValue* tensor, - bool data_is_external); - - /** \brief Add an OrtNode to an OrtGraph - * - * Add the node to the graph. The OrtGraph will take ownership of OrtNode and you should NOT call ReleaseOrtNode. - * - * \param[in] graph The OrtGraph instance to update. - * \param[in] node The OrtNode instance to add to the graph. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(AddNodeToGraph, _Inout_ OrtGraph* graph, _In_ OrtNode* node); - - /** \brief Create an OrtModel. - * - * Create an OrtModel. - * - * This can be used to build a new model, or to augment an existing model. - * - * \param[in] domain_names The domain names for the model. - * If augmenting an existing model add additional domains if needed. - * \param[in] opset_versions The opset versions for the model. - * If augmenting an existing model add additional opset versions if needed. - * \param[in] opset_entries_len The number of domain_names and opset_versions entries. - * Domain and opset entries should be 1:1 - * \param[out] model The OrtModel instance. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(CreateModel, - _In_reads_(opset_entries_len) const char* const* domain_names, - _In_reads_(opset_entries_len) const int* opset_versions, - size_t opset_entries_len, - _Outptr_ OrtModel** model); - - /** \brief Add an OrtGraph to an OrtModel. - * - * Add the graph to a model. This should be called once when creating a new model. - * - * The OrtModel takes ownership of the OrtGraph and you should NOT call ReleaseOrtGraph. - * - * \param[in] model The OrtModel instance to update. - * \param[in] graph The OrtGraph instance to add to the model. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(AddGraphToModel, _Inout_ OrtModel* model, _In_ OrtGraph* graph); - - /** \brief Create an OrtSession using the OrtModel. - * - * Create an inference session using the OrtModel instance. - * The OrtModel should have been populated with an OrtGraph containing nodes and initializers, and SetGraphInputs - * and SetGraphOutputs must have been called. - * This will validate the model, run optimizers, and prepare the session for inferencing. - * - * ReleaseOrtModel must be called to free the OrtModel after session creation. - * - * \param[in] env The OrtEnv instance. - * \param[in] model The OrtModel instance. - * \param[in] options The OrtSessionOptions instance. - * \param[out] out The OrtSession instance. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(CreateSessionFromModel, _In_ const OrtEnv* env, _In_ const OrtModel* model, - _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out); - - /** \brief Create an OrtSession to augment an existing model. - * - * Create an OrtSession with an existing model that will be augmented with additional nodes and initializers. - * Nodes can be added before or after the existing nodes in the model. ONNX Runtime will connect the nodes when the - * model is finalized. - * - * To add nodes and initializers to the existing model, first create an OrtModel using CreateModel. - * Add nodes and initializers to the OrtModel using AddNodeToGraph and AddInitializerToGraph. - * Graph inputs/outputs should be updated with SetGraphInputs and SetGraphOutputs as needed to reflect changes made - * by the new nodes. The list of graph inputs/outputs should be for the overall model and not just the new nodes. - * - * Add the new information from the OrtModel to the original model using ApplyModelToSession, and prepare the - * session for inferencing by calling FinalizeModelEditorSession. - * - * \param{in} env The OrtEnv instance. - * \param{in} model_path The path to the existing ONNX model to augment. - * \param{in} options The OrtSessionOptions instance. - * \param{out} out The created OrtSession instance. - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(CreateModelEditorSession, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path, - _In_ const OrtSessionOptions* options, - _Outptr_ OrtSession** out); - - /** \brief Create an OrtSession to augment an existing model. - * - * Create an OrtSession with an existing model that will be augmented with additional nodes and initializers. - * Nodes can be added before or after the existing nodes in the model. ONNX Runtime will connect the nodes when the - * model is finalized. - * - * To add nodes and initializers to the existing model, first create an OrtModel using CreateModel. - * Add nodes and initializers to the OrtModel using AddNodeToGraph and AddInitializerToGraph. - * Graph inputs/outputs should be updated with SetGraphInputs and SetGraphOutputs as needed to reflect changes made - * by the new nodes. The list of graph inputs/outputs should be for the overall model and not just the new nodes. - * - * Add the new information from the OrtModel to the original model using ApplyModelToSession, and prepare the - * session for inferencing by calling FinalizeModelEditorSession. - * - * \param{in} env The OrtEnv instance. - * \param{in} model_data The model data for the existing model to augment. - * \param{in} model_data_length The length of the model data. - * \param{in} options The OrtSessionOptions instance. - * \param{out} out The created OrtSession instance. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(CreateModelEditorSessionFromArray, _In_ const OrtEnv* env, - _In_ const void* model_data, size_t model_data_length, - _In_ const OrtSessionOptions* options, - _Outptr_ OrtSession** out); - - /** \brief Query the session for the opset version of a domain. - * - * When using the Model Editor API to augment a model, any new nodes must conform to the opset version of the - * original model. To do that the user must be able to discover that opset version. - * Returns an error if the domain is not used in the model. - * - * \param[in] session OrtSession to query - * \param[in] domain Domain to query. The ONNX domain is an empty string. - * \param[out] opset The opset version of the domain. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(SessionGetOpsetForDomain, _In_ const OrtSession* session, _In_ const char* domain, _Out_ int* opset); - - /** \brief Apply changes to augment the ONNX model in a session created using CreateModelEditorSession[FromArray] - * - * Adds new nodes and updates graph inputs/outputs using `model` to augment the original ONNX model in the session. - * All changes will be validated. - * Call FinalizeModelEditorSession to prepare the session for inferencing. - * - * Existing input/outputs will only be updated if the OrtGraph inputs/outputs are set in the OrtModel. - * i.e. you don't need to call SetGraphInputs/SetGraphOutputs if they are unchanged. - * - * ReleaseOrtModel must be called to free the OrtModel after it is applied to the session. - * - * \param[in] session OrtSession to update. Session must have been created using CreateModelEditorSession[FromArray]. - * \param[in] model OrtModel containing new nodes, new initializers, and updated graph input and/or output info. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(ApplyModelToModelEditorSession, _Inout_ OrtSession* session, _In_ OrtModel* model); - - /** \brief Finalize the Model Editor session that was created using CreateModelEditorSession[FromArray]. - * - * Finalize the Model Editor session that augmented an ONNX model by adding new nodes. - * This will run optimizers and prepare the session for inferencing. - * - * \param[in] session OrtSession to finalize. Session must have been created using CreateModelEditorSession[FromArray]. - * \param[in] options OrtSessionOptions to use for the session. - * \param[in] prepacked_weights_container Optional OrtPrepackedWeightsContainer to use for the session. - Set to nullptr if not used. - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(FinalizeModelEditorSession, _Inout_ OrtSession* session, _In_ const OrtSessionOptions* options, - _In_opt_ OrtPrepackedWeightsContainer* prepacked_weights_container); -#endif // !defined(ORT_MINIMAL_BUILD) -}; - -/** - * ORT Compile API - */ - -/** \brief Flags representing options to enable when compiling a model. - */ -typedef enum OrtCompileApiFlags { - // Default. Do not enable any additional compilation options. - OrtCompileApiFlags_NONE = 0, - - // Force compilation to return an error (ORT_FAIL) if no nodes were compiled. - // Otherwise, a model with basic optimizations (ORT_ENABLE_BASIC) is still generated by default. - OrtCompileApiFlags_ERROR_IF_NO_NODES_COMPILED = 1 << 0, - - // Force compilation to return an error (ORT_FAIL) if a file with the same filename as the output model exists. - // Otherwise, compilation will automatically overwrite the output file if it exists. - OrtCompileApiFlags_ERROR_IF_OUTPUT_FILE_EXISTS = 1 << 1, -} OrtCompileApiFlags; - -/** - * \brief The OrtCompileApi struct provides functions to compile ONNX models. - * - * Execution providers that support compilation fuse a subgraph into an EPContext node that wraps a provider-specific - * binary representation of the subgraph. - * For more details about the EPContext design, refer to: - * \htmlonly - * EPContext design document. - * \endhtmlonly - * - * Example (error handling not shown): - * OrtStatus* status = NULL; - * OrtCompileApi* compile_api = ort_api->GetCompileApi(); - * OrtModelCompilationOptions* compile_options = NULL; - * - * status = compile_api->CreateModelCompilationOptionsFromSessionOptions(env, session_options, &compile_options); - * status = compile_api->ModelCompilationOptions_SetInputModelPath(compile_options, ORT_TSTR("model.onnx")); - * status = compile_api->ModelCompilationOptions_SetOutputModelPath(compile_options, ORT_TSTR("model.compiled.onnx")); - * status = compile_api->CompileModel(env, compile_options); - * compile_api->ReleaseModelCompilationOptions(compile_options); - * - * \since Version 1.22. - */ -struct OrtCompileApi { - /// \name OrtModelCompilationOptions - /// @{ - ORT_CLASS_RELEASE(ModelCompilationOptions); - - /** \brief Creates an OrtModelCompilationOptions object from an existing OrtSessionOptions object. - * - * An OrtModelCompilationOptions object contains the settings used to generate a compiled ONNX model. - * The OrtSessionOptions object has the execution providers with which the model will be compiled. - * - * ReleaseOrtModelCompilationsOptions must be called to free the OrtModelCompilationOptions after calling - * CompileModel. - * - * \note By default, the GraphOptimizationLevel is set to ORT_DISABLE_ALL. Use - * ModelCompilationOptions_SetGraphOptimizationLevel to enable graph optimizations. - * - * \param[in] env OrtEnv object. - * \param[in] session_options The OrtSessionOptions instance from which to create the OrtModelCompilationOptions. - * \param[out] out The created OrtModelCompilationOptions instance. - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(CreateModelCompilationOptionsFromSessionOptions, _In_ const OrtEnv* env, - _In_ const OrtSessionOptions* session_options, _Outptr_ OrtModelCompilationOptions** out); - - /** \brief Sets the file path to the input ONNX model to compile. - * - * The input model's location (e.g., file path or memory buffer) must be set with either - * ModelCompilationOptions_SetInputModelPath or ModelCompilationOptions_SetInputModelFromBuffer. - * - * \param[in] model_compile_options The OrtModelCompilationOptions instance. - * \param[in] input_model_path Null terminated string of the path (wchar on Windows, char otherwise). - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(ModelCompilationOptions_SetInputModelPath, _In_ OrtModelCompilationOptions* model_compile_options, - _In_ const ORTCHAR_T* input_model_path); - - /** \brief Sets the buffer that stores the bytes of the loaded ONNX model to compile. - * - * The input model's location (e.g., file path or memory buffer) must be set with either - * ModelCompilationOptions_SetInputModelPath or ModelCompilationOptions_SetInputModelFromBuffer. - * - * \param[in] model_compile_options The OrtModelCompilationOptions instance. - * \param[in] input_model_data Buffer containing the loaded ONNX model bytes. - * \param[in] input_model_data_size The number of bytes in the `input_model_data` buffer. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(ModelCompilationOptions_SetInputModelFromBuffer, - _In_ OrtModelCompilationOptions* model_compile_options, - _In_ const void* input_model_data, - size_t input_model_data_size); - - /** \brief Sets the file path for the output ONNX model generated by CompileModel. - * - * The output model's location (e.g., file path or memory buffer) can be set with either - * ModelCompilationOptions_SetOutputModelPath or ModelCompilationOptions_SetOutputModelBuffer. - * - * If the output model's location is not set, ONNX Runtime will generate an output file with a path based on - * the input model's file path. Examples: - * /Path/my_model.onnx -> /Path/my_model_ctx.onnx - * /Path/my_model -> /Path/my_model_ctx.onnx - * - * \param[in] model_compile_options The OrtModelCompilationOptions instance. - * \param[in] output_model_path Null terminated string of the path (wchar on Windows, char otherwise). - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(ModelCompilationOptions_SetOutputModelPath, _In_ OrtModelCompilationOptions* model_compile_options, - _In_ const ORTCHAR_T* output_model_path); - - /** \brief Optionally sets the file that should store external initializers for the compiled ONNX model. - * If not set, initializers are stored within the model. - * - * Only initializers for nodes that were not compiled are stored in the external initializers file. - * Compiled nodes contain their initializer data within the `ep_cache_context` attribute of EPContext nodes. - * Refer to ModelCompilationOptions_SetEpContextEmbedMode. - * - * \param[in] model_compile_options The OrtModelCompilationOptions instance. - * \param[in] external_initializers_file_path Null terminated string of the path to the file. - * \param[in] external_initializers_size_threshold Initializers larger than this threshold are stored in the file. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(ModelCompilationOptions_SetOutputModelExternalInitializersFile, - _In_ OrtModelCompilationOptions* model_compile_options, - _In_ const ORTCHAR_T* external_initializers_file_path, - size_t external_initializers_size_threshold); - - /** \brief Configures model compilation to store the output compiled ONNX model in a buffer. - * - * The caller passes an OrtAllocator that ONNX Runtime uses to allocate memory for the buffer. - * - * The output model's location (e.g., file path or memory buffer) can be set with either - * ModelCompilationOptions_SetOutputModelPath or ModelCompilationOptions_SetOutputModelBuffer. - * - * If the output model's location is not set, ONNX Runtime will generate an output file with a path based on - * the input model's file path. Examples: - * /Path/my_model.onnx -> /Path/my_model_ctx.onnx - * /Path/my_model -> /Path/my_model_ctx.onnx - * - * \param[in] model_compile_options The OrtModelCompilationOptions instance. - * \param[in] allocator The allocator used to allocate the buffer for the compiled model. - * \param[out] output_model_buffer_ptr Pointer to the buffer that stores the compiled model. - * \param[out] output_model_buffer_size_ptr Pointer set to the size of output model in bytes. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(ModelCompilationOptions_SetOutputModelBuffer, - _In_ OrtModelCompilationOptions* model_compile_options, - _Inout_ OrtAllocator* allocator, - _Outptr_ void** output_model_buffer_ptr, - _Out_ size_t* output_model_buffer_size_ptr); - - /** \brief Enables or disables the embedding of EPContext binary data into the `ep_cache_context` attribute - * of EPContext nodes. Defaults to false. - * - * If enabled, the `ep_cache_context` attribute of EPContext nodes will store the context binary data, which may - * include weights for compiled subgraphs. - * - * If disabled, the `ep_cache_context` attribute of EPContext nodes will contain the path to the file containing the - * context binary data. The path is set by the execution provider creating the EPContext node. - * - * More details relate to EPContext design refers to: - * \htmlonly - * EPContext design document. - * \endhtmlonly - * - * \param[in] model_compile_options The OrtModelCompilationOptions instance. - * \param[in] embed_ep_context_in_model True to embed EPContext binary data into the EPContext node - * `ep_cache_context` attributes. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(ModelCompilationOptions_SetEpContextEmbedMode, _In_ OrtModelCompilationOptions* model_compile_options, - bool embed_ep_context_in_model); - - /** \brief Compiles an input ONNX model with the given compilation options. - * - * \param[in] env OrtEnv object. - * \param[in] model_options The compilation options that defines compilation options for a model. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(CompileModel, _In_ const OrtEnv* env, _In_ const OrtModelCompilationOptions* model_options); - - /** \brief Sets flags from OrtCompileApiFlags that represent one or more boolean options to enable. - * - * \param[in] model_compile_options The OrtModelCompilationOptions instance. - * \param[in] flags bitwise OR of flags in OrtCompileApiFlags to enable. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(ModelCompilationOptions_SetFlags, _In_ OrtModelCompilationOptions* model_compile_options, - uint32_t flags); - - /** Sets information related to EP context binary file. - * - * EP uses this information to decide the location and context binary file name. - * Used while compiling model with input and output in memory buffer - * - * \param[in] model_compile_options The OrtModelCompilationOptions instance. - * \param[in] output_directory Null terminated string of the path (wchar on Windows, char otherwise). - * \param[in] model_name Null terminated string of the model name (wchar on Windows, char otherwise). - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(ModelCompilationOptions_SetEpContextBinaryInformation, - _In_ OrtModelCompilationOptions* model_compile_options, - _In_ const ORTCHAR_T* output_directory, - _In_ const ORTCHAR_T* model_name); - - /** Set the graph optimization level. - * - * \param[in] model_compile_options The OrtModelCompilationOptions instance. - * \param[in] graph_optimization_level The graph optimization level. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(ModelCompilationOptions_SetGraphOptimizationLevel, - _In_ OrtModelCompilationOptions* model_compile_options, - _In_ GraphOptimizationLevel graph_optimization_level); - - /** \brief Sets a OrtWriteBufferFunc function that is called by ORT to write out the output model's serialized - * ONNX bytes. - * - * The provided write function may be called repeatedly until then entire output model has been written out. Each call - * to the write function is expected to consume the entire input buffer. - * - * The output model's destination (e.g., file path, memory buffer, or stream) can be set with any of the functions - * that begin with ModelCompilationOptions_SetOutputModel____. - * - * \param[in] model_compile_options The OrtModelCompilationOptions instance. - * \param[in] write_func The OrtWriteBufferFunc function called by ORT when writing out the model. - * \param[in] state Opaque state passed as the first argument to OrtWriteBufferFunc. Can be NULL. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(ModelCompilationOptions_SetOutputModelWriteFunc, - _In_ OrtModelCompilationOptions* model_compile_options, - _In_ OrtWriteBufferFunc write_func, _In_ void* state); - - /** \brief Sets a OrtGetInitializerLocationFunc function that is called by ORT for every initializer in the generated - * model. Allows implementer to specify whether initializers should be stored within the model or externally. - * - * \param[in] model_compile_options The OrtModelCompilationOptions instance. - * \param[in] get_initializer_location_func The OrtGetInitializerLocationFunc function called by ORT when - * to determine the location of the initializer. - * \param[in] state Opaque state passed as the first argument to OrtGetInitializerLocationFunc. Can be NULL. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(ModelCompilationOptions_SetOutputModelGetInitializerLocationFunc, - _In_ OrtModelCompilationOptions* model_compile_options, - _In_ OrtGetInitializerLocationFunc get_initializer_location_func, _In_ void* state); -}; - /* * This is the old way to add the CUDA provider to the session, please use SessionOptionsAppendExecutionProvider_CUDA above to access the latest functionality * This function always exists, but will only succeed if Onnxruntime was built with CUDA support and the CUDA provider shared library exists @@ -7425,5 +4820,3 @@ ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Tensorrt, _In_ OrtSessio } #endif /// @} - -#include "onnxruntime_ep_c_api.h" diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_cxx_api.h b/prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_cxx_api.h similarity index 73% rename from prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_cxx_api.h rename to prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_cxx_api.h index 879c232..8091fd4 100644 --- a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_cxx_api.h +++ b/prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_cxx_api.h @@ -26,17 +26,16 @@ #include "onnxruntime_c_api.h" #include "onnxruntime_float16.h" -#include #include #include +#include #include #include #include -#include +#include #include #include -#include -#include +#include #ifdef ORT_NO_EXCEPTIONS #include @@ -52,7 +51,6 @@ namespace Ort { * If ORT_NO_EXCEPTIONS is defined, then any error will result in a call to abort() */ struct Exception : std::exception { - Exception(const std::string& string, OrtErrorCode code) : message_{string}, code_{code} {} Exception(std::string&& string, OrtErrorCode code) : message_{std::move(string)}, code_{code} {} OrtErrorCode GetOrtErrorCode() const { return code_; } @@ -122,7 +120,7 @@ const OrtApi* Global::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION); #endif #endif -/// This returns a reference to the ORT C API. +/// This returns a reference to the OrtApi interface in use inline const OrtApi& GetApi() noexcept { return *Global::api_; } /// @@ -145,48 +143,6 @@ std::string GetBuildInfoString(); /// vector of strings std::vector GetAvailableProviders(); -/// -/// This returns a reference to the ORT C Model Editor API. Used if building or augmenting a model at runtime. -/// -/// ORT C Model Editor API reference -inline const OrtModelEditorApi& GetModelEditorApi() { - auto* api = GetApi().GetModelEditorApi(); - if (api == nullptr) { - // minimal build - ORT_CXX_API_THROW("Model Editor API is not available in this build", ORT_FAIL); - } - - return *api; -} - -/// -/// This returns a reference to the ORT C Compile API. Used if compiling a model at runtime. -/// -/// ORT C Compile API reference -inline const OrtCompileApi& GetCompileApi() { - auto* api = GetApi().GetCompileApi(); - if (api == nullptr) { - // minimal build - ORT_CXX_API_THROW("Compile API is not available in this build", ORT_FAIL); - } - - return *api; -} - -/// -/// This returns a reference to the ORT C EP API. Used if authoring a plugin execution provider. -/// -/// ORT C EP API reference -inline const OrtEpApi& GetEpApi() { - auto* api = GetApi().GetEpApi(); - if (api == nullptr) { - // minimal build - ORT_CXX_API_THROW("EP API is not available in this build", ORT_FAIL); - } - - return *api; -} - /** \brief IEEE 754 half-precision floating point data type * * \details This struct is used for converting float to float16 and back @@ -196,7 +152,7 @@ inline const OrtEpApi& GetEpApi() { * uint16_t buffers to/from Ort::Float16_t to feed and retrieve data. * * \code{.unparsed} - * // This example demonstrates conversion from float to float16 + * // This example demonstrates converion from float to float16 * constexpr float values[] = {1.f, 2.f, 3.f, 4.f, 5.f}; * std::vector fp16_values; * fp16_values.reserve(std::size(values)); @@ -338,7 +294,7 @@ static_assert(sizeof(Float16_t) == sizeof(uint16_t), "Sizes must match"); * uint16_t buffers to/from Ort::BFloat16_t to feed and retrieve data. * * \code{.unparsed} - * // This example demonstrates conversion from float to float16 + * // This example demonstrates converion from float to float16 * constexpr float values[] = {1.f, 2.f, 3.f, 4.f, 5.f}; * std::vector bfp16_values; * bfp16_values.reserve(std::size(values)); @@ -546,49 +502,28 @@ namespace detail { #define ORT_DEFINE_RELEASE(NAME) \ inline void OrtRelease(Ort##NAME* ptr) { GetApi().Release##NAME(ptr); } -#define ORT_DEFINE_RELEASE_FROM_API_STRUCT(NAME, API_GETTER) \ - inline void OrtRelease(Ort##NAME* ptr) { API_GETTER().Release##NAME(ptr); } - ORT_DEFINE_RELEASE(Allocator); -ORT_DEFINE_RELEASE(ArenaCfg); -ORT_DEFINE_RELEASE(CustomOpDomain); -ORT_DEFINE_RELEASE(Env); -ORT_DEFINE_RELEASE(ExternalInitializerInfo); -ORT_DEFINE_RELEASE(Graph); -ORT_DEFINE_RELEASE(IoBinding); -ORT_DEFINE_RELEASE(KernelInfo); -ORT_DEFINE_RELEASE(KeyValuePairs); -ORT_DEFINE_RELEASE(LoraAdapter); ORT_DEFINE_RELEASE(MemoryInfo); -ORT_DEFINE_RELEASE(MapTypeInfo); -ORT_DEFINE_RELEASE(Model); -ORT_DEFINE_RELEASE(ModelMetadata); -ORT_DEFINE_RELEASE(Node); -ORT_DEFINE_RELEASE(Op); -ORT_DEFINE_RELEASE(OpAttr); -ORT_DEFINE_RELEASE(PrepackedWeightsContainer); +ORT_DEFINE_RELEASE(CustomOpDomain); +ORT_DEFINE_RELEASE(ThreadingOptions); +ORT_DEFINE_RELEASE(Env); ORT_DEFINE_RELEASE(RunOptions); ORT_DEFINE_RELEASE(Session); ORT_DEFINE_RELEASE(SessionOptions); -ORT_DEFINE_RELEASE(SequenceTypeInfo); -ORT_DEFINE_RELEASE(Status); -ORT_DEFINE_RELEASE(SyncStream); ORT_DEFINE_RELEASE(TensorTypeAndShapeInfo); -ORT_DEFINE_RELEASE(ThreadingOptions); +ORT_DEFINE_RELEASE(SequenceTypeInfo); +ORT_DEFINE_RELEASE(MapTypeInfo); ORT_DEFINE_RELEASE(TypeInfo); ORT_DEFINE_RELEASE(Value); -ORT_DEFINE_RELEASE(ValueInfo); - -ORT_DEFINE_RELEASE_FROM_API_STRUCT(ModelCompilationOptions, GetCompileApi); -ORT_DEFINE_RELEASE_FROM_API_STRUCT(EpDevice, GetEpApi); - -// This is defined explicitly since OrtTensorRTProviderOptionsV2 is not a C API type, -// but the struct has V2 in its name to indicate that it is the second version of the options. -inline void OrtRelease(OrtTensorRTProviderOptionsV2* ptr) { GetApi().ReleaseTensorRTProviderOptions(ptr); } -inline void OrtRelease(OrtCUDAProviderOptionsV2* ptr) { GetApi().ReleaseCUDAProviderOptions(ptr); } +ORT_DEFINE_RELEASE(ModelMetadata); +ORT_DEFINE_RELEASE(IoBinding); +ORT_DEFINE_RELEASE(ArenaCfg); +ORT_DEFINE_RELEASE(Status); +ORT_DEFINE_RELEASE(OpAttr); +ORT_DEFINE_RELEASE(Op); +ORT_DEFINE_RELEASE(KernelInfo); #undef ORT_DEFINE_RELEASE -#undef ORT_DEFINE_RELEASE_FROM_API_STRUCT /** \brief This is a tagging template type. Use it with Base to indicate that the C++ interface object * has no ownership of the underlying C object. @@ -623,9 +558,7 @@ struct Base { constexpr Base() = default; constexpr explicit Base(contained_type* p) noexcept : p_{p} {} - ~Base() { - OrtRelease(p_); - } + ~Base() { OrtRelease(p_); } Base(const Base&) = delete; Base& operator=(const Base&) = delete; @@ -638,7 +571,6 @@ struct Base { } constexpr operator contained_type*() const noexcept { return p_; } - constexpr contained_type& operator*() const noexcept { return *p_; } /// \brief Relinquishes ownership of the contained C object pointer /// The underlying object is not destroyed @@ -683,7 +615,6 @@ struct Base> { } constexpr operator contained_type*() const noexcept { return p_; } - constexpr contained_type& operator*() const noexcept { return *p_; } protected: contained_type* p_{}; @@ -703,20 +634,9 @@ struct AllocatedFree { struct AllocatorWithDefaultOptions; struct Env; -struct EpDevice; -struct ExternalInitializerInfo; -struct Graph; -struct Model; -struct Node; -struct ModelMetadata; struct TypeInfo; -struct PrepackedWeightsContainer; -struct Session; -struct SessionOptions; -struct SyncStream; -struct TensorRTProviderOptions; struct Value; -struct ValueInfo; +struct ModelMetadata; /** \brief unique_ptr typedef used to own strings allocated by OrtAllocators * and release them at the end of the scope. The lifespan of the given allocator @@ -729,12 +649,11 @@ using AllocatedStringPtr = std::unique_ptr; * constructors to construct an instance of a Status object from exceptions. */ struct Status : detail::Base { - Status() = default; // Same as with std::nullptr_t. But can be used in re-sizable containers and represent success. - explicit Status(std::nullptr_t) noexcept {} ///< Create an empty object, must be assigned a valid one to be used - explicit Status(OrtStatus* status) noexcept; ///< Takes ownership of OrtStatus instance returned from the C API. - explicit Status(const Exception&); ///< Creates status instance out of exception - explicit Status(const std::exception&); ///< Creates status instance out of exception - Status(const char* message, OrtErrorCode code); ///< Creates status instance out of null-terminated string message. + explicit Status(std::nullptr_t) noexcept {} ///< Create an empty object, must be assigned a valid one to be used + explicit Status(OrtStatus* status) noexcept; ///< Takes ownership of OrtStatus instance returned from the C API. + explicit Status(const Exception&) noexcept; ///< Creates status instance out of exception + explicit Status(const std::exception&) noexcept; ///< Creates status instance out of exception + Status(const char* message, OrtErrorCode code) noexcept; ///< Creates status instance out of null-terminated string message. std::string GetErrorMessage() const; OrtErrorCode GetErrorCode() const; bool IsOK() const noexcept; ///< Returns true if instance represents an OK (non-error) status. @@ -770,317 +689,6 @@ struct ThreadingOptions : detail::Base { ThreadingOptions& SetGlobalCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn); }; -/** \brief The TensorRTOptions (V2) - * - * Used to pass options to TRT EP - */ -struct TensorRTProviderOptions : detail::Base { - TensorRTProviderOptions(std::nullptr_t) {} - /// \brief Wraps OrtApi::CreateTensorRTProviderOptionsV2 - TensorRTProviderOptions(); - ///< Wrapper around OrtApi::UpdateTensorRTProviderOptions - void Update(const std::unordered_map& options); - ///< Wrapper around OrtApi::UpdateTensorRTProviderOptions - void UpdateWithValue(const char* key, void* value); - - ///< Wrapper around OrtApi::GetTensorRTProviderOptionsByName - void* GetOptionByName(const char* name) const; - ///< Wrapper around OrtApi::GetTensorRTProviderOptionsAsString - std::string GetTensorRTProviderOptionsAsString() const; -}; - -/** \brief The CUDAProviderOptions (V2) - * - * Used to pass options to CUDA EP - */ -struct CUDAProviderOptions : detail::Base { - CUDAProviderOptions(std::nullptr_t) {} - /// \brief Wraps OrtApi::CreateCUDAProviderOptions - CUDAProviderOptions(); - ///< Wrapper around OrtApi::UpdateCUDAProviderOptions - void Update(const std::unordered_map& options); - ///< Wrapper around OrtApi::GetCUDAProviderOptionsAsString - std::string GetCUDAProviderOptionsAsString() const; - ///< Wrapper around OrtApi::UpdateCUDAProviderOptionsWithValue - void UpdateWithValue(const char* key, void* value); - ///< Wrapper around OrtApi::GetCUDAProviderOptionsByName - void* GetOptionByName(const char* name) const; -}; - -/** \brief The PrepackedWeightsContainer - * - * Create only and pass to Ort::Session constructor for multiple sessions - * to share pre-packed weights. - */ -struct PrepackedWeightsContainer : detail::Base { - using Base = detail::Base; - ///< No instance is created - explicit PrepackedWeightsContainer(std::nullptr_t) {} - ///< Take ownership of a pointer created by C API - explicit PrepackedWeightsContainer(OrtPrepackedWeightsContainer* p) : Base{p} {} - /// \brief Wraps OrtApi::CreatePrepackedWeightsContainer - PrepackedWeightsContainer(); -}; - -namespace detail { -template -struct ConstExternalInitializerInfoImpl : Base { - using B = Base; - using B::B; - - // Wraps OrtApi::ExternalInitializerInfo_GetFilePath - const std::basic_string GetFilePath() const; - // Wraps OrtApi::ExternalInitializerInfo_GetFileOffset - int64_t GetFileOffset() const; - // Wraps OrtApi::ExternalInitializerInfo_GetByteSize - size_t GetByteSize() const; -}; -} // namespace detail - -// Const object holder that does not own the underlying object -using ConstExternalInitializerInfo = - detail::ConstExternalInitializerInfoImpl>; - -/** \brief Wrapper around ::OrtExternalInitializerInfo - * - */ -struct ExternalInitializerInfo : detail::ConstExternalInitializerInfoImpl { - using Base = detail::ConstExternalInitializerInfoImpl; - using Base::Base; - - explicit ExternalInitializerInfo(std::nullptr_t) {} - explicit ExternalInitializerInfo(OrtExternalInitializerInfo* p) - : detail::ConstExternalInitializerInfoImpl{p} {} - - ConstExternalInitializerInfo GetConst() const { return ConstExternalInitializerInfo{this->p_}; } - - ///< Wraps OrtApi::CreateExternalInitializerInfo - ExternalInitializerInfo(const ORTCHAR_T* filepath, int64_t file_offset, size_t byte_size); - - ///< Wrapper around CreateExternalInitializerInfo that does not throw an exception. - static Status Create(const ORTCHAR_T* filepath, int64_t file_offset, size_t byte_size, - /*out*/ ExternalInitializerInfo& out); -}; - -namespace detail { -template -struct KeyValuePairsImpl : Ort::detail::Base { - using B = Ort::detail::Base; - using B::B; - - const char* GetValue(const char* key) const; - - // get the pairs in unordered_map. needs to copy to std::string so the hash works as expected - std::unordered_map GetKeyValuePairs() const; - // get the pairs in two vectors. entries will be 1:1 between keys and values. avoids copying to std::string - void GetKeyValuePairs(std::vector& keys, std::vector& values) const; -}; -} // namespace detail - -// Const object holder that does not own the underlying object -using ConstKeyValuePairs = detail::KeyValuePairsImpl>; - -/** \brief Wrapper around ::OrtKeyValuePairs */ -struct KeyValuePairs : detail::KeyValuePairsImpl { - explicit KeyValuePairs(std::nullptr_t) {} ///< No instance is created - /// Take ownership of a pointer created by C API - explicit KeyValuePairs(OrtKeyValuePairs* p) : KeyValuePairsImpl{p} {} - - /// \brief Wraps OrtApi::CreateKeyValuePairs - explicit KeyValuePairs(); - - /// \brief Wraps OrtApi::CreateKeyValuePairs and OrtApi::AddKeyValuePair - explicit KeyValuePairs(const std::unordered_map& kv_pairs); - - /// \brief Wraps OrtApi::AddKeyValuePair - void Add(const char* key, const char* value); - - /// \brief Wraps OrtApi::RemoveKeyValuePair - void Remove(const char* key); - - ConstKeyValuePairs GetConst() const { return ConstKeyValuePairs{this->p_}; } -}; - -namespace detail { -template -struct MemoryInfoImpl : Base { - using B = Base; - using B::B; - - std::string GetAllocatorName() const; ///< Wrapper MemoryInfoGetName - OrtAllocatorType GetAllocatorType() const; ///< Wrapper MemoryInfoGetType - int GetDeviceId() const; ///< Wrapper MemoryInfoGetId - OrtMemoryInfoDeviceType GetDeviceType() const; ///< Wrapper MemoryInfoGetDeviceType - OrtMemType GetMemoryType() const; ///< Wrapper MemoryInfoGetMemType - OrtDeviceMemoryType GetDeviceMemoryType() const; ///< Wrapper MemoryInfoGetDeviceMemType - uint32_t GetVendorId() const; ///< Wrapper MemoryInfoGetVendorId - - template - bool operator==(const MemoryInfoImpl& o) const; -}; -} // namespace detail - -// Const object holder that does not own the underlying object -using ConstMemoryInfo = detail::MemoryInfoImpl>; - -/** \brief Wrapper around ::OrtMemoryInfo - * - */ -struct MemoryInfo : detail::MemoryInfoImpl { - static MemoryInfo CreateCpu(OrtAllocatorType type, OrtMemType mem_type1); - explicit MemoryInfo(std::nullptr_t) {} ///< No instance is created - explicit MemoryInfo(OrtMemoryInfo* p) : MemoryInfoImpl{p} {} ///< Take ownership of a pointer created by C API - MemoryInfo(const char* name, OrtAllocatorType type, int id, OrtMemType mem_type); - MemoryInfo(const char* name, OrtMemoryInfoDeviceType device_type, uint32_t vendor_id, uint32_t device_id, - OrtDeviceMemoryType mem_type, size_t alignment, OrtAllocatorType allocator_type); ///< Wrapper around CreateMemoryInfo_V2 - ConstMemoryInfo GetConst() const { return ConstMemoryInfo{this->p_}; } -}; - -/// -/// Represents native memory allocation coming from one of the -/// OrtAllocators registered with OnnxRuntime. -/// Use it to wrap an allocation made by an allocator -/// so it can be automatically released when no longer needed. -/// -struct MemoryAllocation { - MemoryAllocation(OrtAllocator* allocator, void* p, size_t size); - ~MemoryAllocation(); - MemoryAllocation(const MemoryAllocation&) = delete; - MemoryAllocation& operator=(const MemoryAllocation&) = delete; - MemoryAllocation(MemoryAllocation&&) noexcept; - MemoryAllocation& operator=(MemoryAllocation&&) noexcept; - - void* get() { return p_; } - size_t size() const { return size_; } - - private: - OrtAllocator* allocator_; - void* p_; - size_t size_; -}; - -namespace detail { -template -struct AllocatorImpl : Base { - using B = Base; - using B::B; - - void* Alloc(size_t size); - MemoryAllocation GetAllocation(size_t size); - void Free(void* p); - ConstMemoryInfo GetInfo() const; - - /** \brief Function that returns the statistics of the allocator. - * - * \return A pointer to a KeyValuePairs object that will be filled with the allocator statistics. - */ - KeyValuePairs GetStats() const; -}; -} // namespace detail - -/** \brief Wrapper around ::OrtAllocator default instance that is owned by Onnxruntime - * - */ -struct AllocatorWithDefaultOptions : detail::AllocatorImpl> { - explicit AllocatorWithDefaultOptions(std::nullptr_t) {} ///< Convenience to create a class member and then replace with an instance - AllocatorWithDefaultOptions(); -}; - -/** \brief Wrapper around ::OrtAllocator - * - */ - -struct Allocator : detail::AllocatorImpl { - explicit Allocator(std::nullptr_t) {} ///< Convenience to create a class member and then replace with an instance - Allocator(const Session& session, const OrtMemoryInfo*); -}; - -using UnownedAllocator = detail::AllocatorImpl>; - -/** \brief Wrapper around ::OrtSyncStream - * - */ - -namespace detail { -template -struct SyncStreamImpl : Base { - using B = Base; - using B::B; - // For some reason this is not a const method on the stream - void* GetHandle(); ///< Wraps SyncStream_GetHandle -}; -} // namespace detail - -struct SyncStream : detail::SyncStreamImpl { - ///< Create an empty SyncStream object, must be assigned a valid one to be used - explicit SyncStream(std::nullptr_t) {} - ///< Take ownership of a pointer created by C API - explicit SyncStream(OrtSyncStream* p) : SyncStreamImpl{p} {} -}; - -using UnownedSyncStream = detail::SyncStreamImpl>; - -namespace detail { -template -struct HardwareDeviceImpl : Ort::detail::Base { - using B = Ort::detail::Base; - using B::B; - - OrtHardwareDeviceType Type() const; - uint32_t VendorId() const; - uint32_t DeviceId() const; - const char* Vendor() const; - ConstKeyValuePairs Metadata() const; -}; -} // namespace detail - -/** \brief Wrapper around ::OrtHardwareDevice - * \remarks HardwareDevice is always read-only for API users. - */ -using ConstHardwareDevice = detail::HardwareDeviceImpl>; - -namespace detail { -template -struct EpDeviceImpl : Ort::detail::Base { - using B = Ort::detail::Base; - using B::B; - - const char* EpName() const; - const char* EpVendor() const; - ConstKeyValuePairs EpMetadata() const; - ConstKeyValuePairs EpOptions() const; - ConstHardwareDevice Device() const; - ConstMemoryInfo GetMemoryInfo(OrtDeviceMemoryType memory_type) const; ///< Wraps EpDevice_MemoryInfo - SyncStream CreateSyncStream(ConstKeyValuePairs stream_options = {}) const; /// Wraps EpDevice_CreateSyncStream -}; -} // namespace detail - -/** \brief Wrapper around ::OrtEpDevice - * \remarks EpDevice is always read-only for ORT API users. - */ -using ConstEpDevice = detail::EpDeviceImpl>; - -/** \brief Mutable EpDevice that is created by EpApi users. - */ -struct EpDevice : detail::EpDeviceImpl { - explicit EpDevice(std::nullptr_t) {} ///< No instance is created - explicit EpDevice(OrtEpDevice* p) : EpDeviceImpl{p} {} ///< Take ownership of a pointer created by C API - - /// \brief Wraps OrtEpApi::CreateEpDevice - EpDevice(OrtEpFactory& ep_factory, ConstHardwareDevice& hardware_device, - ConstKeyValuePairs ep_metadata = {}, ConstKeyValuePairs ep_options = {}); -}; - -/** \brief Validate a compiled model's compatibility for one or more EP devices. - * - * Throws on error. Returns the resulting compatibility status. - * /// \param ep_devices The EP devices to check compatibility against. - * /// \param compatibility_info The compatibility string from the precompiled model to validate. - */ -OrtCompiledModelCompatibility GetModelCompatibilityForEpDevices( - const std::vector& ep_devices, - const char* compatibility_info); - /** \brief The Env (Environment) * * The Env holds the logging state used by all other objects. @@ -1112,41 +720,13 @@ struct Env : detail::Base { Env& CreateAndRegisterAllocator(const OrtMemoryInfo* mem_info, const OrtArenaCfg* arena_cfg); ///< Wraps OrtApi::CreateAndRegisterAllocator - Env& CreateAndRegisterAllocatorV2(const std::string& provider_type, const OrtMemoryInfo* mem_info, - const std::unordered_map& options, - const OrtArenaCfg* arena_cfg); ///< Wraps OrtApi::CreateAndRegisterAllocatorV2 - - Env& RegisterAllocator(OrtAllocator* allocator); ///< Wraps OrtApi::RegisterAllocator - - Env& UnregisterAllocator(const OrtMemoryInfo* mem_info); ///< Wraps OrtApi::UnregisterAllocator - - UnownedAllocator CreateSharedAllocator(const OrtEpDevice* ep_device, OrtDeviceMemoryType mem_type, - OrtAllocatorType allocator_type, - const OrtKeyValuePairs* allocator_options); ///< Wraps OrtApi::CreateSharedAllocator - - // Result may be nullptr - UnownedAllocator GetSharedAllocator(const OrtMemoryInfo* mem_info); ///< Wraps OrtApi::GetSharedAllocator - - void ReleaseSharedAllocator(const OrtEpDevice* ep_device, - OrtDeviceMemoryType mem_type); ///< Wraps OrtApi::ReleaseSharedAllocator - - Env& RegisterExecutionProviderLibrary(const char* registration_name, const std::basic_string& path); ///< Wraps OrtApi::RegisterExecutionProviderLibrary - Env& UnregisterExecutionProviderLibrary(const char* registration_name); ///< Wraps OrtApi::UnregisterExecutionProviderLibrary - - std::vector GetEpDevices() const; - - Status CopyTensors(const std::vector& src_tensors, - const std::vector& dst_tensors, - OrtSyncStream* stream) const; ///< Wraps OrtApi::CopyTensors + Env& CreateAndRegisterAllocatorV2(const std::string& provider_type, const OrtMemoryInfo* mem_info, const std::unordered_map& options, const OrtArenaCfg* arena_cfg); ///< Wraps OrtApi::CreateAndRegisterAllocatorV2 }; /** \brief Custom Op Domain * */ struct CustomOpDomain : detail::Base { - using Base = detail::Base; - using Base::Base; - explicit CustomOpDomain(std::nullptr_t) {} ///< Create an empty CustomOpDomain object, must be assigned a valid one to be used /// \brief Wraps OrtApi::CreateCustomOpDomain @@ -1156,32 +736,6 @@ struct CustomOpDomain : detail::Base { void Add(const OrtCustomOp* op); ///< Wraps CustomOpDomain_Add }; -/// \brief LoraAdapter holds a set of Lora Parameters loaded from a single file -struct LoraAdapter : detail::Base { - using Base = detail::Base; - using Base::Base; - - explicit LoraAdapter(std::nullptr_t) {} ///< Create an empty LoraAdapter object, must be assigned a valid one to be used - /// \brief Wraps OrtApi::CreateLoraAdapter - /// - /// The function attempts to load the adapter from the specified file - /// \param adapter_path The path to the Lora adapter - /// \param allocator optional pointer to a device allocator. If nullptr, the data stays on CPU. It would still - /// be copied to device if required by the model at inference time. - static LoraAdapter CreateLoraAdapter(const std::basic_string& adapter_path, - OrtAllocator* allocator); - - /// \brief Wraps OrtApi::CreateLoraAdapterFromArray - /// - /// The function attempts to load the adapter from the specified byte array. - /// \param bytes The byte array containing file LoraAdapter format - /// \param num_bytes The number of bytes in the byte array - /// \param allocator optional pointer to a device allocator. If nullptr, the data stays on CPU. It would still - /// be copied to device if required by the model at inference time. - static LoraAdapter CreateLoraAdapterFromArray(const void* bytes, size_t num_bytes, - OrtAllocator* allocator); -}; - /** \brief RunOptions * */ @@ -1199,7 +753,6 @@ struct RunOptions : detail::Base { const char* GetRunTag() const; ///< Wraps OrtApi::RunOptionsGetRunTag RunOptions& AddConfigEntry(const char* config_key, const char* config_value); ///< Wraps OrtApi::AddRunConfigEntry - const char* GetConfigEntry(const char* config_key); ///< Wraps OrtApi::GetRunConfigEntry /** \brief Terminates all currently executing Session::Run calls that were made using this RunOptions instance * @@ -1213,14 +766,6 @@ struct RunOptions : detail::Base { * Wraps OrtApi::RunOptionsUnsetTerminate */ RunOptions& UnsetTerminate(); - - /** \brief Add the LoraAdapter to the list of active adapters. - * The setting does not affect RunWithBinding() calls. - * - * Wraps OrtApi::RunOptionsAddActiveLoraAdapter - * \param adapter The LoraAdapter to be used as the active adapter - */ - RunOptions& AddActiveLoraAdapter(const LoraAdapter& adapter); }; namespace detail { @@ -1275,6 +820,8 @@ struct CustomOpConfigs { * Wraps ::OrtSessionOptions object and methods */ +struct SessionOptions; + namespace detail { // we separate const-only methods because passing const ptr to non-const methods // is only discovered when inline methods are compiled which is counter-intuitive @@ -1287,7 +834,7 @@ struct ConstSessionOptionsImpl : Base { std::string GetConfigEntry(const char* config_key) const; ///< Wraps OrtApi::GetSessionConfigEntry bool HasConfigEntry(const char* config_key) const; ///< Wraps OrtApi::HasSessionConfigEntry - std::string GetConfigEntryOrDefault(const char* config_key, const std::string& def) const; + std::string GetConfigEntryOrDefault(const char* config_key, const std::string& def); }; template @@ -1315,8 +862,6 @@ struct SessionOptionsImpl : ConstSessionOptionsImpl { SessionOptionsImpl& SetExecutionMode(ExecutionMode execution_mode); ///< Wraps OrtApi::SetSessionExecutionMode - SessionOptionsImpl& SetLoadCancellationFlag(bool value); ///< Wraps OrtApi::SessionOptionsSetLoadCancellationFlag - SessionOptionsImpl& SetLogId(const char* logid); ///< Wraps OrtApi::SetSessionLogId SessionOptionsImpl& SetLogSeverityLevel(int level); ///< Wraps OrtApi::SetSessionLogSeverityLevel @@ -1332,7 +877,6 @@ struct SessionOptionsImpl : ConstSessionOptionsImpl { const std::vector& external_initializer_file_buffer_array, const std::vector& external_initializer_file_lengths); ///< Wraps OrtApi::AddExternalInitializersFromFilesInMemory - SessionOptionsImpl& AppendExecutionProvider_CPU(int use_arena); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_CPU SessionOptionsImpl& AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_CUDA SessionOptionsImpl& AppendExecutionProvider_CUDA_V2(const OrtCUDAProviderOptionsV2& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_CUDA_V2 SessionOptionsImpl& AppendExecutionProvider_ROCM(const OrtROCMProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_ROCM @@ -1342,29 +886,14 @@ struct SessionOptionsImpl : ConstSessionOptionsImpl { SessionOptionsImpl& AppendExecutionProvider_TensorRT(const OrtTensorRTProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_TensorRT SessionOptionsImpl& AppendExecutionProvider_TensorRT_V2(const OrtTensorRTProviderOptionsV2& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_TensorRT SessionOptionsImpl& AppendExecutionProvider_MIGraphX(const OrtMIGraphXProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_MIGraphX - /// Wraps OrtApi::SessionOptionsAppendExecutionProvider_CANN + ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_CANN SessionOptionsImpl& AppendExecutionProvider_CANN(const OrtCANNProviderOptions& provider_options); - /// Wraps OrtApi::SessionOptionsAppendExecutionProvider_Dnnl + ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_Dnnl SessionOptionsImpl& AppendExecutionProvider_Dnnl(const OrtDnnlProviderOptions& provider_options); /// Wraps OrtApi::SessionOptionsAppendExecutionProvider. Currently supports QNN, SNPE and XNNPACK. SessionOptionsImpl& AppendExecutionProvider(const std::string& provider_name, const std::unordered_map& provider_options = {}); - /// Append EPs that have been registered previously with the OrtEnv. - /// Wraps OrtApi::SessionOptionsAppendExecutionProvider_V2 - SessionOptionsImpl& AppendExecutionProvider_V2(Env& env, const std::vector& ep_devices, - const KeyValuePairs& ep_options); - /// Append EPs that have been registered previously with the OrtEnv. - /// Wraps OrtApi::SessionOptionsAppendExecutionProvider_V2 - SessionOptionsImpl& AppendExecutionProvider_V2(Env& env, const std::vector& ep_devices, - const std::unordered_map& ep_options); - - /// Wraps OrtApi::SessionOptionsSetEpSelectionPolicy - SessionOptionsImpl& SetEpSelectionPolicy(OrtExecutionProviderDevicePolicy policy); - - /// Wraps OrtApi::SessionOptionsSetEpSelectionPolicyDelegate - SessionOptionsImpl& SetEpSelectionPolicy(EpSelectionDelegate delegate, void* state = nullptr); - SessionOptionsImpl& SetCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn); ///< Wraps OrtApi::SessionOptionsSetCustomCreateThreadFn SessionOptionsImpl& SetCustomThreadCreationOptions(void* ort_custom_thread_creation_options); ///< Wraps OrtApi::SessionOptionsSetCustomThreadCreationOptions SessionOptionsImpl& SetCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn); ///< Wraps OrtApi::SessionOptionsSetCustomJoinThreadFn @@ -1395,61 +924,12 @@ struct SessionOptions : detail::SessionOptionsImpl { ConstSessionOptions GetConst() const { return ConstSessionOptions{this->p_}; } }; -/** \brief Options object used when compiling a model. - * - * Wraps ::OrtModelCompilationOptions object and methods - */ -struct ModelCompilationOptions : detail::Base { - using Base = detail::Base; - using Base::Base; - - explicit ModelCompilationOptions(std::nullptr_t) {} ///< Create an empty ModelCompilationOptions object, must be assigned a valid one to be used. - - ModelCompilationOptions(const Env& env, const SessionOptions& session_options); ///< Wraps OrtApi::CreateModelCompilationOptionsFromSessionOptions - ModelCompilationOptions(const Env& env, ConstSessionOptions session_options); ///< Wraps OrtApi::CreateModelCompilationOptionsFromSessionOptions - - ModelCompilationOptions& SetInputModelPath(const ORTCHAR_T* input_model_path); ///< Wraps OrtApi::ModelCompilationOptions_SetInputModelPath - ModelCompilationOptions& SetInputModelFromBuffer(const void* input_model_data, - size_t input_model_data_size); ///< Wraps OrtApi::ModelCompilationOptions_SetInputModelFromBuffer - ModelCompilationOptions& SetEpContextEmbedMode(bool embed_ep_context_in_model); ///< Wraps OrtApi::ModelCompilationOptions_SetEpContextEmbedMode - ModelCompilationOptions& SetOutputModelPath(const ORTCHAR_T* output_model_path); ///< Wraps OrtApi::ModelCompilationOptions_SetOutputModelPath - ModelCompilationOptions& SetOutputModelExternalInitializersFile(const ORTCHAR_T* file_path, - size_t initializer_size_threshold); ///< Wraps OrtApi::ModelCompilationOptions_SetOutputModelExternalInitializersFile - - ///< Wraps OrtApi::ModelCompilationOptions_SetOutputModelGetInitializerLocationFunc - ModelCompilationOptions& SetOutputModelGetInitializerLocationFunc( - OrtGetInitializerLocationFunc get_initializer_location_func, - void* state); - - ModelCompilationOptions& SetOutputModelBuffer(OrtAllocator* allocator, void** output_model_buffer_ptr, - size_t* output_model_buffer_size_ptr); ///< Wraps OrtApi::ModelCompilationOptions_SetOutputModelBuffer - - ///< Wraps OrtApi::ModelCompilationOptions_SetOutputModelWriteFunc - ModelCompilationOptions& SetOutputModelWriteFunc(OrtWriteBufferFunc write_func, void* state); - - ModelCompilationOptions& SetEpContextBinaryInformation(const ORTCHAR_T* output_directory, - const ORTCHAR_T* model_name); ///< Wraps OrtApi::ModelCompilationOptions_SetEpContextBinaryInformation - ModelCompilationOptions& SetFlags(uint32_t flags); ///< Wraps OrtApi::ModelCompilationOptions_SetFlags - - ModelCompilationOptions& SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level); ///< Wraps OrtApi::ModelCompilationOptions_SetGraphOptimizationLevel -}; - -/** \brief Compiles an input model to generate a model with EPContext nodes that execute EP-specific kernels. Wraps OrtApi::CompileModels. - * - * \param env: ORT environment object. - * \param model_compilation_options: Compilation options for a model. - * \return A Status indicating success or failure. - */ -Status CompileModel(const Env& env, const ModelCompilationOptions& model_compilation_options); - /** \brief Wrapper around ::OrtModelMetadata * */ struct ModelMetadata : detail::Base { - using Base = detail::Base; - using Base::Base; - - explicit ModelMetadata(std::nullptr_t) {} ///< Create an empty ModelMetadata object, must be assigned a valid one to be used + explicit ModelMetadata(std::nullptr_t) {} ///< Create an empty ModelMetadata object, must be assigned a valid one to be used + explicit ModelMetadata(OrtModelMetadata* p) : Base{p} {} ///< Used for interop with the C API /** \brief Returns a copy of the producer name. * @@ -1528,14 +1008,6 @@ struct ConstSessionImpl : Base { size_t GetOutputCount() const; ///< Returns the number of model outputs size_t GetOverridableInitializerCount() const; ///< Returns the number of inputs that have defaults that can be overridden - std::vector GetInputNames() const; - std::vector GetOutputNames() const; - std::vector GetOverridableInitializerNames() const; - - std::vector GetMemoryInfoForInputs() const; ///< Wrapper for OrtApi::SessionGetMemoryInfoForInputs - std::vector GetMemoryInfoForOutputs() const; ///< Wrapper for OrtApi::SessionGetMemoryInfoForOutputs - std::vector GetEpDeviceForInputs() const; ///< Wrapper for OrtApi::SessionGetEpDeviceForInputs - /** \brief Returns a copy of input name at the specified index. * * \param index must less than the value returned by GetInputCount() @@ -1569,12 +1041,6 @@ struct ConstSessionImpl : Base { TypeInfo GetInputTypeInfo(size_t index) const; ///< Wraps OrtApi::SessionGetInputTypeInfo TypeInfo GetOutputTypeInfo(size_t index) const; ///< Wraps OrtApi::SessionGetOutputTypeInfo TypeInfo GetOverridableInitializerTypeInfo(size_t index) const; ///< Wraps OrtApi::SessionGetOverridableInitializerTypeInfo - - int GetOpset(const std::string& domain) const; ///< Wraps OrtApi::SessionGetOpsetForDomain - - // Will move before checkin if that's the case. - std::vector GetInputs() const; - std::vector GetOutputs() const; }; template @@ -1639,22 +1105,6 @@ struct SessionImpl : ConstSessionImpl { * The OrtAllocator instances must be valid at the point of memory release. */ AllocatedStringPtr EndProfilingAllocated(OrtAllocator* allocator); ///< Wraps OrtApi::SessionEndProfiling - - /** \brief Set DynamicOptions for EPs (Execution Providers) - * - * Wraps OrtApi::SetEpDynamicOptions - * - * Valid options can be found in `include\onnxruntime\core\session\onnxruntime_session_options_config_keys.h` - * Look for `kOrtEpDynamicOptions` - * - * \param[in] keys Array of null terminated UTF8 encoded strings of EP dynamic option keys - * \param[in] values Array of null terminated UTF8 encoded string of EP dynamic option values - * \param[in] kv_len Number of elements in the keys and values arrays - */ - void SetEpDynamicOptions(const char* const* keys, const char* const* values, size_t kv_len); - - void FinalizeModelEditorSession(const Model& model, const SessionOptions& options, - OrtPrepackedWeightsContainer* prepacked_weights_container = nullptr); }; } // namespace detail @@ -1666,39 +1116,49 @@ using UnownedSession = detail::SessionImpl>; * */ struct Session : detail::SessionImpl { - /// Create an empty Session object, must be assigned a valid one to be used. Wraps OrtApi::CreateSession - explicit Session(std::nullptr_t) {} - explicit Session(OrtSession* p) : SessionImpl{p} {} ///< C API Interop - - Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options); - - /// Wraps OrtApi::CreateSessionWithPrepackedWeightsContainer + explicit Session(std::nullptr_t) {} ///< Create an empty Session object, must be assigned a valid one to be used + Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options); ///< Wraps OrtApi::CreateSession Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options, - OrtPrepackedWeightsContainer* prepacked_weights_container); - - /// Wraps OrtApi::CreateSessionFromArray - Session(const Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options); - - /// Wraps OrtApi::CreateSessionFromArrayWithPrepackedWeightsContainer + OrtPrepackedWeightsContainer* prepacked_weights_container); ///< Wraps OrtApi::CreateSessionWithPrepackedWeightsContainer + Session(const Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options); ///< Wraps OrtApi::CreateSessionFromArray Session(const Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options, - OrtPrepackedWeightsContainer* prepacked_weights_container); - -#if !defined(ORT_MINIMAL_BUILD) - /// Wraps OrtModelEditorApi::CreateSessionFromModel - Session(const Env& env, const Model& model, const SessionOptions& options); - - /// Wraps OrtModelEditorApi::CreateModelEditorSession - static Session CreateModelEditorSession(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options); - - /// Wraps OrtModelEditorApi::CreateModelEditorSession - static Session CreateModelEditorSession(const Env& env, const void* model_data, size_t model_data_length, - const SessionOptions& options); -#endif // !defined(ORT_MINIMAL_BUILD) + OrtPrepackedWeightsContainer* prepacked_weights_container); ///< Wraps OrtApi::CreateSessionFromArrayWithPrepackedWeightsContainer ConstSession GetConst() const { return ConstSession{this->p_}; } UnownedSession GetUnowned() const { return UnownedSession{this->p_}; } }; +namespace detail { +template +struct MemoryInfoImpl : Base { + using B = Base; + using B::B; + + std::string GetAllocatorName() const; + OrtAllocatorType GetAllocatorType() const; + int GetDeviceId() const; + OrtMemoryInfoDeviceType GetDeviceType() const; + OrtMemType GetMemoryType() const; + + template + bool operator==(const MemoryInfoImpl& o) const; +}; +} // namespace detail + +// Const object holder that does not own the underlying object +using ConstMemoryInfo = detail::MemoryInfoImpl>; + +/** \brief Wrapper around ::OrtMemoryInfo + * + */ +struct MemoryInfo : detail::MemoryInfoImpl { + static MemoryInfo CreateCpu(OrtAllocatorType type, OrtMemType mem_type1); + explicit MemoryInfo(std::nullptr_t) {} ///< No instance is created + explicit MemoryInfo(OrtMemoryInfo* p) : MemoryInfoImpl{p} {} ///< Take ownership of a pointer created by C Api + MemoryInfo(const char* name, OrtAllocatorType type, int id, OrtMemType mem_type); + ConstMemoryInfo GetConst() const { return ConstMemoryInfo{this->p_}; } +}; + namespace detail { template struct TensorTypeAndShapeInfoImpl : Base { @@ -1717,7 +1177,6 @@ struct TensorTypeAndShapeInfoImpl : Base { [[deprecated("use GetShape()")]] void GetDimensions(int64_t* values, size_t values_count) const; ///< Wraps OrtApi::GetDimensions void GetSymbolicDimensions(const char** values, size_t values_count) const; ///< Wraps OrtApi::GetSymbolicDimensions - std::vector GetSymbolicDimensions() const; std::vector GetShape() const; ///< Uses GetDimensionsCount & GetDimensions to return a std::vector of the shape }; @@ -1730,21 +1189,8 @@ using ConstTensorTypeAndShapeInfo = detail::TensorTypeAndShapeInfoImpl { - using Base = detail::TensorTypeAndShapeInfoImpl; - using Base::Base; - - /// Create an empty TensorTypeAndShapeInfo object, must be assigned a valid one to be used - explicit TensorTypeAndShapeInfo(std::nullptr_t) {} - /// Used for interop with the C API - explicit TensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* p) : TensorTypeAndShapeInfoImpl{p} {} - - // Create a TensorTypeAndShapeInfo object with the specified element type and dimensions - // symbolic_dims are optional, but should be 1:1 with dims. - // The value in symbolic_dims will be used for all entries in dims that are -1. - explicit TensorTypeAndShapeInfo(ONNXTensorElementDataType element_type, - const std::vector& dims, - const std::vector* symbolic_dims = nullptr); - + explicit TensorTypeAndShapeInfo(std::nullptr_t) {} ///< Create an empty TensorTypeAndShapeInfo object, must be assigned a valid one to be used + explicit TensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* p) : TensorTypeAndShapeInfoImpl{p} {} ///< Used for interop with the C API ConstTensorTypeAndShapeInfo GetConst() const { return ConstTensorTypeAndShapeInfo{this->p_}; } }; @@ -1764,9 +1210,6 @@ using ConstSequenceTypeInfo = detail::SequenceTypeInfoImpl { - using Base = detail::SequenceTypeInfoImpl; - using Base::Base; - explicit SequenceTypeInfo(std::nullptr_t) {} ///< Create an empty SequenceTypeInfo object, must be assigned a valid one to be used explicit SequenceTypeInfo(OrtSequenceTypeInfo* p) : SequenceTypeInfoImpl{p} {} ///< Used for interop with the C API ConstSequenceTypeInfo GetConst() const { return ConstSequenceTypeInfo{this->p_}; } @@ -1802,9 +1245,6 @@ using ConstMapTypeInfo = detail::MapTypeInfoImpl { - using Base = detail::MapTypeInfoImpl; - using Base::Base; - explicit MapTypeInfo(std::nullptr_t) {} ///< Create an empty MapTypeInfo object, must be assigned a valid one to be used explicit MapTypeInfo(OrtMapTypeInfo* p) : MapTypeInfoImpl{p} {} ///< Used for interop with the C API ConstMapTypeInfo GetConst() const { return ConstMapTypeInfo{this->p_}; } @@ -1836,21 +1276,9 @@ using ConstTypeInfo = detail::TypeInfoImpl>; /// the information about contained sequence or map depending on the ONNXType. /// struct TypeInfo : detail::TypeInfoImpl { - using Base = detail::TypeInfoImpl; - using Base::Base; - - /// Create an empty TypeInfo object, must be assigned a valid one to be used - explicit TypeInfo(std::nullptr_t) {} + explicit TypeInfo(std::nullptr_t) {} ///< Create an empty TypeInfo object, must be assigned a valid one to be used explicit TypeInfo(OrtTypeInfo* p) : TypeInfoImpl{p} {} ///< C API Interop -#if !defined(ORT_MINIMAL_BUILD) - static TypeInfo CreateTensorInfo(ConstTensorTypeAndShapeInfo tensor_info); - static TypeInfo CreateSparseTensorInfo(ConstTensorTypeAndShapeInfo sparse_tensor_info); - static TypeInfo CreateSequenceTypeInfo(ConstTypeInfo sequence_type); - static TypeInfo CreateMapTypeInfo(ONNXTensorElementDataType key_type, ConstTypeInfo value_type); - static TypeInfo CreateOptionalTypeInfo(ConstTypeInfo contained_type); -#endif // !defined(ORT_MINIMAL_BUILD) - ConstTypeInfo GetConst() const { return ConstTypeInfo{this->p_}; } }; @@ -1927,7 +1355,7 @@ struct ConstValueImpl : Base { /// /// const pointer to data, no copies made template - const R* GetTensorData() const; ///< Wraps OrtApi::GetTensorData /// + const R* GetTensorData() const; ///< Wraps OrtApi::GetTensorMutableData /// /// /// Returns a non-typed pointer to a tensor contained data. @@ -1983,14 +1411,6 @@ struct ConstValueImpl : Base { /// byte length for the specified string element size_t GetStringTensorElementLength(size_t element_index) const; - /// - /// Returns the total size of the tensor data in bytes. Throws an exception if the OrtValue - /// does not contain a tensor or if it contains a tensor that contains strings. - /// For numeric tensors, this is sizeof(element_type) * total_element_count. - /// - /// The total size of the tensor data in bytes - size_t GetTensorSizeInBytes() const; ///< Wraps OrtApi::GetTensorSizeInBytes - #if !defined(DISABLE_SPARSE_TENSORS) /// /// The API returns the sparse data format this OrtValue holds in a sparse tensor. @@ -2072,7 +1492,7 @@ struct ValueImpl : ConstValueImpl { /// by the vector of dims. /// /// - /// [in] expressed by a vector of dimensions offsets + /// [in] expressed by a vecotr of dimensions offsets /// template R& At(const std::vector& location); @@ -2193,11 +1613,11 @@ using UnownedValue = detail::ValueImpl>; */ struct Value : detail::ValueImpl { using Base = detail::ValueImpl; - using Base::Base; using OrtSparseValuesParam = detail::OrtSparseValuesParam; using Shape = detail::Shape; - Value(std::nullptr_t) {} ///< Create an empty Value object, must be assigned a valid one to be used + explicit Value(std::nullptr_t) {} ///< Create an empty Value object, must be assigned a valid one to be used + explicit Value(OrtValue* p) : Base{p} {} ///< Used for interop with the C API Value(Value&&) = default; Value& operator=(Value&&) = default; @@ -2213,8 +1633,7 @@ struct Value : detail::ValueImpl { * \param shape_len The number of tensor shape dimensions. */ template - static Value CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count, - const int64_t* shape, size_t shape_len); + static Value CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count, const int64_t* shape, size_t shape_len); /** \brief Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAsOrtValue. * @@ -2225,25 +1644,11 @@ struct Value : detail::ValueImpl { * \param shape_len The number of tensor shape dimensions. * \param type The data type. */ - static Value CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, - const int64_t* shape, size_t shape_len, - ONNXTensorElementDataType type); - - /** \brief Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAndDeleterAsOrtValue. - * - * \param deleter OrtAllocator that will be used to free the buffer when no longer required. - * \param p_data Pointer to the data buffer. - * \param p_data_byte_count The number of bytes in the data buffer. - * \param shape Pointer to the tensor shape dimensions. - * \param shape_len The number of tensor shape dimensions. - * \param type The data type. - */ - static Value CreateTensor(OrtAllocator* deleter, void* p_data, size_t p_data_byte_count, - const int64_t* shape, size_t shape_len, + static Value CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type); /** \brief Creates an OrtValue with a tensor using a supplied OrtAllocator. Wraps OrtApi::CreateTensorAsOrtValue. - * This overload will allocate the buffer for the tensor according to the supplied shape and data type. + * This overload will allocate the buffer for the tensor according to the supplied shape and data type. * The allocated buffer will be owned by the returned OrtValue and will be freed when the OrtValue is released. * The input data would need to be copied into the allocated buffer. * This API is not suitable for strings. @@ -2267,8 +1672,7 @@ struct Value : detail::ValueImpl { * \param shape_len The number of tensor shape dimensions. * \param type The data type. */ - static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, - ONNXTensorElementDataType type); + static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type); /** \brief Creates an OrtValue with a Map Onnx type representation. * The API would ref-count the supplied OrtValues and they will be released @@ -2362,6 +1766,61 @@ struct Value : detail::ValueImpl { #endif // !defined(DISABLE_SPARSE_TENSORS) }; +/// +/// Represents native memory allocation coming from one of the +/// OrtAllocators registered with OnnxRuntime. +/// Use it to wrap an allocation made by an allocator +/// so it can be automatically released when no longer needed. +/// +struct MemoryAllocation { + MemoryAllocation(OrtAllocator* allocator, void* p, size_t size); + ~MemoryAllocation(); + MemoryAllocation(const MemoryAllocation&) = delete; + MemoryAllocation& operator=(const MemoryAllocation&) = delete; + MemoryAllocation(MemoryAllocation&&) noexcept; + MemoryAllocation& operator=(MemoryAllocation&&) noexcept; + + void* get() { return p_; } + size_t size() const { return size_; } + + private: + OrtAllocator* allocator_; + void* p_; + size_t size_; +}; + +namespace detail { +template +struct AllocatorImpl : Base { + using B = Base; + using B::B; + + void* Alloc(size_t size); + MemoryAllocation GetAllocation(size_t size); + void Free(void* p); + ConstMemoryInfo GetInfo() const; +}; + +} // namespace detail + +/** \brief Wrapper around ::OrtAllocator default instance that is owned by Onnxruntime + * + */ +struct AllocatorWithDefaultOptions : detail::AllocatorImpl> { + explicit AllocatorWithDefaultOptions(std::nullptr_t) {} ///< Convenience to create a class member and then replace with an instance + AllocatorWithDefaultOptions(); +}; + +/** \brief Wrapper around ::OrtAllocator + * + */ +struct Allocator : detail::AllocatorImpl { + explicit Allocator(std::nullptr_t) {} ///< Convenience to create a class member and then replace with an instance + Allocator(const Session& session, const OrtMemoryInfo*); +}; + +using UnownedAllocator = detail::AllocatorImpl>; + namespace detail { namespace binding_utils { // Bring these out of template @@ -2424,58 +1883,17 @@ struct ArenaCfg : detail::Base { * See docs/C_API.md for details on what the following parameters mean and how to choose these values */ ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk); - - /** - * Wraps Ort::CreateArenaCfgV2 - * See C API for details on what the following parameters mean and how to choose these values - */ - explicit ArenaCfg(const std::unordered_map& arena_config); }; // // Custom OPs (only needed to implement custom OPs) // -namespace detail { -// Need to define a templated ConstOpAttr with const members -template -struct ConstOpAttrImpl : Base { - using B = detail::Base; - using B::B; - - // Wraps OrtApi::OpAttr_GetName - std::string GetName() const; - // Wraps OrtApi::OpAttr_GetType - OrtOpAttrType GetType() const; - - // Wraps OrtApi::ReadAttr for a single value - // This does not support Tensor Attribute - // Call GetTensorAttributeAsOrtValue() instead. - template - Status GetValue(R& out) const; - - // Wraps OrtApi::ReadAttr for an array of values - template - Status GetValueArray(std::vector& out) const; - // Wraps OrtApi::OpAttr_GetTensorAttributeAsOrtValue - Status GetTensorAttributeAsOrtValue(Value&) const; -}; -} // namespace detail - -using ConstOpAttr = detail::ConstOpAttrImpl>; - /// /// This struct provides life time management for custom op attribute /// -struct OpAttr : detail::ConstOpAttrImpl { - using Base = detail::ConstOpAttrImpl; - using Base::Base; - - OpAttr() = default; // Enable storing it in the container for resize() - explicit OpAttr(std::nullptr_t) {} +struct OpAttr : detail::Base { OpAttr(const char* name, const void* data, int len, OrtOpAttrType type); - - ConstOpAttr GetConst() const { return ConstOpAttr{this->p_}; } }; /** @@ -2640,10 +2058,10 @@ struct KernelContext { explicit KernelContext(OrtKernelContext* context); size_t GetInputCount() const; size_t GetOutputCount() const; - // If input is optional and is not present, the method returns an empty ConstValue + // If input is optional and is not present, the method returns en empty ConstValue // which can be compared to nullptr. ConstValue GetInput(size_t index) const; - // If output is optional and is not present, the method returns an empty UnownedValue + // If outout is optional and is not present, the method returns en empty UnownedValue // which can be compared to nullptr. UnownedValue GetOutput(size_t index, const int64_t* dim_values, size_t dim_count) const; UnownedValue GetOutput(size_t index, const std::vector& dims) const; @@ -2717,8 +2135,6 @@ using ConstKernelInfo = detail::KernelInfoImpl struct KernelInfo : detail::KernelInfoImpl { - using Base = detail::KernelInfoImpl; - using Base::Base; explicit KernelInfo(std::nullptr_t) {} ///< Create an empty instance to initialize later explicit KernelInfo(OrtKernelInfo* info); ///< Take ownership of the instance ConstKernelInfo GetConst() const { return ConstKernelInfo{this->p_}; } @@ -2728,9 +2144,6 @@ struct KernelInfo : detail::KernelInfoImpl { /// Create and own custom defined operation. /// struct Op : detail::Base { - using Base = detail::Base; - using Base::Base; - explicit Op(std::nullptr_t) {} ///< Create an empty Operator object, must be assigned a valid one to be used explicit Op(OrtOp*); ///< Take ownership of the OrtOp @@ -2762,8 +2175,8 @@ struct Op : detail::Base { /// struct ShapeInferContext { struct SymbolicInteger { - SymbolicInteger(int64_t i) : i_(i), is_int_(true) {}; - SymbolicInteger(const char* s) : s_(s), is_int_(false) {}; + SymbolicInteger(int64_t i) : i_(i), is_int_(true){}; + SymbolicInteger(const char* s) : s_(s), is_int_(false){}; SymbolicInteger(const SymbolicInteger&) = default; SymbolicInteger(SymbolicInteger&&) = default; @@ -2803,7 +2216,7 @@ struct ShapeInferContext { size_t GetInputCount() const { return input_shapes_.size(); } - Status SetOutputShape(size_t indice, const Shape& shape, ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT); + Status SetOutputShape(size_t indice, const Shape& shape); int64_t GetAttrInt(const char* attr_name); @@ -2821,7 +2234,7 @@ struct ShapeInferContext { Strings GetAttrStrings(const char* attr_name); private: - ConstOpAttr GetAttrHdl(const char* attr_name) const; + const OrtOpAttr* GetAttrHdl(const char* attr_name) const; const OrtApi* ort_api_; OrtShapeInferContext* ctx_; std::vector input_shapes_; @@ -2911,7 +2324,7 @@ struct CustomOpBase : OrtCustomOp { return OrtCustomOpInputOutputCharacteristic::INPUT_OUTPUT_REQUIRED; } - // Default implementation of GetInputMemoryType() that returns OrtMemTypeDefault + // Default implemention of GetInputMemoryType() that returns OrtMemTypeDefault OrtMemType GetInputMemoryType(size_t /*index*/) const { return OrtMemTypeDefault; } @@ -2947,9 +2360,6 @@ struct CustomOpBase : OrtCustomOp { return std::vector{}; } - // Ort::CustomOpBase derived class should provide the following static method with the type/shape inferencing - // implementation if needed: - // static OrtStatusPtr InferOutputShape(Ort::ShapeInferContext& context) template decltype(&C::InferOutputShape) SetShapeInferFn(decltype(&C::InferOutputShape)) { OrtCustomOp::InferOutputShapeFn = [](const OrtCustomOp*, OrtShapeInferContext* ort_ctx) -> OrtStatusPtr { @@ -2972,253 +2382,6 @@ struct CustomOpBase : OrtCustomOp { int end_ver_ = MAX_CUSTOM_OP_END_VER; }; -// Forward declaration to resolve circular dependency -// on ConstNode -struct ValueInfoConsumerProducerInfo; - -namespace detail { -template -struct ConstValueInfoImpl : Base { - using B = Base; - using B::B; - - /// < A wrapper around OrtApi::GetValueInfoName - std::string GetName() const; - /// < A wrapper around OrtApi::GetValueInfoTypeInfo - ConstTypeInfo TypeInfo() const; - ///< Wraps OrtApi::ValueInfo_GetProducerNode - ValueInfoConsumerProducerInfo GetProducerNode() const; - /// < A wrapper around OrtApi::ValueInfo_GetValueConsumers - std::vector GetConsumers() const; - /// < A wrapper around OrtApi::ValueInfo_GetInitializerValue - Status GetInitializer(ConstValue& value) const; - /// < A wrapper around OrtApi::ValueInfo_GetExternalInitializerInfo - Status GetExternalInitializerInfo(ExternalInitializerInfo& info) const; - /// < A wrapper around OrtApi::ValueInfo_IsRequiredGraphInput - bool IsRequiredGraphInput() const; - /// < A wrapper around OrtApi::ValueInfo_IsOptionalGraphInput - bool IsOptionalGraphInput() const; - /// < A wrapper around OrtApi::ValueInfo_IsGraphOutput - bool IsGraphOutput() const; - /// < A wrapper around OrtApi::ValueInfo_IsConstantInitializer - bool IsConstantInitializer() const; - /// < A wrapper around OrtApi::ValueInfo_IsFromOuterScope - bool IsFromOuterScope() const; -}; -} // namespace detail - -// Const object holder that does not own the underlying object -using ConstValueInfo = detail::ConstValueInfoImpl>; - -/** \brief Wrapper around ::OrtValueInfo - * - */ -struct ValueInfo : detail::ConstValueInfoImpl { - ValueInfo() = default; // Same thing as with nullptr - explicit ValueInfo(std::nullptr_t) {} ///< No instance is created - /// Take ownership of a pointer created by C API - explicit ValueInfo(OrtValueInfo* p) : ConstValueInfoImpl{p} {} - -#if !defined(ORT_MINIMAL_BUILD) - // Create ValueInfo for a tensor - explicit ValueInfo(const std::string& name, const ConstTypeInfo& type_info); -#endif - ConstValueInfo GetConst() const { return ConstValueInfo{this->p_}; } -}; - -// Forward declaration -struct AttrNameSubgraph; - -namespace detail { -// Forward decl -template -struct ConstGraphImpl; - -template -struct ConstNodeImpl : Base { - using B = Base; - using B::B; - - // GetInputs() const; - // GetOutputs() const; - // GetImplicitInputs() const; - // GetAttributes() const; - // GetSubgraphs() const; - // > GetGraph() const; - // >; - -/** \brief Wrapper around ::OrtNode - * - */ -struct Node : detail::ConstNodeImpl { - Node() = default; // Same thing as with nullptr - explicit Node(std::nullptr_t) {} ///< No instance is created - explicit Node(OrtNode* p) : ConstNodeImpl{p} {} ///< Take ownership of a pointer created by C API - -#if !defined(ORT_MINIMAL_BUILD) - Node(const std::string& operator_name, const std::string& operator_domain, - const std::string& node_name, - const std::vector& input_names, - const std::vector& output_names); - - /// - /// Wraps CreateNode. Node takes ownership of attributes on success and updates the OpAttr in `attributes` to do so. - /// - Node(const std::string& operator_name, const std::string& operator_domain, - const std::string& node_name, - const std::vector& input_names, - const std::vector& output_names, - std::vector& attributes); - - private: - static void Init(const std::string& operator_name, const std::string& operator_domain, - const std::string& node_name, - const std::vector& input_names, - const std::vector& output_names, - std::vector& attributes, - OrtNode*& node); -#endif // !defined(ORT_MINIMAL_BUILD) -}; - -// Return struct for some of ValueInfo APIs. -// Must be declared after ConstNode is available. -struct ValueInfoConsumerProducerInfo { - ConstNode node; - // either producer output or consumer output index - // producer is unsigned only, output can be -1 - int64_t index; -}; - -// Represents a return value for Graph::GetOperatorSets() -struct OperatorSet { - std::string domain; - int64_t version; -}; - -namespace detail { -template -struct ConstGraphImpl : Base { - using B = Base; - using B::B; - - // GetModelPath() const; - // GetOperatorSets() const; - // GetInputs() const; - // GetOutputs() const; - // GetInitializers() const; - // GetNodes() const; - // & nodes) const; - // -struct GraphImpl : ConstGraphImpl { - using B = ConstGraphImpl; - using B::B; - -#if !defined(ORT_MINIMAL_BUILD) - // & inputs); - // & outputs); - // >; - -// Return value for Node API -// Must be declared after ConstGraph -struct AttrNameSubgraph { - std::string attr_name; - ConstGraph sub_graph; -}; - -/** \brief Wrapper around ::OrtGraph - * - */ -struct Graph : detail::GraphImpl { - explicit Graph(std::nullptr_t) {} ///< No instance is created - explicit Graph(OrtGraph* p) : GraphImpl{p} {} ///< Take ownership of a pointer created by C API -#if !defined(ORT_MINIMAL_BUILD) - // -struct ModelImpl : detail::Base { - using B = Ort::detail::Base; - using B::B; - -#if !defined(ORT_MINIMAL_BUILD) - // >; - -/** \brief Wrapper around ::OrtModel - * - */ -struct Model : detail::ModelImpl { - using DomainOpsetPair = std::pair; - - explicit Model(std::nullptr_t) {} ///< No instance is created - explicit Model(OrtModel* p) : ModelImpl{p} {} ///< Take ownership of a pointer created by C API - -#if !defined(ORT_MINIMAL_BUILD) - //< Wraps GetModelEditorApi().CreateModel() - explicit Model(const std::vector& opsets); -#endif -}; } // namespace Ort + #include "onnxruntime_cxx_inline.h" diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_cxx_inline.h b/prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_cxx_inline.h similarity index 56% rename from prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_cxx_inline.h rename to prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_cxx_inline.h index cb6448a..a732bf1 100644 --- a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_cxx_inline.h +++ b/prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_cxx_inline.h @@ -10,9 +10,7 @@ #include #include #include -#include #include -#include // Convert OrtStatus to Ort::Status and return // instead of throwing @@ -53,18 +51,18 @@ inline void ThrowOnError(const Status& st) { } } -inline Status::Status(OrtStatus* status) noexcept : detail::Base{status} { +inline Status::Status(OrtStatus* status) noexcept : Base{status} { } -inline Status::Status(const std::exception& e) { +inline Status::Status(const std::exception& e) noexcept { p_ = GetApi().CreateStatus(ORT_FAIL, e.what()); } -inline Status::Status(const Exception& e) { +inline Status::Status(const Exception& e) noexcept { p_ = GetApi().CreateStatus(e.GetOrtErrorCode(), e.what()); } -inline Status::Status(const char* message, OrtErrorCode code) { +inline Status::Status(const char* message, OrtErrorCode code) noexcept { p_ = GetApi().CreateStatus(code, message); } @@ -243,12 +241,6 @@ inline ConstMemoryInfo AllocatorImpl::GetInfo() const { return ConstMemoryInfo{out}; } -template -inline KeyValuePairs AllocatorImpl::GetStats() const { - OrtKeyValuePairs* out; - ThrowOnError(GetApi().AllocatorGetStats(this->p_, &out)); - return KeyValuePairs(out); -} } // namespace detail inline AllocatorWithDefaultOptions::AllocatorWithDefaultOptions() { @@ -296,16 +288,6 @@ inline OrtMemType MemoryInfoImpl::GetMemoryType() const { return type; } -template -inline OrtDeviceMemoryType MemoryInfoImpl::GetDeviceMemoryType() const { - return GetApi().MemoryInfoGetDeviceMemType(this->p_); -} - -template -inline uint32_t MemoryInfoImpl::GetVendorId() const { - return GetApi().MemoryInfoGetVendorId(this->p_); -} - template template inline bool MemoryInfoImpl::operator==(const MemoryInfoImpl& o) const { @@ -326,12 +308,6 @@ inline MemoryInfo::MemoryInfo(const char* name, OrtAllocatorType type, int id, O ThrowOnError(GetApi().CreateMemoryInfo(name, type, id, mem_type, &this->p_)); } -inline MemoryInfo::MemoryInfo(const char* name, OrtMemoryInfoDeviceType device_type, uint32_t vendor_id, uint32_t device_id, - OrtDeviceMemoryType mem_type, size_t alignment, OrtAllocatorType allocator_type) { - ThrowOnError(GetApi().CreateMemoryInfo_V2(name, device_type, vendor_id, device_id, mem_type, alignment, - allocator_type, &this->p_)); -} - namespace detail { template inline std::vector ConstIoBindingImpl::GetOutputNames() const { @@ -420,7 +396,20 @@ inline std::vector GetOutputNamesHelper(const OrtIoBinding* binding inline std::vector GetOutputValuesHelper(const OrtIoBinding* binding, OrtAllocator* allocator) { std::vector result; + size_t owned = 0; size_t output_count = 0; + // Lambda to release the buffer when no longer needed and + // make sure that we destroy all instances on exception + auto free_fn = [&owned, &output_count, allocator](OrtValue** buffer) { + if (buffer) { + while (owned < output_count) { + auto* p = buffer + owned++; + GetApi().ReleaseValue(*p); + } + allocator->Free(allocator, buffer); + } + }; + using Ptr = std::unique_ptr; OrtValue** output_buffer = nullptr; ThrowOnError(GetApi().GetBoundOutputValues(binding, allocator, &output_buffer, &output_count)); @@ -428,11 +417,12 @@ inline std::vector GetOutputValuesHelper(const OrtIoBinding* binding, Ort return result; } - std::unique_ptr buffer_g(output_buffer, AllocatedFree(allocator)); + Ptr buffer_g(output_buffer, free_fn); result.reserve(output_count); for (size_t i = 0; i < output_count; ++i) { result.emplace_back(output_buffer[i]); + ++owned; } return result; } @@ -448,18 +438,6 @@ inline ArenaCfg::ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial ThrowOnError(GetApi().CreateArenaCfg(max_mem, arena_extend_strategy, initial_chunk_size_bytes, max_dead_bytes_per_chunk, &p_)); } -inline ArenaCfg::ArenaCfg(const std::unordered_map& arena_config) { - std::vector keys; - std::vector values; - keys.reserve(arena_config.size()); - values.reserve(arena_config.size()); - for (const auto& kv : arena_config) { - keys.push_back(kv.first.c_str()); - values.push_back(kv.second); - } - ThrowOnError(GetApi().CreateArenaCfgV2(keys.data(), values.data(), arena_config.size(), &p_)); -} - inline ThreadingOptions::ThreadingOptions() { ThrowOnError(GetApi().CreateThreadingOptions(&p_)); } @@ -499,253 +477,6 @@ inline ThreadingOptions& ThreadingOptions::SetGlobalCustomJoinThreadFn(OrtCustom return *this; } -inline TensorRTProviderOptions::TensorRTProviderOptions() { - ThrowOnError(GetApi().CreateTensorRTProviderOptions(&this->p_)); -} - -inline void TensorRTProviderOptions::Update(const std::unordered_map& options) { - std::vector keys; - std::vector values; - keys.reserve(options.size()); - values.reserve(options.size()); - for (const auto& kv : options) { - keys.push_back(kv.first.c_str()); - values.push_back(kv.second.c_str()); - } - ThrowOnError(GetApi().UpdateTensorRTProviderOptions(p_, keys.data(), values.data(), options.size())); -} - -inline void TensorRTProviderOptions::UpdateWithValue(const char* key, void* value) { - ThrowOnError(GetApi().UpdateTensorRTProviderOptionsWithValue(p_, key, value)); -} - -inline void* TensorRTProviderOptions::GetOptionByName(const char* name) const { - void* value = nullptr; - ThrowOnError(GetApi().GetTensorRTProviderOptionsByName(p_, name, &value)); - return value; -} - -inline std::string TensorRTProviderOptions::GetTensorRTProviderOptionsAsString() const { - AllocatorWithDefaultOptions allocator; - char* options_str = nullptr; - ThrowOnError(GetApi().GetTensorRTProviderOptionsAsString(p_, allocator, &options_str)); - std::unique_ptr options_str_g(options_str, detail::AllocatedFree(allocator)); - return std::string(options_str); -} - -inline CUDAProviderOptions::CUDAProviderOptions() { - ThrowOnError(GetApi().CreateCUDAProviderOptions(&this->p_)); -} - -inline void CUDAProviderOptions::Update(const std::unordered_map& options) { - std::vector keys; - std::vector values; - keys.reserve(options.size()); - values.reserve(options.size()); - for (const auto& kv : options) { - keys.push_back(kv.first.c_str()); - values.push_back(kv.second.c_str()); - } - ThrowOnError(GetApi().UpdateCUDAProviderOptions(p_, keys.data(), values.data(), options.size())); -} - -inline std::string CUDAProviderOptions::GetCUDAProviderOptionsAsString() const { - AllocatorWithDefaultOptions allocator; - char* options_str = nullptr; - ThrowOnError(GetApi().GetCUDAProviderOptionsAsString(p_, allocator, &options_str)); - std::unique_ptr options_str_g(options_str, detail::AllocatedFree(allocator)); - return std::string(options_str); -} - -inline void CUDAProviderOptions::UpdateWithValue(const char* key, void* value) { - ThrowOnError(GetApi().UpdateCUDAProviderOptionsWithValue(p_, key, value)); -} - -inline void* CUDAProviderOptions::GetOptionByName(const char* name) const { - void* value = nullptr; - ThrowOnError(GetApi().GetCUDAProviderOptionsByName(p_, name, &value)); - return value; -} - -inline PrepackedWeightsContainer::PrepackedWeightsContainer() { - ThrowOnError(GetApi().CreatePrepackedWeightsContainer(&this->p_)); -} - -namespace detail { - -template -inline const std::basic_string ConstExternalInitializerInfoImpl::GetFilePath() const { - return GetApi().ExternalInitializerInfo_GetFilePath(this->p_); -} - -template -inline int64_t ConstExternalInitializerInfoImpl::GetFileOffset() const { - return GetApi().ExternalInitializerInfo_GetFileOffset(this->p_); -} - -template -inline size_t ConstExternalInitializerInfoImpl::GetByteSize() const { - return GetApi().ExternalInitializerInfo_GetByteSize(this->p_); -} -} // namespace detail - -inline ExternalInitializerInfo::ExternalInitializerInfo(const ORTCHAR_T* filepath, int64_t file_offset, - size_t byte_size) { - ThrowOnError(GetApi().CreateExternalInitializerInfo(filepath, file_offset, byte_size, &this->p_)); -} - -inline Status ExternalInitializerInfo::Create(const ORTCHAR_T* filepath, int64_t file_offset, size_t byte_size, - /*out*/ ExternalInitializerInfo& out) { - OrtExternalInitializerInfo* info = nullptr; - OrtStatus* status = GetApi().CreateExternalInitializerInfo(filepath, file_offset, byte_size, &info); - if (status != nullptr) { - return Status{status}; - } - - out = ExternalInitializerInfo(info); - - return Status{nullptr}; -} - -namespace detail { -template -inline const char* KeyValuePairsImpl::GetValue(const char* key) const { - return GetApi().GetKeyValue(this->p_, key); -} - -template -inline std::unordered_map KeyValuePairsImpl::GetKeyValuePairs() const { - std::unordered_map out; - - size_t num_pairs = 0; - const char* const* keys = nullptr; - const char* const* values = nullptr; - GetApi().GetKeyValuePairs(this->p_, &keys, &values, &num_pairs); - if (num_pairs > 0) { - out.reserve(num_pairs); - for (size_t i = 0; i < num_pairs; ++i) { - out.emplace(keys[i], values[i]); - } - } - - return out; -} - -template -inline void KeyValuePairsImpl::GetKeyValuePairs(std::vector& keys, - std::vector& values) const { - keys.clear(); - values.clear(); - - size_t num_pairs = 0; - const char* const* keys_ptr = nullptr; - const char* const* values_ptr = nullptr; - GetApi().GetKeyValuePairs(this->p_, &keys_ptr, &values_ptr, &num_pairs); - if (num_pairs > 0) { - keys.resize(num_pairs); - values.resize(num_pairs); - std::copy(keys_ptr, keys_ptr + num_pairs, keys.begin()); - std::copy(values_ptr, values_ptr + num_pairs, values.begin()); - } -} -} // namespace detail - -inline KeyValuePairs::KeyValuePairs() { - GetApi().CreateKeyValuePairs(&p_); -} - -inline KeyValuePairs::KeyValuePairs(const std::unordered_map& kv_pairs) { - GetApi().CreateKeyValuePairs(&p_); - for (const auto& kv : kv_pairs) { - GetApi().AddKeyValuePair(this->p_, kv.first.c_str(), kv.second.c_str()); - } -} - -inline void KeyValuePairs::Add(const char* key, const char* value) { - GetApi().AddKeyValuePair(this->p_, key, value); -} - -inline void KeyValuePairs::Remove(const char* key) { - GetApi().RemoveKeyValuePair(this->p_, key); -} - -namespace detail { -template -inline void* SyncStreamImpl::GetHandle() { - return GetApi().SyncStream_GetHandle(this->p_); -} -} // namespace detail - -namespace detail { -template -inline OrtHardwareDeviceType HardwareDeviceImpl::Type() const { - return GetApi().HardwareDevice_Type(this->p_); -} - -template -inline uint32_t HardwareDeviceImpl::VendorId() const { - return GetApi().HardwareDevice_VendorId(this->p_); -} - -template -inline uint32_t HardwareDeviceImpl::DeviceId() const { - return GetApi().HardwareDevice_DeviceId(this->p_); -} - -template -inline const char* HardwareDeviceImpl::Vendor() const { - return GetApi().HardwareDevice_Vendor(this->p_); -} - -template -inline ConstKeyValuePairs HardwareDeviceImpl::Metadata() const { - return ConstKeyValuePairs{GetApi().HardwareDevice_Metadata(this->p_)}; -} - -template -inline const char* EpDeviceImpl::EpName() const { - return GetApi().EpDevice_EpName(this->p_); -} - -template -inline const char* EpDeviceImpl::EpVendor() const { - return GetApi().EpDevice_EpVendor(this->p_); -} - -template -inline ConstKeyValuePairs EpDeviceImpl::EpMetadata() const { - return ConstKeyValuePairs(GetApi().EpDevice_EpMetadata(this->p_)); -} - -template -inline ConstKeyValuePairs EpDeviceImpl::EpOptions() const { - return ConstKeyValuePairs(GetApi().EpDevice_EpOptions(this->p_)); -} - -template -inline ConstHardwareDevice EpDeviceImpl::Device() const { - return ConstHardwareDevice(GetApi().EpDevice_Device(this->p_)); -} - -template -inline ConstMemoryInfo EpDeviceImpl::GetMemoryInfo(OrtDeviceMemoryType memory_type) const { - const auto* mem_info = GetApi().EpDevice_MemoryInfo(this->p_, memory_type); - return ConstMemoryInfo{mem_info}; -} - -template -inline SyncStream EpDeviceImpl::CreateSyncStream(ConstKeyValuePairs stream_options) const { - OrtSyncStream* stream = nullptr; - ThrowOnError(GetApi().CreateSyncStreamForEpDevice(this->p_, stream_options, &stream)); - return SyncStream{stream}; -} -} // namespace detail - -inline EpDevice::EpDevice(OrtEpFactory& ep_factory, ConstHardwareDevice& hardware_device, - ConstKeyValuePairs ep_metadata, ConstKeyValuePairs ep_options) { - ThrowOnError(GetEpApi().CreateEpDevice(&ep_factory, hardware_device, ep_metadata, ep_options, &p_)); -} - inline Env::Env(OrtLoggingLevel logging_level, _In_ const char* logid) { ThrowOnError(GetApi().CreateEnv(logging_level, logid, &p_)); if (strcmp(logid, "onnxruntime-node") == 0) { @@ -818,78 +549,6 @@ inline Env& Env::CreateAndRegisterAllocatorV2(const std::string& provider_type, return *this; } -inline Env& Env::RegisterAllocator(OrtAllocator* allocator) { - ThrowOnError(GetApi().RegisterAllocator(p_, allocator)); - return *this; -} - -inline Env& Env::UnregisterAllocator(const OrtMemoryInfo* mem_info) { - ThrowOnError(GetApi().UnregisterAllocator(p_, mem_info)); - return *this; -} - -inline Env& Env::RegisterExecutionProviderLibrary(const char* registration_name, - const std::basic_string& path) { - ThrowOnError(GetApi().RegisterExecutionProviderLibrary(p_, registration_name, path.c_str())); - return *this; -} - -inline Env& Env::UnregisterExecutionProviderLibrary(const char* registration_name) { - ThrowOnError(GetApi().UnregisterExecutionProviderLibrary(p_, registration_name)); - return *this; -} - -inline std::vector Env::GetEpDevices() const { - size_t num_devices = 0; - const OrtEpDevice* const* device_ptrs = nullptr; - ThrowOnError(GetApi().GetEpDevices(p_, &device_ptrs, &num_devices)); - - std::vector devices; - if (num_devices > 0) { - devices.reserve(num_devices); - for (size_t i = 0; i < num_devices; ++i) { - devices.emplace_back(device_ptrs[i]); - } - } - - return devices; -} - -inline Status Env::CopyTensors(const std::vector& src_tensors, - const std::vector& dst_tensors, - OrtSyncStream* stream) const { - if (src_tensors.size() != dst_tensors.size()) { - return Status("Source and destination tensor vectors must have the same size", ORT_INVALID_ARGUMENT); - } - if (src_tensors.empty()) { - return Status(nullptr); - } - - const OrtValue* const* src_tensors_ptr = reinterpret_cast(src_tensors.data()); - OrtValue* const* dst_tensors_ptr = reinterpret_cast(dst_tensors.data()); - OrtStatus* status = GetApi().CopyTensors(p_, src_tensors_ptr, dst_tensors_ptr, stream, src_tensors.size()); - return Status(status); -} - -inline UnownedAllocator Env::CreateSharedAllocator(const OrtEpDevice* ep_device, OrtDeviceMemoryType mem_type, - OrtAllocatorType allocator_type, - const OrtKeyValuePairs* allocator_options) { - OrtAllocator* p; - ThrowOnError(GetApi().CreateSharedAllocator(p_, ep_device, mem_type, allocator_type, allocator_options, &p)); - return UnownedAllocator{p}; -} - -inline UnownedAllocator Env::GetSharedAllocator(const OrtMemoryInfo* mem_info) { - OrtAllocator* p; - ThrowOnError(GetApi().GetSharedAllocator(p_, mem_info, &p)); - return UnownedAllocator{p}; -} - -inline void Env::ReleaseSharedAllocator(const OrtEpDevice* ep_device, - OrtDeviceMemoryType mem_type) { - ThrowOnError(GetApi().ReleaseSharedAllocator(p_, ep_device, mem_type)); -} - inline CustomOpDomain::CustomOpDomain(const char* domain) { ThrowOnError(GetApi().CreateCustomOpDomain(domain, &p_)); } @@ -898,40 +557,6 @@ inline void CustomOpDomain::Add(const OrtCustomOp* op) { ThrowOnError(GetApi().CustomOpDomain_Add(p_, op)); } -inline OrtCompiledModelCompatibility GetModelCompatibilityForEpDevices( - const std::vector& ep_devices, - const char* compatibility_info) { - if (ep_devices.empty()) { - ORT_CXX_API_THROW("ep_devices is empty", ORT_INVALID_ARGUMENT); - } - - std::vector ptrs; - ptrs.reserve(ep_devices.size()); - for (const auto& d : ep_devices) ptrs.push_back(d); - - OrtCompiledModelCompatibility status = OrtCompiledModelCompatibility_EP_NOT_APPLICABLE; - ThrowOnError(GetApi().GetModelCompatibilityForEpDevices( - reinterpret_cast(ptrs.data()), - ptrs.size(), - compatibility_info, - &status)); - return status; -} - -inline LoraAdapter LoraAdapter::CreateLoraAdapter(const std::basic_string& adapter_path, - OrtAllocator* allocator) { - OrtLoraAdapter* p; - ThrowOnError(GetApi().CreateLoraAdapter(adapter_path.c_str(), allocator, &p)); - return LoraAdapter{p}; -} - -inline LoraAdapter LoraAdapter::CreateLoraAdapterFromArray(const void* bytes, size_t num_bytes, - OrtAllocator* allocator) { - OrtLoraAdapter* p; - ThrowOnError(GetApi().CreateLoraAdapterFromArray(bytes, num_bytes, allocator, &p)); - return LoraAdapter{p}; -} - inline RunOptions::RunOptions() { ThrowOnError(GetApi().CreateRunOptions(&p_)); } @@ -974,10 +599,6 @@ inline RunOptions& RunOptions::AddConfigEntry(const char* config_key, const char return *this; } -inline const char* RunOptions::GetConfigEntry(const char* config_key) { - return GetApi().GetRunConfigEntry(p_, config_key); -} - inline RunOptions& RunOptions::SetTerminate() { ThrowOnError(GetApi().RunOptionsSetTerminate(p_)); return *this; @@ -988,104 +609,6 @@ inline RunOptions& RunOptions::UnsetTerminate() { return *this; } -inline RunOptions& RunOptions::AddActiveLoraAdapter(const LoraAdapter& adapter) { - ThrowOnError(GetApi().RunOptionsAddActiveLoraAdapter(p_, adapter)); - return *this; -} - -inline ModelCompilationOptions::ModelCompilationOptions(const Env& env, const SessionOptions& session_options) { - ThrowOnError(GetCompileApi().CreateModelCompilationOptionsFromSessionOptions(env, session_options, &this->p_)); -} - -inline ModelCompilationOptions::ModelCompilationOptions(const Env& env, ConstSessionOptions session_options) { - ThrowOnError(GetCompileApi().CreateModelCompilationOptionsFromSessionOptions(env, session_options, &this->p_)); -} - -inline Status CompileModel(const Env& env, const ModelCompilationOptions& model_compilation_options) { - return Ort::Status(GetCompileApi().CompileModel(env, model_compilation_options)); -} - -inline ModelCompilationOptions& ModelCompilationOptions::SetInputModelPath( - const ORTCHAR_T* input_model_path) { - Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetInputModelPath(this->p_, input_model_path)); - return *this; -} - -inline ModelCompilationOptions& ModelCompilationOptions::SetInputModelFromBuffer( - const void* input_model_data, size_t input_model_data_size) { - Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetInputModelFromBuffer(this->p_, input_model_data, - input_model_data_size)); - return *this; -} - -inline ModelCompilationOptions& ModelCompilationOptions::SetOutputModelPath( - const ORTCHAR_T* output_model_path) { - Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetOutputModelPath(this->p_, output_model_path)); - return *this; -} - -inline ModelCompilationOptions& ModelCompilationOptions::SetEpContextBinaryInformation( - const ORTCHAR_T* output_directory, const ORTCHAR_T* model_name) { - Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetEpContextBinaryInformation( - this->p_, - output_directory, - model_name)); - return *this; -} - -inline ModelCompilationOptions& ModelCompilationOptions::SetOutputModelExternalInitializersFile( - const ORTCHAR_T* file_path, size_t initializer_size_threshold) { - Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetOutputModelExternalInitializersFile( - this->p_, - file_path, - initializer_size_threshold)); - return *this; -} - -inline ModelCompilationOptions& -ModelCompilationOptions::SetOutputModelGetInitializerLocationFunc( - OrtGetInitializerLocationFunc get_initializer_location_func, void* state) { - Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetOutputModelGetInitializerLocationFunc( - this->p_, - get_initializer_location_func, - state)); - return *this; -} - -inline ModelCompilationOptions& ModelCompilationOptions::SetOutputModelBuffer( - OrtAllocator* allocator, void** output_model_buffer_ptr, size_t* output_model_buffer_size_ptr) { - Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetOutputModelBuffer(this->p_, allocator, - output_model_buffer_ptr, - output_model_buffer_size_ptr)); - return *this; -} - -inline ModelCompilationOptions& ModelCompilationOptions::SetOutputModelWriteFunc(OrtWriteBufferFunc write_func, - void* state) { - Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetOutputModelWriteFunc(this->p_, write_func, state)); - return *this; -} - -inline ModelCompilationOptions& ModelCompilationOptions::SetEpContextEmbedMode( - bool embed_ep_context_in_model) { - Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetEpContextEmbedMode( - this->p_, - embed_ep_context_in_model)); - return *this; -} - -inline ModelCompilationOptions& ModelCompilationOptions::SetFlags(uint32_t flags) { - Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetFlags(this->p_, flags)); - return *this; -} - -inline ModelCompilationOptions& ModelCompilationOptions::SetGraphOptimizationLevel( - GraphOptimizationLevel graph_optimization_level) { - Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetGraphOptimizationLevel(this->p_, - graph_optimization_level)); - return *this; -} - namespace detail { template @@ -1117,8 +640,7 @@ inline bool ConstSessionOptionsImpl::HasConfigEntry(const char* config_key) c } template -inline std::string ConstSessionOptionsImpl::GetConfigEntryOrDefault(const char* config_key, - const std::string& def) const { +inline std::string ConstSessionOptionsImpl::GetConfigEntryOrDefault(const char* config_key, const std::string& def) { if (!this->HasConfigEntry(config_key)) { return def; } @@ -1204,12 +726,6 @@ inline SessionOptionsImpl& SessionOptionsImpl::SetExecutionMode(ExecutionM return *this; } -template -inline SessionOptionsImpl& SessionOptionsImpl::SetLoadCancellationFlag(bool value) { - ThrowOnError(GetApi().SessionOptionsSetLoadCancellationFlag(this->p_, value)); - return *this; -} - template inline SessionOptionsImpl& SessionOptionsImpl::SetLogId(const char* logid) { ThrowOnError(GetApi().SetSessionLogId(this->p_, logid)); @@ -1286,12 +802,6 @@ inline SessionOptionsImpl& SessionOptionsImpl::AddExternalInitializersFrom return *this; } -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_CPU(int use_arena) { - ThrowOnError(OrtSessionOptionsAppendExecutionProvider_CPU(this->p_, use_arena)); - return *this; -} - template inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions& provider_options) { ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_CUDA(this->p_, &provider_options)); @@ -1362,65 +872,6 @@ inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider( return *this; } -namespace { -template -void SessionOptionsAppendEP(detail::SessionOptionsImpl& session_options, - Env& env, const std::vector& ep_devices, - const std::vector& ep_options_keys, - const std::vector& ep_options_values) { - std::vector ep_devices_ptrs; - ep_devices_ptrs.reserve(ep_devices.size()); - for (const auto& ep_device : ep_devices) { - ep_devices_ptrs.push_back(ep_device); - } - - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_V2( - session_options, env, ep_devices_ptrs.data(), ep_devices_ptrs.size(), - ep_options_keys.data(), ep_options_values.data(), ep_options_keys.size())); -} -} // namespace - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_V2( - Env& env, const std::vector& ep_devices, const KeyValuePairs& ep_options) { - std::vector ep_options_keys, ep_options_values; - ep_options.GetKeyValuePairs(ep_options_keys, ep_options_values); - - SessionOptionsAppendEP(*this, env, ep_devices, ep_options_keys, ep_options_values); - - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_V2( - Env& env, const std::vector& ep_devices, - const std::unordered_map& ep_options) { - std::vector ep_options_keys, ep_options_values; - ep_options_keys.reserve(ep_options.size()); - ep_options_values.reserve(ep_options.size()); - - for (const auto& [key, value] : ep_options) { - ep_options_keys.push_back(key.c_str()); - ep_options_values.push_back(value.c_str()); - } - - SessionOptionsAppendEP(*this, env, ep_devices, ep_options_keys, ep_options_values); - - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetEpSelectionPolicy(OrtExecutionProviderDevicePolicy policy) { - ThrowOnError(GetApi().SessionOptionsSetEpSelectionPolicy(this->p_, policy)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetEpSelectionPolicy(EpSelectionDelegate delegate, void* state) { - ThrowOnError(GetApi().SessionOptionsSetEpSelectionPolicyDelegate(this->p_, delegate, state)); - return *this; -} - template inline SessionOptionsImpl& SessionOptionsImpl::SetCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn) { ThrowOnError(GetApi().SessionOptionsSetCustomCreateThreadFn(this->p_, ort_custom_create_thread_fn)); @@ -1525,94 +976,6 @@ inline size_t ConstSessionImpl::GetOverridableInitializerCount() const { return out; } -template -inline std::vector ConstSessionImpl::GetInputNames() const { - AllocatorWithDefaultOptions allocator; - - auto num_inputs = GetInputCount(); - std::vector input_names; - input_names.reserve(num_inputs); - - for (size_t i = 0; i < num_inputs; ++i) { - char* name; - ThrowOnError(GetApi().SessionGetInputName(this->p_, i, allocator, &name)); - input_names.emplace_back(name); - allocator.Free(name); - } - - return input_names; -} - -template -inline std::vector ConstSessionImpl::GetOutputNames() const { - AllocatorWithDefaultOptions allocator; - - auto num_inputs = GetOutputCount(); - std::vector output_names; - output_names.reserve(num_inputs); - - for (size_t i = 0; i < num_inputs; ++i) { - char* name; - ThrowOnError(GetApi().SessionGetOutputName(this->p_, i, allocator, &name)); - output_names.emplace_back(name); - allocator.Free(name); - } - - return output_names; -} - -template -inline std::vector ConstSessionImpl::GetOverridableInitializerNames() const { - AllocatorWithDefaultOptions allocator; - - auto num_initializers = GetOverridableInitializerCount(); - std::vector initializer_names; - initializer_names.reserve(num_initializers); - - for (size_t i = 0; i < num_initializers; ++i) { - char* name; - ThrowOnError(GetApi().SessionGetOverridableInitializerName(this->p_, i, allocator, &name)); - initializer_names.emplace_back(name); - } - - return initializer_names; -} - -template -inline std::vector ConstSessionImpl::GetMemoryInfoForInputs() const { - static_assert(sizeof(ConstMemoryInfo) == sizeof(OrtMemoryInfo*), - "ConstMemoryInfo must be compatible with OrtMemoryInfo*"); - - auto num_inputs = GetInputCount(); - std::vector mem_infos; - if (num_inputs > 0) { - mem_infos.resize(num_inputs); - - ThrowOnError(GetApi().SessionGetMemoryInfoForInputs(this->p_, - reinterpret_cast(mem_infos.data()), - num_inputs)); - } - - return mem_infos; -} - -template -inline std::vector ConstSessionImpl::GetMemoryInfoForOutputs() const { - static_assert(sizeof(ConstMemoryInfo) == sizeof(OrtMemoryInfo*), - "ConstMemoryInfo must be compatible with OrtMemoryInfo*"); - - auto num_outputs = GetOutputCount(); - std::vector mem_infos; - if (num_outputs > 0) { - mem_infos.resize(num_outputs); - - ThrowOnError(GetApi().SessionGetMemoryInfoForOutputs(this->p_, - reinterpret_cast(mem_infos.data()), - num_outputs)); - } - return mem_infos; -} - template inline AllocatedStringPtr ConstSessionImpl::GetInputNameAllocated(size_t index, OrtAllocator* allocator) const { char* out; @@ -1634,19 +997,6 @@ inline AllocatedStringPtr ConstSessionImpl::GetOverridableInitializerNameAllo return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); } -template -inline std::vector ConstSessionImpl::GetEpDeviceForInputs() const { - auto num_inputs = GetInputCount(); - std::vector input_devices; - if (num_inputs > 0) { - input_devices.resize(num_inputs); - ThrowOnError(GetApi().SessionGetEpDeviceForInputs(this->p_, - reinterpret_cast(input_devices.data()), - num_inputs)); - } - return input_devices; -} - template inline uint64_t ConstSessionImpl::GetProfilingStartTimeNs() const { uint64_t out; @@ -1682,45 +1032,6 @@ inline TypeInfo ConstSessionImpl::GetOverridableInitializerTypeInfo(size_t in return TypeInfo{out}; } -#if !defined(ORT_MINIMAL_BUILD) -template -inline int ConstSessionImpl::GetOpset(const std::string& domain) const { - int opset; - ThrowOnError(GetModelEditorApi().SessionGetOpsetForDomain(this->p_, domain.c_str(), &opset)); - return opset; -} -#endif // !defined(ORT_MINIMAL_BUILD) - -template -std::vector ConstSessionImpl::GetInputs() const { - const std::vector input_names = GetInputNames(); - - std::vector inputs; - inputs.reserve(input_names.size()); - - for (size_t i = 0; i < input_names.size(); ++i) { - auto type_info = GetInputTypeInfo(i); - inputs.emplace_back(ValueInfo{input_names[i], type_info.GetConst()}); - } - - return inputs; -} - -template -std::vector ConstSessionImpl::GetOutputs() const { - const std::vector output_names = GetOutputNames(); - - std::vector outputs; - outputs.reserve(output_names.size()); - - for (size_t i = 0; i < output_names.size(); ++i) { - auto type_info = GetOutputTypeInfo(i); - outputs.emplace_back(ValueInfo{output_names[i], type_info.GetConst()}); - } - - return outputs; -} - template inline std::vector SessionImpl::Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, const char* const* output_names, size_t output_count) { @@ -1763,20 +1074,6 @@ inline AllocatedStringPtr SessionImpl::EndProfilingAllocated(OrtAllocator* al return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); } -template -inline void SessionImpl::SetEpDynamicOptions(const char* const* keys, const char* const* values, size_t kv_len) { - ThrowOnError(GetApi().SetEpDynamicOptions(this->p_, keys, values, kv_len)); -} - -#if !defined(ORT_MINIMAL_BUILD) -template -inline void SessionImpl::FinalizeModelEditorSession(const Model& model, const SessionOptions& options, - OrtPrepackedWeightsContainer* prepacked_weights_container) { - ThrowOnError(GetModelEditorApi().ApplyModelToModelEditorSession(this->p_, model)); - ThrowOnError(GetModelEditorApi().FinalizeModelEditorSession(this->p_, options, prepacked_weights_container)); -} -#endif // #if !defined(ORT_MINIMAL_BUILD) - } // namespace detail inline SessionOptions::SessionOptions() { @@ -1823,32 +1120,6 @@ inline Session::Session(const Env& env, const void* model_data, size_t model_dat prepacked_weights_container, &this->p_)); } -#if !defined(ORT_MINIMAL_BUILD) -inline Session::Session(const Env& env, const Model& model, const SessionOptions& options) { - ThrowOnError(GetModelEditorApi().CreateSessionFromModel(env, model, options, &this->p_)); -} - -// static -inline Session Session::CreateModelEditorSession(const Env& env, const ORTCHAR_T* model_path, - const SessionOptions& options) { - OrtSession* session = nullptr; - ThrowOnError(GetModelEditorApi().CreateModelEditorSession(env, model_path, options, &session)); - return Session(session); -} - -// static -inline Session Session::CreateModelEditorSession(const Env& env, const void* model_data, size_t model_data_length, - const SessionOptions& options) { - OrtSession* session = nullptr; - ThrowOnError(GetModelEditorApi().CreateModelEditorSessionFromArray(env, model_data, model_data_length, options, - &session)); - return Session(session); -} - -void FinalizeModelEditorSession(const Model& model, const SessionOptions& options, - OrtPrepackedWeightsContainer* prepacked_weights_container); -#endif // #if !defined(ORT_MINIMAL_BUILD) - inline AllocatedStringPtr ModelMetadata::GetProducerNameAllocated(OrtAllocator* allocator) const { char* out; ThrowOnError(GetApi().ModelMetadataGetProducerName(p_, allocator, &out)); @@ -1916,59 +1187,6 @@ inline int64_t ModelMetadata::GetVersion() const { return out; } -inline TensorTypeAndShapeInfo::TensorTypeAndShapeInfo(ONNXTensorElementDataType element_type, - const std::vector& dims, - const std::vector* symbolic_dims) { - ThrowOnError(GetApi().CreateTensorTypeAndShapeInfo(&p_)); - ThrowOnError(GetApi().SetTensorElementType(p_, element_type)); - ThrowOnError(GetApi().SetDimensions(p_, dims.data(), dims.size())); - - if (symbolic_dims) { - std::vector symbolic_dims_cstr; - symbolic_dims_cstr.reserve(symbolic_dims->size()); - std::transform(symbolic_dims->begin(), symbolic_dims->end(), std::back_inserter(symbolic_dims_cstr), - [](const std::string& s) { return s.c_str(); }); - ThrowOnError(GetApi().SetSymbolicDimensions(p_, symbolic_dims_cstr.data(), symbolic_dims_cstr.size())); - } -} - -#if !defined(ORT_MINIMAL_BUILD) -// static -inline TypeInfo TypeInfo::CreateTensorInfo(ConstTensorTypeAndShapeInfo tensor_type_and_shape_info) { - OrtTypeInfo* output = nullptr; - ThrowOnError(GetModelEditorApi().CreateTensorTypeInfo(tensor_type_and_shape_info, &output)); - return TypeInfo{output}; -} - -// static -inline TypeInfo TypeInfo::CreateSparseTensorInfo(ConstTensorTypeAndShapeInfo sparse_tensor_type_and_shape_info) { - OrtTypeInfo* output = nullptr; - ThrowOnError(GetModelEditorApi().CreateSparseTensorTypeInfo(sparse_tensor_type_and_shape_info, &output)); - return TypeInfo{output}; -} - -// static -inline TypeInfo TypeInfo::CreateSequenceTypeInfo(ConstTypeInfo sequence_type) { - OrtTypeInfo* output; - ThrowOnError(GetModelEditorApi().CreateSequenceTypeInfo(sequence_type, &output)); - return TypeInfo{output}; -} - -// static -inline TypeInfo TypeInfo::CreateMapTypeInfo(ONNXTensorElementDataType key_type, ConstTypeInfo value_type) { - OrtTypeInfo* output; - ThrowOnError(GetModelEditorApi().CreateMapTypeInfo(key_type, value_type, &output)); - return TypeInfo{output}; -} - -// static -inline TypeInfo TypeInfo::CreateOptionalTypeInfo(ConstTypeInfo contained_type) { - OrtTypeInfo* output; - ThrowOnError(GetModelEditorApi().CreateOptionalTypeInfo(contained_type, &output)); - return TypeInfo{output}; -} -#endif // #if !defined(ORT_MINIMAL_BUILD) - namespace detail { template @@ -2002,16 +1220,9 @@ inline void TensorTypeAndShapeInfoImpl::GetSymbolicDimensions(const char** va ThrowOnError(GetApi().GetSymbolicDimensions(this->p_, values, values_count)); } -template -inline std::vector TensorTypeAndShapeInfoImpl::GetSymbolicDimensions() const { - std::vector out(GetDimensionsCount(), nullptr); - ThrowOnError(GetApi().GetSymbolicDimensions(this->p_, out.data(), out.size())); - return out; -} - template inline std::vector TensorTypeAndShapeInfoImpl::GetShape() const { - std::vector out(GetDimensionsCount(), -1); + std::vector out(GetDimensionsCount(), 0); ThrowOnError(GetApi().GetDimensions(this->p_, out.data(), out.size())); return out; } @@ -2131,25 +1342,18 @@ inline size_t ConstValueImpl::GetStringTensorElementLength(size_t element_ind return out; } -template -inline size_t ConstValueImpl::GetTensorSizeInBytes() const { - size_t out; - ThrowOnError(GetApi().GetTensorSizeInBytes(this->p_, &out)); - return out; -} - template template inline const R* ConstValueImpl::GetTensorData() const { - const R* out; - ThrowOnError(GetApi().GetTensorData(this->p_, reinterpret_cast(&out))); + R* out; + ThrowOnError(GetApi().GetTensorMutableData(const_cast(this->p_), (void**)&out)); return out; } template inline const void* ConstValueImpl::GetTensorRawData() const { - const void* out; - ThrowOnError(GetApi().GetTensorData(this->p_, &out)); + void* out; + ThrowOnError(GetApi().GetTensorMutableData(const_cast(this->p_), &out)); return out; } @@ -2332,35 +1536,23 @@ void ValueImpl::FillSparseTensorBlockSparse(const OrtMemoryInfo* data_mem_inf } // namespace detail template -inline Value Value::CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count, - const int64_t* shape, size_t shape_len) { +inline Value Value::CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count, const int64_t* shape, size_t shape_len) { return CreateTensor(info, p_data, p_data_element_count * sizeof(T), shape, shape_len, TypeToTensorType::type); } -inline Value Value::CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, - const int64_t* shape, size_t shape_len, +inline Value Value::CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type) { OrtValue* out; ThrowOnError(GetApi().CreateTensorWithDataAsOrtValue(info, p_data, p_data_byte_count, shape, shape_len, type, &out)); return Value{out}; } -inline Value Value::CreateTensor(OrtAllocator* deleter, void* p_data, size_t p_data_byte_count, - const int64_t* shape, size_t shape_len, - ONNXTensorElementDataType type) { - OrtValue* out; - ThrowOnError(GetApi().CreateTensorWithDataAndDeleterAsOrtValue(deleter, p_data, p_data_byte_count, - shape, shape_len, type, &out)); - return Value{out}; -} - template inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len) { return CreateTensor(allocator, shape, shape_len, TypeToTensorType::type); } -inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, - ONNXTensorElementDataType type) { +inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type) { OrtValue* out; ThrowOnError(GetApi().CreateTensorAsOrtValue(allocator, shape, shape_len, type, &out)); return Value{out}; @@ -2378,8 +1570,7 @@ inline Value Value::CreateSparseTensor(const OrtMemoryInfo* info, void* p_data, const Shape& values_shape, ONNXTensorElementDataType type) { OrtValue* out; ThrowOnError(GetApi().CreateSparseTensorWithValuesAsOrtValue(info, p_data, dense_shape.shape, dense_shape.shape_len, - values_shape.shape, values_shape.shape_len, type, - &out)); + values_shape.shape, values_shape.shape_len, type, &out)); return Value{out}; } @@ -2541,172 +1732,6 @@ inline void KernelContext::ParallelFor(void (*fn)(void*, size_t), size_t total, ThrowOnError(GetApi().KernelContext_ParallelFor(ctx_, fn, total, num_batch, usr_data)); } -namespace detail { - -template -constexpr OrtOpAttrType TypeToAttrType(); - -template <> -inline constexpr OrtOpAttrType TypeToAttrType() { - return OrtOpAttrType::ORT_OP_ATTR_INT; -} - -template <> -inline constexpr OrtOpAttrType TypeToAttrType() { - return OrtOpAttrType::ORT_OP_ATTR_FLOAT; -} - -template -inline constexpr OrtOpAttrType TypeToAttrsType(); - -template <> -inline constexpr OrtOpAttrType TypeToAttrsType() { - return OrtOpAttrType::ORT_OP_ATTR_INTS; -} - -template <> -inline constexpr OrtOpAttrType TypeToAttrsType() { - return OrtOpAttrType::ORT_OP_ATTR_FLOATS; -} - -inline Status CheckAttrType(const OrtOpAttr* attr, OrtOpAttrType requested_type) { - OrtOpAttrType type; - Ort::Status status(GetApi().OpAttr_GetType(attr, &type)); - if (!status.IsOK()) return status; - if (requested_type != type) { - std::string msg = "Attribute type mismatch: expected " + std::to_string(requested_type) + - ", but got " + std::to_string(type); - return Ort::Status(msg.c_str(), OrtErrorCode::ORT_INVALID_ARGUMENT); - } - return Ort::Status{}; -} - -inline size_t GetDataSize(const OrtOpAttr* attr, OrtOpAttrType attr_type) { - size_t result{}; - // Ignore the status here because we check the data type so the error should only be about - // the size - [[maybe_unused]] Status status{GetApi().ReadOpAttr(attr, attr_type, nullptr, 0, &result)}; - return result; -} - -template -Ort::Status GetNumericValue(const OrtOpAttr* attr, T& out) { - static_assert(std::is_arithmetic::value, "T must be an arithmetic type"); - size_t size{}; - return Ort::Status{GetApi().ReadOpAttr(attr, TypeToAttrType(), &out, sizeof(out), &size)}; -} - -template -struct GetValueImpl { - static Status GetValue(const OrtOpAttr* attr, T& out) { - return GetNumericValue(attr, out); - } - static Status GetValues(const OrtOpAttr* attr, std::vector& out) { - // Api deficiency when it comes to value arrays. It is not possible - // to tell if the error is due to the type mismatch or the size - // so we check the type first, and then ignore the status of the size check - constexpr auto deduced_type = TypeToAttrsType(); - auto status = CheckAttrType(attr, deduced_type); - if (!status.IsOK()) return status; - auto size = GetDataSize(attr, deduced_type); - std::vector result; - if (size > 0) { - result.resize(size / sizeof(T)); - status = Status{GetApi().ReadOpAttr( - attr, deduced_type, result.data(), size, &size)}; - if (!status.IsOK()) return status; - } - out.swap(result); - return status; - } -}; - -// Create GetValueImpl specializations for std::string -template <> -struct GetValueImpl { - static Status GetValue(const OrtOpAttr* attr, std::string& out) { - // Api deficiency when it comes to value arrays. It is not possible - // to tell if the error is due to the type mismatch or the size - // so we check the type first, and then ignore the status of the size check - auto status = CheckAttrType(attr, OrtOpAttrType::ORT_OP_ATTR_STRING); - if (!status.IsOK()) return status; - auto size = GetDataSize(attr, OrtOpAttrType::ORT_OP_ATTR_STRING); - std::string result; - if (size > 0) { - result.resize(size); - // some compilers in use do not support std::string::data() non-const - auto* buffer = &result[0]; - status = Status{GetApi().ReadOpAttr( - attr, OrtOpAttrType::ORT_OP_ATTR_STRING, buffer, size, &size)}; - if (!status.IsOK()) return status; - } - out.swap(result); - return status; - } - static Status GetValues(const OrtOpAttr* attr, std::vector& out) { - auto status = CheckAttrType(attr, OrtOpAttrType::ORT_OP_ATTR_STRINGS); - if (!status.IsOK()) return status; - - std::vector result; - size_t total_buffer_size = GetDataSize(attr, OrtOpAttrType::ORT_OP_ATTR_STRINGS); - if (total_buffer_size > 0) { - // Create a temporary buffer to hold the string data - std::vector buffer(total_buffer_size); - status = Status{GetApi().ReadOpAttr(attr, OrtOpAttrType::ORT_OP_ATTR_STRINGS, buffer.data(), - total_buffer_size, &total_buffer_size)}; - if (!status.IsOK()) return status; - - const char* data = buffer.data(); - const char* end = data + total_buffer_size; - while (data < end) { - result.emplace_back(data); - data += result.back().size() + 1; // Move past the null terminator - } - } - out.swap(result); - return status; - } -}; - -template -template -inline Status ConstOpAttrImpl::GetValue(R& out) const { - return GetValueImpl::GetValue(this->p_, out); -} - -template -template -inline Status ConstOpAttrImpl::GetValueArray(std::vector& out) const { - return GetValueImpl::GetValues(this->p_, out); -} - -template -inline Status ConstOpAttrImpl::GetTensorAttributeAsOrtValue(Value& out) const { - OrtValue* tensor_value = nullptr; - auto status = Status(GetApi().OpAttr_GetTensorAttributeAsOrtValue(this->p_, &tensor_value)); - if (!status.IsOK()) return status; - out = Value{tensor_value}; - return status; -} - -template -inline std::string ConstOpAttrImpl::GetName() const { - const char* name = nullptr; - ThrowOnError(GetApi().OpAttr_GetName(this->p_, &name)); - if (name != nullptr) { - return name; - } - return {}; -} - -template -inline OrtOpAttrType ConstOpAttrImpl::GetType() const { - OrtOpAttrType type; - ThrowOnError(GetApi().OpAttr_GetType(this->p_, &type)); - return type; -} -} // namespace detail - inline OpAttr::OpAttr(const char* name, const void* data, int len, OrtOpAttrType type) { Ort::ThrowOnError(GetApi().CreateOpAttr(name, data, len, type, &p_)); } @@ -2859,7 +1884,7 @@ inline void attr_utils::GetAttrs(const OrtKernelInfo* p, const char* name, std:: inline KernelInfo::KernelInfo(OrtKernelInfo* info) : detail::KernelInfoImpl{info} {} -inline Op::Op(OrtOp* p) : detail::Base(p) {} +inline Op::Op(OrtOp* p) : Base(p) {} inline Op Op::Create(const OrtKernelInfo* info, const char* op_name, const char* domain, int version, const char** type_constraint_names, @@ -2957,9 +1982,7 @@ inline ShapeInferContext::ShapeInferContext(const OrtApi* ort_api, TensorTypeAndShapeInfo type_shape_info(info); auto integer_shape = type_shape_info.GetShape(); std::vector symbolic_shape(integer_shape.size(), {}); - if (!integer_shape.empty()) { - type_shape_info.GetSymbolicDimensions(&symbolic_shape[0], integer_shape.size()); - } + type_shape_info.GetSymbolicDimensions(&symbolic_shape[0], integer_shape.size()); Shape shape; for (size_t ith = 0; ith < integer_shape.size(); ++ith) { if (symbolic_shape[ith] && std::string{symbolic_shape[ith]}.size() > 0) { @@ -2973,10 +1996,9 @@ inline ShapeInferContext::ShapeInferContext(const OrtApi* ort_api, } } -inline Status ShapeInferContext::SetOutputShape(size_t indice, const Shape& shape, ONNXTensorElementDataType type) { +inline Status ShapeInferContext::SetOutputShape(size_t indice, const Shape& shape) { OrtTensorTypeAndShapeInfo* info = {}; ORT_CXX_RETURN_ON_API_FAIL(ort_api_->CreateTensorTypeAndShapeInfo(&info)); - ORT_CXX_RETURN_ON_API_FAIL(ort_api_->SetTensorElementType(info, type)); using InfoPtr = std::unique_ptr>; @@ -2989,7 +2011,7 @@ inline Status ShapeInferContext::SetOutputShape(size_t indice, const Shape& shap for (const auto dim : shape) { if (dim.IsInt()) { - integer_dims.push_back(dim.AsInt()); + integer_dims.push_back(dim.IsInt()); symbolic_dims.push_back(""); } else { if (!dim.AsSym() || std::string{dim.AsSym()}.empty()) { @@ -3007,541 +2029,97 @@ inline Status ShapeInferContext::SetOutputShape(size_t indice, const Shape& shap } inline int64_t ShapeInferContext::GetAttrInt(const char* attr_name) { - auto attr = GetAttrHdl(attr_name); - int64_t value; - Status status = attr.GetValue(value); - if (!status.IsOK()) { - ORT_CXX_API_THROW("Getting int attribute failed: " + status.GetErrorMessage(), status.GetErrorCode()); - } - return value; + const auto* attr = GetAttrHdl(attr_name); + int64_t i = {}; + size_t out = {}; + Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_INT, &i, sizeof(i), &out)); + return i; } inline ShapeInferContext::Ints ShapeInferContext::GetAttrInts(const char* attr_name) { - auto attr = GetAttrHdl(attr_name); - ShapeInferContext::Ints result; - auto status = attr.GetValueArray(result); - if (!status.IsOK()) { - ORT_CXX_API_THROW("Getting ints attribute failed: " + status.GetErrorMessage(), status.GetErrorCode()); + const auto* attr = GetAttrHdl(attr_name); + int64_t i = {}; + size_t out = {}; + // first call to get the bytes needed + auto status = ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_INTS, &i, sizeof(i), &out); + if (status) { + size_t num_i = out / sizeof(int64_t); + ShapeInferContext::Ints ints(num_i, 0); + Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_INTS, ints.data(), out, &out)); + return ints; + } else { + return {i}; } - return result; } inline float ShapeInferContext::GetAttrFloat(const char* attr_name) { - auto attr = GetAttrHdl(attr_name); - float value; - Status status = attr.GetValue(value); - if (!status.IsOK()) { - ORT_CXX_API_THROW("Getting float attribute failed: " + status.GetErrorMessage(), status.GetErrorCode()); - } - return value; + const auto* attr = GetAttrHdl(attr_name); + float f = {}; + size_t out = {}; + Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_FLOAT, &f, sizeof(f), &out)); + return f; } inline ShapeInferContext::Floats ShapeInferContext::GetAttrFloats(const char* attr_name) { - auto attr = GetAttrHdl(attr_name); - ShapeInferContext::Floats result; - auto status = attr.GetValueArray(result); - if (!status.IsOK()) { - ORT_CXX_API_THROW("Getting floats attribute failed: " + status.GetErrorMessage(), status.GetErrorCode()); + const auto* attr = GetAttrHdl(attr_name); + float f = {}; + size_t out = {}; + // first call to get the bytes needed + auto status = ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_FLOATS, &f, sizeof(f), &out); + if (status) { + size_t num_f = out / sizeof(float); + ShapeInferContext::Floats floats(num_f, 0); + Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_FLOATS, floats.data(), out, &out)); + return floats; + } else { + return {f}; } - return result; } inline std::string ShapeInferContext::GetAttrString(const char* attr_name) { - auto attr = GetAttrHdl(attr_name); - std::string value; - Status status = attr.GetValue(value); - if (!status.IsOK()) { - ORT_CXX_API_THROW("Getting string attribute failed: " + status.GetErrorMessage(), status.GetErrorCode()); + const auto* attr = GetAttrHdl(attr_name); + char c = {}; + size_t out = {}; + // first call to get the bytes needed + auto status = ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_STRING, &c, sizeof(char), &out); + if (status) { + std::vector chars(out, '\0'); + Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_STRING, chars.data(), out, &out)); + return {chars.data()}; + } else { + return {c}; } - return value; } inline ShapeInferContext::Strings ShapeInferContext::GetAttrStrings(const char* attr_name) { - auto attr = GetAttrHdl(attr_name); - ShapeInferContext::Strings result; - auto status = attr.GetValueArray(result); - if (!status.IsOK()) { - ORT_CXX_API_THROW("Getting strings attribute failed: " + status.GetErrorMessage(), status.GetErrorCode()); + const auto* attr = GetAttrHdl(attr_name); + char c = {}; + size_t out = {}; + // first call to get the bytes needed + auto status = ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_STRINGS, &c, sizeof(char), &out); + if (status) { + std::vector chars(out, '\0'); + Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_STRINGS, chars.data(), out, &out)); + ShapeInferContext::Strings strings; + char* char_st = chars.data(); + char* char_ed = char_st + out; + while (char_st < char_ed) { + strings.emplace_back(char_st); + while (*char_st != '\0') { + char_st++; + } + char_st++; + } + return strings; + } else { + return {std::string{c}}; } - return result; } -inline ConstOpAttr ShapeInferContext::GetAttrHdl(const char* attr_name) const { +inline const OrtOpAttr* ShapeInferContext::GetAttrHdl(const char* attr_name) const { const OrtOpAttr* attr_hdl = {}; Ort::ThrowOnError(ort_api_->ShapeInferContext_GetAttribute(ctx_, attr_name, &attr_hdl)); - return ConstOpAttr{attr_hdl}; + return attr_hdl; } -namespace detail { -inline std::vector StringsToCharPtrs(const std::vector& strings) { - std::vector ptrs; - ptrs.reserve(strings.size()); - std::transform(strings.begin(), strings.end(), std::back_inserter(ptrs), - [](const std::string& s) { return s.c_str(); }); - - return ptrs; -} -} // namespace detail - -namespace detail { -template -inline size_t ConstNodeImpl::GetId() const { - size_t id; - ThrowOnError(GetApi().Node_GetId(this->p_, &id)); - return id; -} - -template -inline std::string ConstNodeImpl::GetName() const { - const char* name; - ThrowOnError(GetApi().Node_GetName(this->p_, &name)); - return std::string(name); -} - -template -inline std::string ConstNodeImpl::GetOperatorType() const { - const char* type; - ThrowOnError(GetApi().Node_GetOperatorType(this->p_, &type)); - return std::string(type); -} - -template -inline std::string ConstNodeImpl::GetDomain() const { - const char* domain; - ThrowOnError(GetApi().Node_GetDomain(this->p_, &domain)); - return std::string(domain); -} - -template -inline int ConstNodeImpl::GetSinceVersion() const { - int since_version; - ThrowOnError(GetApi().Node_GetSinceVersion(this->p_, &since_version)); - return since_version; -} - -template -inline std::vector ConstNodeImpl::GetInputs() const { - static_assert(sizeof(const OrtValueInfo*) == sizeof(ConstValueInfo)); - size_t num_vi; - ThrowOnError(GetApi().Node_GetNumInputs(this->p_, &num_vi)); - std::vector result; - if (num_vi > 0) { - result.resize(num_vi); - ThrowOnError(GetApi().Node_GetInputs(this->p_, reinterpret_cast(result.data()), num_vi)); - } - return result; -} - -template -inline std::vector ConstNodeImpl::GetOutputs() const { - static_assert(sizeof(const OrtValueInfo*) == sizeof(ConstValueInfo)); - size_t num_vi; - ThrowOnError(GetApi().Node_GetNumOutputs(this->p_, &num_vi)); - std::vector result; - if (num_vi > 0) { - result.resize(num_vi); - ThrowOnError(GetApi().Node_GetOutputs(this->p_, reinterpret_cast(result.data()), num_vi)); - } - return result; -} - -template -inline std::vector ConstNodeImpl::GetImplicitInputs() const { - static_assert(sizeof(const OrtValueInfo*) == sizeof(ConstValueInfo)); - size_t num_vi; - ThrowOnError(GetApi().Node_GetNumImplicitInputs(this->p_, &num_vi)); - std::vector result; - if (num_vi > 0) { - result.resize(num_vi); - ThrowOnError(GetApi().Node_GetImplicitInputs(this->p_, reinterpret_cast(result.data()), - num_vi)); - } - return result; -} - -template -inline std::vector ConstNodeImpl::GetAttributes() const { - static_assert(sizeof(const OrtOpAttr*) == sizeof(ConstOpAttr), "Must be the same size"); - size_t num_attrs; - ThrowOnError(GetApi().Node_GetNumAttributes(this->p_, &num_attrs)); - std::vector attrs; - if (num_attrs > 0) { - attrs.resize(num_attrs); - ThrowOnError(GetApi().Node_GetAttributes(this->p_, reinterpret_cast(attrs.data()), num_attrs)); - } - return attrs; -} - -template -inline Status ConstNodeImpl::GetAttributeByName(const std::string& name, ConstOpAttr& out) const { - const OrtOpAttr* attr = nullptr; - auto status = Status(GetApi().Node_GetAttributeByName(this->p_, name.c_str(), &attr)); - out = ConstOpAttr{attr}; - return status; -} - -template -inline std::vector ConstNodeImpl::GetSubgraphs() const { - size_t num_graphs; - ThrowOnError(GetApi().Node_GetNumSubgraphs(this->p_, &num_graphs)); - std::vector result; - if (num_graphs > 0) { - std::vector sub_graphs(num_graphs); - std::vector attr_names(num_graphs); - ThrowOnError(GetApi().Node_GetSubgraphs(this->p_, sub_graphs.data(), num_graphs, attr_names.data())); - result.reserve(num_graphs); - for (size_t i = 0; i < num_graphs; ++i) { - result.push_back({std::string(attr_names[i]), ConstGraph{sub_graphs[i]}}); - } - } - return result; -} - -template -inline ConstGraph ConstNodeImpl::GetGraph() const { - const OrtGraph* graph; - ThrowOnError(GetApi().Node_GetGraph(this->p_, &graph)); - return ConstGraph{graph}; -} - -template -inline std::string ConstNodeImpl::GetEpName() const { - const char* name; - ThrowOnError(GetApi().Node_GetEpName(this->p_, &name)); - return std::string(name); -} - -} // namespace detail - -#if !defined(ORT_MINIMAL_BUILD) -// static -inline void Node::Init(const std::string& operator_name, const std::string& operator_domain, - const std::string& node_name, - const std::vector& input_names, - const std::vector& output_names, - std::vector& attributes, - OrtNode*& node) { - auto inputs = detail::StringsToCharPtrs(input_names); - auto outputs = detail::StringsToCharPtrs(output_names); - - std::vector attributes_ptrs; - attributes_ptrs.reserve(attributes.size()); - std::transform(attributes.begin(), attributes.end(), std::back_inserter(attributes_ptrs), - [](OpAttr& attr) -> OrtOpAttr* { return attr; }); - - ThrowOnError(GetModelEditorApi().CreateNode(operator_name.c_str(), operator_domain.c_str(), node_name.c_str(), - inputs.data(), inputs.size(), - outputs.data(), outputs.size(), - attributes_ptrs.data(), attributes_ptrs.size(), - &node)); - - // Node now owns the attributes - std::for_each(attributes.begin(), attributes.end(), [](OpAttr& attr) { attr.release(); }); -} - -inline Node::Node(const std::string& operator_name, const std::string& operator_domain, - const std::string& node_name, - const std::vector& input_names, - const std::vector& output_names, - std::vector& attributes) { - Init(operator_name, operator_domain, node_name, input_names, output_names, attributes, p_); -} - -inline Node::Node(const std::string& operator_name, const std::string& operator_domain, - const std::string& node_name, - const std::vector& input_names, - const std::vector& output_names) { - std::vector empty_attributes; - Init(operator_name, operator_domain, node_name, input_names, output_names, empty_attributes, p_); -} -inline ValueInfo::ValueInfo(const std::string& name, const ConstTypeInfo& type_info) { - ThrowOnError(GetModelEditorApi().CreateValueInfo(name.c_str(), type_info, &p_)); -} -#endif // !defined(ORT_MINIMAL_BUILD) - -namespace detail { -template -inline std::string ConstValueInfoImpl::GetName() const { - const char* p = nullptr; - ThrowOnError(GetApi().GetValueInfoName(this->p_, &p)); - return std::string(p); -} - -template -inline ConstTypeInfo ConstValueInfoImpl::TypeInfo() const { - const OrtTypeInfo* type_info = nullptr; - ThrowOnError(GetApi().GetValueInfoTypeInfo(this->p_, &type_info)); - return ConstTypeInfo{type_info}; -} - -template -inline ValueInfoConsumerProducerInfo ConstValueInfoImpl::GetProducerNode() const { - ValueInfoConsumerProducerInfo info; - const OrtNode* producer; - size_t index; - ThrowOnError(GetApi().ValueInfo_GetValueProducer(this->p_, &producer, &index)); - info.node = ConstNode(producer); - info.index = static_cast(index); - return info; -} - -template -inline std::vector ConstValueInfoImpl::GetConsumers() const { - size_t num = 0; - ThrowOnError(GetApi().ValueInfo_GetValueNumConsumers(this->p_, &num)); - std::vector out; - if (num > 0) { - std::vector nodes(num); - std::vector indices(num); - ThrowOnError(GetApi().ValueInfo_GetValueConsumers(this->p_, nodes.data(), indices.data(), num)); - out.reserve(num); - for (size_t i = 0; i < num; ++i) { - out.push_back({ConstNode{nodes[i]}, indices[i]}); - } - } - return out; -} - -template -inline Status ConstValueInfoImpl::GetInitializer(ConstValue& value) const { - const OrtValue* out = nullptr; - auto status = Status(GetApi().ValueInfo_GetInitializerValue(this->p_, &out)); - if (!status.IsOK()) return status; - value = ConstValue{out}; - return status; -} - -template -inline Status ConstValueInfoImpl::GetExternalInitializerInfo(ExternalInitializerInfo& info) const { - OrtExternalInitializerInfo* out = nullptr; - auto status = Status(GetApi().ValueInfo_GetExternalInitializerInfo(this->p_, &out)); - if (!status.IsOK()) return status; - info = ExternalInitializerInfo{out}; - return status; -} - -template -inline bool ConstValueInfoImpl::IsRequiredGraphInput() const { - bool out = false; - ThrowOnError(GetApi().ValueInfo_IsRequiredGraphInput(this->p_, &out)); - return out; -} - -template -inline bool ConstValueInfoImpl::IsOptionalGraphInput() const { - bool out = false; - ThrowOnError(GetApi().ValueInfo_IsOptionalGraphInput(this->p_, &out)); - return out; -} - -template -inline bool ConstValueInfoImpl::IsGraphOutput() const { - bool out = false; - ThrowOnError(GetApi().ValueInfo_IsGraphOutput(this->p_, &out)); - return out; -} - -template -inline bool ConstValueInfoImpl::IsConstantInitializer() const { - bool out = false; - ThrowOnError(GetApi().ValueInfo_IsConstantInitializer(this->p_, &out)); - return out; -} - -template -inline bool ConstValueInfoImpl::IsFromOuterScope() const { - bool out = false; - ThrowOnError(GetApi().ValueInfo_IsFromOuterScope(this->p_, &out)); - return out; -} - -template -inline ModelMetadata ConstGraphImpl::GetModelMetadata() const { - OrtModelMetadata* out; - ThrowOnError(GetApi().Graph_GetModelMetadata(this->p_, &out)); - return ModelMetadata{out}; -} - -template -inline std::string ConstGraphImpl::GetName() const { - const char* name; - ThrowOnError(GetApi().Graph_GetName(this->p_, &name)); - return std::string(name); -} - -template -inline std::basic_string ConstGraphImpl::GetModelPath() const { - const ORTCHAR_T* path; - ThrowOnError(GetApi().Graph_GetModelPath(this->p_, &path)); - return std::basic_string(path); -} - -template -inline int64_t ConstGraphImpl::GetOnnxIRVersion() const { - int64_t version; - ThrowOnError(GetApi().Graph_GetOnnxIRVersion(this->p_, &version)); - return version; -} - -template -inline std::vector ConstGraphImpl::GetOperatorSets() const { - size_t num_opsets; - ThrowOnError(GetApi().Graph_GetNumOperatorSets(this->p_, &num_opsets)); - std::vector result; - if (num_opsets > 0) { - std::vector domains; - std::vector versions; - domains.resize(num_opsets); - versions.resize(num_opsets); - ThrowOnError(GetApi().Graph_GetOperatorSets(this->p_, domains.data(), versions.data(), num_opsets)); - result.reserve(num_opsets); - for (size_t i = 0; i < num_opsets; ++i) { - result.push_back({domains[i], versions[i]}); - } - } - return result; -} - -template -inline std::vector ConstGraphImpl::GetInputs() const { - static_assert(sizeof(const OrtValueInfo*) == sizeof(ConstValueInfo)); - size_t num_vi; - ThrowOnError(GetApi().Graph_GetNumInputs(this->p_, &num_vi)); - std::vector result; - if (num_vi > 0) { - result.resize(num_vi); - ThrowOnError(GetApi().Graph_GetInputs(this->p_, reinterpret_cast(result.data()), num_vi)); - } - return result; -} - -template -inline std::vector ConstGraphImpl::GetOutputs() const { - static_assert(sizeof(const OrtValueInfo*) == sizeof(ConstValueInfo)); - size_t num_vi; - ThrowOnError(GetApi().Graph_GetNumOutputs(this->p_, &num_vi)); - std::vector result; - if (num_vi > 0) { - result.resize(num_vi); - ThrowOnError(GetApi().Graph_GetOutputs(this->p_, reinterpret_cast(result.data()), num_vi)); - } - return result; -} - -template -inline std::vector ConstGraphImpl::GetInitializers() const { - static_assert(sizeof(const OrtValueInfo*) == sizeof(ConstValueInfo)); - size_t num_vi; - ThrowOnError(GetApi().Graph_GetNumInitializers(this->p_, &num_vi)); - std::vector result; - if (num_vi > 0) { - result.resize(num_vi); - ThrowOnError(GetApi().Graph_GetInitializers(this->p_, reinterpret_cast(result.data()), - num_vi)); - } - return result; -} - -template -inline std::vector ConstGraphImpl::GetNodes() const { - static_assert(sizeof(const OrtNode*) == sizeof(ConstNode)); - size_t num_nodes; - ThrowOnError(GetApi().Graph_GetNumNodes(this->p_, &num_nodes)); - std::vector result; - if (num_nodes > 0) { - result.resize(num_nodes); - ThrowOnError(GetApi().Graph_GetNodes(this->p_, reinterpret_cast(result.data()), num_nodes)); - } - return result; -} - -template -inline ConstNode ConstGraphImpl::GetParentNode() const { - const OrtNode* parent; - ThrowOnError(GetApi().Graph_GetParentNode(this->p_, &parent)); - return ConstNode{parent}; -} - -template -inline Graph ConstGraphImpl::GetGraphView(const std::vector& nodes) const { - OrtGraph* graph_viewer; - std::vector inputs_ptrs; - inputs_ptrs.reserve(nodes.size()); - std::transform(nodes.begin(), nodes.end(), std::back_inserter(inputs_ptrs), - [](ConstNode n) -> const OrtNode* { return n; }); - ThrowOnError(GetApi().Graph_GetGraphView(this->p_, inputs_ptrs.data(), - nodes.size(), &graph_viewer)); - return Graph{graph_viewer}; -} - -#if !defined(ORT_MINIMAL_BUILD) -template -inline void GraphImpl::SetInputs(std::vector& inputs) { - std::vector inputs_ptrs; - inputs_ptrs.reserve(inputs.size()); - std::transform(inputs.begin(), inputs.end(), std::back_inserter(inputs_ptrs), - [](ValueInfo& vi) -> OrtValueInfo* { return vi; }); - - ThrowOnError(GetModelEditorApi().SetGraphInputs(this->p_, inputs_ptrs.data(), inputs_ptrs.size())); - - // Graph now owns the inputs - std::for_each(inputs.begin(), inputs.end(), [](ValueInfo& vi) { vi.release(); }); -} - -template -inline void GraphImpl::SetOutputs(std::vector& outputs) { - std::vector outputs_ptrs; - outputs_ptrs.reserve(outputs.size()); - std::transform(outputs.begin(), outputs.end(), std::back_inserter(outputs_ptrs), - [](ValueInfo& vi) -> OrtValueInfo* { return vi; }); - - ThrowOnError(GetModelEditorApi().SetGraphOutputs(this->p_, outputs_ptrs.data(), outputs_ptrs.size())); - - // Graph now owns the outputs - std::for_each(outputs.begin(), outputs.end(), [](ValueInfo& vi) { vi.release(); }); -} - -template -inline void GraphImpl::AddInitializer(const std::string& name, Value& initializer, bool data_is_external) { - // Graph takes ownership of `initializer` - // On error the ownership is not transferred. - ThrowOnError(GetModelEditorApi().AddInitializerToGraph(this->p_, name.c_str(), initializer, data_is_external)); - initializer.release(); -} - -template -inline void GraphImpl::AddNode(Node& node) { - // Graph takes ownership of `node` - ThrowOnError(GetModelEditorApi().AddNodeToGraph(this->p_, node.release())); -} - -template -inline void ModelImpl::AddGraph(Graph& graph) { - // Model takes ownership of `graph` - ThrowOnError(GetModelEditorApi().AddGraphToModel(this->p_, graph.release())); -} -#endif // !defined(ORT_MINIMAL_BUILD) - -} // namespace detail - -#if !defined(ORT_MINIMAL_BUILD) -inline Graph::Graph() { - ThrowOnError(GetModelEditorApi().CreateGraph(&p_)); -} - -inline Model::Model(const std::vector& opsets) { - std::vector domains; - std::vector versions; - domains.reserve(opsets.size()); - versions.reserve(opsets.size()); - - for (const auto& pair : opsets) { - domains.push_back(pair.first.c_str()); - versions.push_back(pair.second); - } - - ThrowOnError(GetModelEditorApi().CreateModel(domains.data(), versions.data(), opsets.size(), &p_)); -} -#endif - } // namespace Ort diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_float16.h b/prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_float16.h similarity index 98% rename from prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_float16.h rename to prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_float16.h index 408d3cc..0b066a9 100644 --- a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_float16.h +++ b/prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_float16.h @@ -75,6 +75,8 @@ struct Float16Impl { static constexpr uint16_t kNegativeInfinityBits = 0xFC00U; static constexpr uint16_t kPositiveQNaNBits = 0x7E00U; static constexpr uint16_t kNegativeQNaNBits = 0xFE00U; + static constexpr uint16_t kEpsilonBits = 0x4170U; + static constexpr uint16_t kMinValueBits = 0xFBFFU; // Minimum normal number static constexpr uint16_t kMaxValueBits = 0x7BFFU; // Largest normal number static constexpr uint16_t kOneBits = 0x3C00U; static constexpr uint16_t kMinusOneBits = 0xBC00U; @@ -362,6 +364,9 @@ struct BFloat16Impl { static constexpr uint16_t kNegativeInfinityBits = 0xFF80U; static constexpr uint16_t kPositiveQNaNBits = 0x7FC1U; static constexpr uint16_t kNegativeQNaNBits = 0xFFC1U; + static constexpr uint16_t kSignaling_NaNBits = 0x7F80U; + static constexpr uint16_t kEpsilonBits = 0x0080U; + static constexpr uint16_t kMinValueBits = 0xFF7FU; static constexpr uint16_t kMaxValueBits = 0x7F7FU; static constexpr uint16_t kRoundToNearest = 0x7FFFU; static constexpr uint16_t kOneBits = 0x3F80U; diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_lite_custom_op.h b/prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_lite_custom_op.h similarity index 99% rename from prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_lite_custom_op.h rename to prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_lite_custom_op.h index 5002e16..ee60f25 100644 --- a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_lite_custom_op.h +++ b/prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_lite_custom_op.h @@ -29,7 +29,7 @@ class ArgBase { ArgBase(OrtKernelContext* ctx, size_t indice, bool is_input) : ctx_(ctx), indice_(indice), is_input_(is_input) {} - virtual ~ArgBase() {}; + virtual ~ArgBase(){}; protected: struct KernelContext ctx_; @@ -361,7 +361,7 @@ struct TensorArray : public ArgBase { tensor = std::make_unique>(ctx, ith_input, true); break; default: - ORT_CXX_API_THROW("unknown input type", ORT_RUNTIME_EXCEPTION); + ORT_CXX_API_THROW("unknow input type", ORT_RUNTIME_EXCEPTION); break; } tensors_.emplace_back(tensor.release()); @@ -403,7 +403,7 @@ using Variadic = TensorArray; Note: OrtLiteCustomOp inherits from OrtCustomOp to bridge tween a custom func/struct and ort core. The lifetime of an OrtLiteCustomOp instance is managed by customer code, not ort, so: -1. DO NOT cast OrtLiteCustomOp to OrtCustomOp and release since there is no virtual destructor in the hierarchy. +1. DO NOT cast OrtLiteCustomOp to OrtCustomOp and release since there is no virtual destructor in the hierachy. 2. OrtLiteCustomFunc and OrtLiteCustomStruct, as two sub-structs, can be released in form of OrtLiteCustomOp since all members are kept in the OrtLiteCustomOp, hence memory could still be recycled properly. Further, OrtCustomOp is a c struct bearing no v-table, so offspring structs are by design to be of zero virtual functions to maintain cast safety. diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_run_options_config_keys.h b/prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_run_options_config_keys.h similarity index 95% rename from prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_run_options_config_keys.h rename to prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_run_options_config_keys.h index f40ea65..c80b8c0 100644 --- a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_run_options_config_keys.h +++ b/prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_run_options_config_keys.h @@ -43,9 +43,6 @@ static const char* const kOrtRunOptionsConfigQnnPerfModePostRun = "qnn.htp_perf_ // Set RPC control latency for QNN HTP backend static const char* const kOrtRunOptionsConfigQnnRpcControlLatency = "qnn.rpc_control_latency"; -// Set QNN Lora Config File for apply Lora in QNN context binary -static const char* const kOrtRunOptionsConfigQnnLoraConfig = "qnn.lora_config"; - // Set graph annotation id for CUDA EP. Use with enable_cuda_graph=true. // The value should be an integer. If the value is not set, the default value is 0 and // ORT session only captures one cuda graph before another capture is requested. diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_session_options_config_keys.h b/prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_session_options_config_keys.h similarity index 61% rename from prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_session_options_config_keys.h rename to prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_session_options_config_keys.h index 64a434e..bb5e034 100644 --- a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_session_options_config_keys.h +++ b/prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_session_options_config_keys.h @@ -10,10 +10,10 @@ * "[Area][.[SubArea1].[SubArea2]...].[Keyname]" * Such as "ep.cuda.use_arena" * The Config Key cannot be empty - * The maximum length of the Config Key is 1024 + * The maximum length of the Config Key is 128 * * The string format of a SessionOptions Config Value is defined individually for each Config. - * The maximum length of the Config Value is 2048 + * The maximum length of the Config Value is 1024 */ // Key for disable PrePacking, @@ -67,10 +67,6 @@ static const char* const kOrtSessionOptionsEnableQuantQDQCleanup = "session.enab // GeluApproximation has side effects which may change the inference results. It is disabled by default due to this. static const char* const kOrtSessionOptionsEnableGeluApproximation = "optimization.enable_gelu_approximation"; -// Enable or disable Cast chain elimination in graph optimization. "0": disable; "1": enable. The default is "0". -// CastElimination with chain elimination has side effects which may change the inference results. It is disabled by default due to this. -static const char* const kOrtSessionOptionsEnableCastChainElimination = "optimization.enable_cast_chain_elimination"; - // This setting controls whether to enable AheadOfTime function inlining. // AOT function inlining examines the graph and attempts to inline as many locally defined functions in the model // as possible with the help of enabled execution providers. @@ -82,20 +78,15 @@ static const char* const kOrtSessionOptionsEnableCastChainElimination = "optimiz static const char* const kOrtSessionOptionsDisableAheadOfTimeFunctionInlining = "session.disable_aot_function_inlining"; #ifdef ENABLE_TRAINING -// Specifies a path of the file containing a list of memory optimization configurations. -// The value should be a string indicating the file path of the config file. -// The content of the config file is a JSON struct like this: -// [ -// "Gelu+Cast+:1:0", -// "Dropout+:1:1" -// ] -// Taking the example of "Gelu+Cast+:1:0", -// > "Gelu+Cast+" is the subgraph string, a valid "subgraph string" should be one subgraph representation -// output by ORT graph transformations. -// > "1" is "optimization strategy", valid values: 0 - disabled, 1 - recompute. -// > "0" is "number of subgraph to apply" which is used to control how many subgraphs to apply optimization, -// to avoid "oversaving" the memory. -static const char* const kOrtSessionOptionsMemoryOptimizerApplyConfig = "optimization.memory_optimizer_config"; +// Specifies a list of op types for memory footprint reduction. +// The value should be a ","-delimited list of pair of +// . +// For example, "Gelu+Cast+:1:0,Dropout+:1:1". +// A valid "subgraph string" should be one subgraph representation output by ORT graph transformations. +// "optimization strategy" currently has valid values: 0 - disabled, 1 - recompute. +// "number of subgraph to apply" is used to control how many subgraphs to apply optimization, to avoid "oversaving" +// the memory. +static const char* const kOrtSessionOptionsMemoryOptimizerEnabler = "optimization.memory_optimizer_config"; // Specifies the config for detecting subgraphs for memory footprint reduction. // The value should be a string contains int separated using commas. The default value is "0:0". @@ -111,46 +102,13 @@ static const char* const kOrtSessionOptionsMemoryOptimizerProbeConfig = "optimiz // Default is an empty string which means no optimizers are disabled. static const char* const kOrtSessionOptionsDisableSpecifiedOptimizers = "optimization.disable_specified_optimizers"; -// It controls whether to run graph optimizations in loop or not. -// -// "0": disable. Graph Optimization Loop is disabled. -// ``` -// Level 2 --> Level 3 --> InsertCastTransforms --> Level 4 -// ^ | -// | "No Loop" | -// | | -// X xxxxxxxxxxx X -// ``` -// "1": enable. Graph Optimization Loop is enabled, such that, if optimizations at Level 4 are applied then -// the loop will check for any other valid optimization that can happen. -// ``` -// Level 2 --> Level 3 --> InsertCastTransforms --> Level 4 -// ^ | -// | "Loop only depending on Level 4" | -// | | -// --------------------------------------------------- -// ``` -// "2": enable. Graph Optimization Loop is enabled, such that, if optimizations at Level 2 or above are applied then -// The loop will check for any other valid optimization that can happen. -// ``` -// Level 2 --> Level 3 --> InsertCastTransforms --> Level 4 -// ^ | -// | "Loop" | -// | | -// --------------------------------------------------- -// ``` -// Default value is set to "1". -static const char* const kOrtSessionOptionsGraphOptimizationsLoopLevel = "session.graph_optimizations_loop_level"; - // Enable or disable using device allocator for allocating initialized tensor memory. "1": enable; "0": disable. The default is "0". // Using device allocators means the memory allocation is made using malloc/new. static const char* const kOrtSessionOptionsUseDeviceAllocatorForInitializers = "session.use_device_allocator_for_initializers"; // Configure whether to allow the inter_op/intra_op threads spinning a number of times before blocking // "0": thread will block if found no job to run -// "1": thread will spin a number of times before blocking -// The default is "0" when ORT is built with "ORT_CLIENT_PACKAGE_BUILD" and "1" otherwise. -// Thread spinning is disabled by default for client/on-device workloads to reduce cpu utilization and improve power efficiency. +// "1": default, thread will spin a number of times before blocking static const char* const kOrtSessionOptionsConfigAllowInterOpSpinning = "session.inter_op.allow_spinning"; static const char* const kOrtSessionOptionsConfigAllowIntraOpSpinning = "session.intra_op.allow_spinning"; @@ -287,51 +245,6 @@ static const char* const kOrtSessionOptionsOptimizedModelExternalInitializersFil static const char* const kOrtSessionOptionsOptimizedModelExternalInitializersMinSizeInBytes = "session.optimized_model_external_initializers_min_size_in_bytes"; -// When loading model from memory buffer and the model has external initializers -// Use this config to set the external data file folder path -// All external data files should be in the same folder -static const char* const kOrtSessionOptionsModelExternalInitializersFileFolderPath = - "session.model_external_initializers_file_folder_path"; - -// Use this config when saving pre-packed constant initializers to an external data file. -// This allows you to memory map pre-packed initializers on model load and leave it to -// to the OS the amount of memory consumed by the pre-packed initializers. Otherwise, -// pre-packed data resides on the heap. -// -// - "0": Default is not save pre-packed initializers to a data file. -// - "1": Save pre-packed constant initializers to an external data file. -// Sample usage: sess_options.add_session_config_entry(kOrtSessionOptionsSavePrePackedConstantInitializers, "1") -static const char* const kOrtSessionOptionsSavePrePackedConstantInitializers = - "session.save_external_prepacked_constant_initializers"; - -// Use this config when you want to collect memory stats for each node in the graph. -// The file format is a CSV file with the following columns: -// The file will be created if it does not exist, and will be overwritten if it does. -// -// The content of the file can be used to estimate memory requirements at run time including -// the temporary allocations. This operation is preferably done on a CPU device, as the model may exceed -// device memory limits in constrained environments. When enabling this option, it is important to disable -// memory patterns, as they tend to allocate large blocks to avoid fragmentation and accommodate needs of multiple -// kernels. Memory patterns may make it difficult to allocate on a device with limited memory. -// -// The collected stats then can be used to partition the graph among the devices in a way that only the -// required memory is allocated on each device. -// -// node_name, initializers_memory, dynamic_outputs_sizes, temp_allocations_size -// -// - "full path to file": there is not a default for this option. If the file can not be opened for writing, an error will be returned. -static const char* const kOrtSessionOptionsCollectNodeMemoryStatsToFile = "session.collect_node_memory_stats_to_file"; - -/// This is a composite CSV setting formatted as "memory limit in kb,file name for collected stats" -/// "limit > 0": enables Capacity Aware Partitioning for Cuda EP. `limit` is optional and when absent -/// the provider may attempt to figure out the memory available automatically. -/// The setting with no limit is expected to look like: ",file name for collected stats" -/// The EP will place nodes on device "file name" : -/// this file is expected to be found at the same folder with the model. The file contains -/// pre-recorded stats collected when running with kOrtSessionOptionsCollectNodeMemoryStatsToFile enforce (see above) -static const char* const kOrtSessionOptionsResourceCudaPartitioningSettings = - "session.resource_cuda_partitioning_settings"; - // Enable EP context feature to dump the partitioned graph which includes the EP context into Onnx file. // The dumped Onnx model with EP context can be used for future inference to avoid the EP graph partitioning/compile overhead. // "0": disable. (default) @@ -340,78 +253,15 @@ static const char* const kOrtSessionOptionEpContextEnable = "ep.context_enable"; // Specify the file path for the Onnx model which has EP context. // Default to original_file_name_ctx.onnx if not specified -// Folder is not a valid option static const char* const kOrtSessionOptionEpContextFilePath = "ep.context_file_path"; // Flag to specify whether to dump the EP context into the Onnx model. -// "0": dump the EP context into separate file, keep the file name in the Onnx model. (default). -// "1": dump the EP context into the Onnx model. +// "0": dump the EP context into separate file, keep the file name in the Onnx model. +// "1": dump the EP context into the Onnx model. (default). static const char* const kOrtSessionOptionEpContextEmbedMode = "ep.context_embed_mode"; -// Specify the EPContext node name prefix to make it unique -// in case user need to merge/connect multiple EPContext nodes in one model -static const char* const kOrtSessionOptionEpContextNodeNamePrefix = "ep.context_node_name_prefix"; - -// Share EP related resources across sessions -static const char* const kOrtSessionOptionShareEpContexts = "ep.share_ep_contexts"; - -// Stop to share EP related resources across sessions from then on -static const char* const kOrtSessionOptionStopShareEpContexts = "ep.stop_share_ep_contexts"; - -// Used only for context model generation. -// This configuration is used when some nodes are partitioned on the CPU EP and those nodes have external initializers. -// When generating the EP context model, the new model should not rely on the old external data file used by the source ONNX model. -// Use this setting when dumping the EP context model with an external initializers file. -// If specified, all initializers will be placed inside the external data file. -// Otherwise, all initializers will be embedded inside the generated ONNX file. -// By default, this option is not set, meaning all initializers will be included within the ONNX file. -static const char* const kOrtSessionOptionsEpContextModelExternalInitializersFileName = - "ep.context_model_external_initializers_file_name"; - // Gemm fastmath mode provides fp32 gemm acceleration with bfloat16 based matmul. // Option values: // - "0": Gemm FastMath mode is not enabled. [DEFAULT] // - "1": Gemm FastMath mode is enabled. static const char* const kOrtSessionOptionsMlasGemmFastMathArm64Bfloat16 = "mlas.enable_gemm_fastmath_arm64_bfloat16"; - -// When converting DQ + MatMul -> MatMulNBits, the accuracy level of the MatMulNBits is controlled by this option. -// Refer to MatMulNBits op schema for more details. -// If not provided, default is 4. -static const char* const kOrtSessionOptionsQDQMatMulNBitsAccuracyLevel = "session.qdq_matmulnbits_accuracy_level"; - -// THIS OPTION IS NOT A REGULAR SESSION OPTION SINCE IT CAN BE MODIFIED AT ANY TIME -// Meant to be used with SetEpDynamicOptions -// Specify the type of workload for this session. -// "Default": OS determines the scheduling priority and processor performance to service this workload. [Default] -// "Efficient": OS treats this workload is efficiency oriented with low scheduling priority and efficient processor performance. -static const char* const kOrtEpDynamicOptionsWorkloadType = "ep.dynamic.workload_type"; - -// Disables model compilation during session initialization. -// -// If this option is set to "1", inference session creation will fail with error code ORT_MODEL_REQUIRES_COMPILATION -// if compilation is required to run the model on any Execution Provider added to the session. -// Only the following kinds of models are valid when this option is set to "1": -// - Pre-compiled models that have EPContext nodes for the compiling Execution Providers in the session. -// - Non-compiled models that run only on non-compiling Execution Providers, like CPU EP. -// -// See \href https://onnxruntime.ai/docs/execution-providers/EP-Context-Design.html for details about -// compiled models with EPContext nodes. -// -// Option values: -// - "0": EP compile is not disabled. [DEFAULT] -// - "1": EP compile is disabled. -static const char* const kOrtSessionOptionsDisableModelCompile = "session.disable_model_compile"; - -// Controls behavior when compiled model compatibility is SUPPORTED_PREFER_RECOMPILATION. -// "0": Allow execution with suboptimal performance. [DEFAULT] -// "1": Fail session creation to require recompilation for optimal performance. -// Note: UNSUPPORTED models always fail regardless of this setting. -static const char* const kOrtSessionOptionsFailOnSuboptimalCompiledModel = - "session.fail_on_suboptimal_compiled_model"; - -// THIS OPTION IS NOT A REGULAR SESSION OPTION SINCE IT CAN BE MODIFIED AT ANY TIME -// Meant to be used with SetEpDynamicOptions -// options for HTP performance mode: "burst", "balanced", "default", "high_performance", -// "high_power_saver", "low_balanced", "extreme_power_saver", "low_power_saver", "power_saver", -// "sustained_high_performance". Default to "default". -static const char* const kOrtEpDynamicOptionsQnnHtpPerformanceMode = "ep.dynamic.qnn_htp_performance_mode"; diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_training_c_api.h b/prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_training_c_api.h similarity index 100% rename from prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_training_c_api.h rename to prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_training_c_api.h diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_training_cxx_api.h b/prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_training_cxx_api.h similarity index 100% rename from prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_training_cxx_api.h rename to prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_training_cxx_api.h diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_training_cxx_inline.h b/prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_training_cxx_inline.h similarity index 100% rename from prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_training_cxx_inline.h rename to prebuilt_libs/onnxruntime-win-x64-1.18.0/include/onnxruntime_training_cxx_inline.h diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/provider_options.h b/prebuilt_libs/onnxruntime-win-x64-1.18.0/include/provider_options.h similarity index 100% rename from prebuilt_libs/onnxruntime-win-x64-1.23.2/include/provider_options.h rename to prebuilt_libs/onnxruntime-win-x64-1.18.0/include/provider_options.h diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/lib/onnxruntime.pdb b/prebuilt_libs/onnxruntime-win-x64-1.18.0/lib/onnxruntime.pdb similarity index 72% rename from prebuilt_libs/onnxruntime-win-x64-1.23.2/lib/onnxruntime.pdb rename to prebuilt_libs/onnxruntime-win-x64-1.18.0/lib/onnxruntime.pdb index 078e602..90bf3c5 100644 Binary files a/prebuilt_libs/onnxruntime-win-x64-1.23.2/lib/onnxruntime.pdb and b/prebuilt_libs/onnxruntime-win-x64-1.18.0/lib/onnxruntime.pdb differ diff --git a/prebuilt_libs/onnxruntime-win-x64-1.18.0/lib/onnxruntime_providers_shared.pdb b/prebuilt_libs/onnxruntime-win-x64-1.18.0/lib/onnxruntime_providers_shared.pdb new file mode 100644 index 0000000..c48d5e3 Binary files /dev/null and b/prebuilt_libs/onnxruntime-win-x64-1.18.0/lib/onnxruntime_providers_shared.pdb differ diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/GIT_COMMIT_ID b/prebuilt_libs/onnxruntime-win-x64-1.23.2/GIT_COMMIT_ID deleted file mode 100644 index 018222a..0000000 --- a/prebuilt_libs/onnxruntime-win-x64-1.23.2/GIT_COMMIT_ID +++ /dev/null @@ -1 +0,0 @@ -a83fc4d58cb48eb68890dd689f94f28288cf2278 diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/README.md b/prebuilt_libs/onnxruntime-win-x64-1.23.2/README.md deleted file mode 100644 index 019bc82..0000000 --- a/prebuilt_libs/onnxruntime-win-x64-1.23.2/README.md +++ /dev/null @@ -1,49 +0,0 @@ -

- -**ONNX Runtime is a cross-platform inference and training machine-learning accelerator**. - -**ONNX Runtime inference** can enable faster customer experiences and lower costs, supporting models from deep learning frameworks such as PyTorch and TensorFlow/Keras as well as classical machine learning libraries such as scikit-learn, LightGBM, XGBoost, etc. ONNX Runtime is compatible with different hardware, drivers, and operating systems, and provides optimal performance by leveraging hardware accelerators where applicable alongside graph optimizations and transforms. [Learn more →](https://www.onnxruntime.ai/docs/#onnx-runtime-for-inferencing) - -**ONNX Runtime training** can accelerate the model training time on multi-node NVIDIA GPUs for transformer models with a one-line addition for existing PyTorch training scripts. [Learn more →](https://www.onnxruntime.ai/docs/#onnx-runtime-for-training) - -## Get Started & Resources - -* **General Information**: [onnxruntime.ai](https://onnxruntime.ai) - -* **Usage documentation and tutorials**: [onnxruntime.ai/docs](https://onnxruntime.ai/docs) - -* **YouTube video tutorials**: [youtube.com/@ONNXRuntime](https://www.youtube.com/@ONNXRuntime) - -* [**Upcoming Release Roadmap**](https://onnxruntime.ai/roadmap) - -* **Companion sample repositories**: - - ONNX Runtime Inferencing: [microsoft/onnxruntime-inference-examples](https://github.com/microsoft/onnxruntime-inference-examples) - - ONNX Runtime Training: [microsoft/onnxruntime-training-examples](https://github.com/microsoft/onnxruntime-training-examples) - -## Releases - -The current release and past releases can be found here: https://github.com/microsoft/onnxruntime/releases. - -For details on the upcoming release, including release dates, announcements, features, and guidance on submitting feature requests, please visit the release roadmap: https://onnxruntime.ai/roadmap. - -## Data/Telemetry - -Windows distributions of this project may collect usage data and send it to Microsoft to help improve our products and services. See the [privacy statement](docs/Privacy.md) for more details. - -## Contributions and Feedback - -We welcome contributions! Please see the [contribution guidelines](CONTRIBUTING.md). - -For feature requests or bug reports, please file a [GitHub Issue](https://github.com/Microsoft/onnxruntime/issues). - -For general discussion or questions, please use [GitHub Discussions](https://github.com/microsoft/onnxruntime/discussions). - -## Code of Conduct - -This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). -For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) -or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. - -## License - -This project is licensed under the [MIT License](LICENSE). diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/VERSION_NUMBER b/prebuilt_libs/onnxruntime-win-x64-1.23.2/VERSION_NUMBER deleted file mode 100644 index 14bee92..0000000 --- a/prebuilt_libs/onnxruntime-win-x64-1.23.2/VERSION_NUMBER +++ /dev/null @@ -1 +0,0 @@ -1.23.2 diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_ep_c_api.h b/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_ep_c_api.h deleted file mode 100644 index 975f6b4..0000000 --- a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_ep_c_api.h +++ /dev/null @@ -1,988 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// Do not include this file directly. Please include "onnxruntime_c_api.h" instead. - -#ifdef __cplusplus -extern "C" { -#endif - -ORT_RUNTIME_CLASS(Ep); -ORT_RUNTIME_CLASS(EpFactory); -ORT_RUNTIME_CLASS(EpGraphSupportInfo); -ORT_RUNTIME_CLASS(MemoryDevice); // opaque class to wrap onnxruntime::OrtDevice -ORT_RUNTIME_CLASS(NodeComputeContext); - -ORT_RUNTIME_CLASS(DataTransferImpl); -ORT_RUNTIME_CLASS(SyncNotificationImpl); -ORT_RUNTIME_CLASS(SyncStreamImpl); - -// struct that an EP implements for IDataTransfer to copy between devices it uses and CPU -struct OrtDataTransferImpl { - uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION - - /** \brief Release the OrtDataTransferImpl instance. - * - * This is called by ORT when the OrtDataTransferImpl instance is no longer needed. - * The implementation should release any resources held by the instance. - * - * \param[in] this_ptr Pointer to the OrtDataTransferImpl instance. - * - * \since Version 1.23. - */ - ORT_API_T(void, Release, _In_ OrtDataTransferImpl* this_ptr); - - /** \brief Check if the implementation can copy between the source and destination memory devices. - * - * \param[in] this_ptr Pointer to the OrtDataTransferImpl instance. - * \param[in] src_memory_device Source OrtMemoryDevice to copy from. - * \param[in] dst_memory_device Destination OrtMemoryDevice to copy to. - * \return True if the implementation can copy between the devices. - * - * \since Version 1.23. - */ - ORT_API_T(bool, CanCopy, _In_ const OrtDataTransferImpl* this_ptr, - _In_ const OrtMemoryDevice* src_memory_device, _In_ const OrtMemoryDevice* dst_memory_device); - - /** \brief Copy tensors from src_tensors to dst_tensors using the provided streams. - * - * The implementation can use the provided streams to perform asynchronous copies if supported. - * If a stream is not available, the copy is performed synchronously. - * - * \param[in] this_ptr Pointer to the OrtDataTransferImpl instance. - * \param[in] src_tensors Array of source OrtValue pointers to copy from. - * \param[in] dst_tensors Array of destination OrtValue pointers to copy to. - * \param[in] streams Array of OrtSyncStream pointers for the copy operations, if the execution provider is stream - * aware. nullptr if it is not. - * \param[in] num_tensors Number of tensors to copy. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(CopyTensors, _In_ OrtDataTransferImpl* this_ptr, - _In_reads_(num_tensors) const OrtValue** src_tensors, - _In_reads_(num_tensors) OrtValue** dst_tensors, - _In_reads_(num_tensors) OrtSyncStream** streams, - _In_ size_t num_tensors); -}; - -/** \brief Struct that an EP implements for Stream Notifications. - * - * \since Version 1.23. - */ -struct OrtSyncNotificationImpl { - uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION - - /** \brief Release the OrtSyncNotificationImpl instance. - * - * This is called by ORT when the OrtSyncNotificationImpl instance is no longer needed. - * The implementation should release any resources held by the instance. - * - * \param[in] this_ptr Pointer to the OrtSyncNotificationImpl instance. - * - * \since Version 1.23. - */ - ORT_API_T(void, Release, _In_ OrtSyncNotificationImpl* this_ptr); - - /** \brief Called by ORT to activate the notification. - * - * \param[in] this_ptr Pointer to the OrtSyncNotificationImpl instance. - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Activate, _In_ OrtSyncNotificationImpl* this_ptr); - - /** \brief Wait for a device to device operation to complete. - * - * \param[in] this_ptr Pointer to the OrtSyncNotificationImpl instance. - * \param[in] stream The OrtSyncStream instance that will wait on this notification to be activated. - * - * \since Version 1.23. - */ - ORT_API2_STATUS(WaitOnDevice, _In_ OrtSyncNotificationImpl* this_ptr, _In_ OrtSyncStream* consumer_stream); - - /** \brief Wait for a device to host operation to complete. - * - * \param[in] this_ptr Pointer to the OrtSyncNotificationImpl instance. - * - * \since Version 1.23. - */ - ORT_API2_STATUS(WaitOnHost, _In_ OrtSyncNotificationImpl* this_ptr); -}; - -/** \brief Struct that an EP implements if it wishes to implement Stream support. - * - * This struct provides the overrides for onnxruntime::Stream's virtual methods. - * - * \since Version 1.23. - */ -struct OrtSyncStreamImpl { - uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION - - /** \brief Release the OrtSyncStreamImpl instance. - * - * This is called by ORT when the OrtSyncStreamImpl instance is no longer needed. - * The implementation should release any resources held by the instance. - * - * \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance. - * - * \since Version 1.23. - */ - ORT_API_T(void, Release, _In_ OrtSyncStreamImpl* this_ptr); - - /** \brief Get the handle of the stream. - * - * This returns the native handle for the stream. e.g. cudaStream_t for CUDA streams. - * - * \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance. - * \return The handle of the stream. - * - * \since Version 1.23. - */ - ORT_API_T(void*, GetHandle, _In_ OrtSyncStreamImpl* this_ptr); - - /** \brief Create an OrtSyncNotificationImpl for the OrtSyncStreamImpl instance. - * - * \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance - * \param[out] notification The new OrtSyncNotificationImpl instance. - * - * \since Version 1.23. - */ - ORT_API2_STATUS(CreateNotification, _In_ OrtSyncStreamImpl* this_ptr, - _Outptr_ OrtSyncNotificationImpl** notification); - - /** \brief Flush the stream. - * - * This is called by ORT to flush the stream, ensuring that all operations submitted to the stream are completed. - * - * \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance. - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Flush, _In_ OrtSyncStreamImpl* this_ptr); - - /** \brief Notify the stream that a session run has ended. - * - * This is called by ORT to notify the stream that a session run has ended, allowing the stream to perform any - * necessary cleanup or finalization. - * - * \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance. - * - * \since Version 1.23. - */ - ORT_API2_STATUS(OnSessionRunEnd, _In_ OrtSyncStreamImpl* this_ptr); -}; - -struct OrtNodeFusionOptions; -typedef struct OrtNodeFusionOptions OrtNodeFusionOptions; - -struct OrtNodeComputeInfo; -typedef struct OrtNodeComputeInfo OrtNodeComputeInfo; - -/** - * \brief The OrtNodeFusionOptions struct specifies options for fusing nodes supported by an execution provider. - * - * Refer to OrtEpApi::EpGraphSupportInfo_AddNodesToFuse. - * - * \since Version 1.23. - */ -struct OrtNodeFusionOptions { - /** \brief The ONNX Runtime version the OrtNodeFusionOptions was compiled with. - * - * Implementation should set to ORT_API_VERSION. - * ORT will use this to ensure it does not use members that were not available when the EP library was compiled. - * - * \since Version 1.23. - */ - uint32_t ort_version_supported; - - /** \brief If set to true, specify that the execution provider does not require ONNX Runtime to provide constant - * initializers as inputs to the fused node during model inference. This is used when the execution - * provider saves a copy of constant initializers, and allows ONNX Runtime to release constant initializers that - * are not used by any execution provider. - * - * If not specified, defaults to false. That is, ONNX Runtime provides constant initializers as inputs to - * the fused node by default. - * - * \since Version 1.23. - */ - bool drop_constant_initializers; - - // const OrtNode* fused_node_schema; -}; - -/** - * \brief The OrtNodeComputeInfo struct provides functions that an OrtEp implements to specify the compute - * function for a compiled OrtGraph instance. - * \since Version 1.23. - */ -struct OrtNodeComputeInfo { - /** \brief The ONNX Runtime version the OrtNodeComputeInfo was compiled with. - * - * Implementation should set to ORT_API_VERSION. - * ORT will use this to ensure it does not call functions that were not available when the EP library was compiled. - * - * \since Version 1.23. - */ - uint32_t ort_version_supported; - - /** \brief Creates an opaque compute state object that is then passed to the Compute() function during inference. - * \param[in] this_ptr The OrtNodeComputeInfo instance. - * \param[in] compute_context OrtNodeComputeContext instance that contains compiled/fused node's name and host - * memory allocation functions. Can optionally be used to build the compute state. - * \param[out] compute_state Output parameter that is assigned the opaque computation state. ONNX Runtime calls - * ReleaseState() (after calling Compute()) to allow the implementer to release the - * compute state. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - OrtStatus*(ORT_API_CALL* CreateState)(_In_ OrtNodeComputeInfo* this_ptr, - _In_ OrtNodeComputeContext* compute_context, - _Outptr_ void** compute_state); - - /** \brief Computation function called to execute the fused node compiled by an OrtEp instance. - * \param[in] this_ptr The OrtNodeComputeInfo instance. - * \param[in] compute_state The opaque computation state returned by CreateState(). - * \param[in] kernel_context The OrtKernelContext instance used to access inputs/outputs. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - OrtStatus*(ORT_API_CALL* Compute)(_In_ OrtNodeComputeInfo* this_ptr, _In_ void* compute_state, - _In_ OrtKernelContext* kernel_context); - - /** \brief Releases the compute state returned by CreateState(). - * \param[in] this_ptr The OrtNodeComputeInfo instance. - * \param[inout] compute_state The opaque compute state returned by CreateState(). - * - * \since Version 1.23. - */ - void(ORT_API_CALL* ReleaseState)(_In_ OrtNodeComputeInfo* this_ptr, _Frees_ptr_opt_ void* compute_state); -}; - -struct OrtEpApi { - /** \brief Create an OrtEpDevice for the EP and an OrtHardwareDevice. - * \param[in] ep_factory Execution provider factory that is creating the instance. - * \param[in] hardware_device Hardware device that the EP can utilize. - * \param[in] ep_metadata Optional OrtKeyValuePairs instance for execution provider metadata that may be used - * during execution provider selection and passed to CreateEp. - * ep_device will copy this instance and the user should call ReleaseKeyValuePairs. - * \param[in] ep_options Optional OrtKeyValuePairs instance for execution provider options that will be added - * to the Session configuration options if the execution provider is selected. - * ep_device will copy this instance and the user should call ReleaseKeyValuePairs. - * \param ep_device OrtExecutionDevice that is created. - * - * \since Version 1.22. - */ - ORT_API2_STATUS(CreateEpDevice, _In_ OrtEpFactory* ep_factory, - _In_ const OrtHardwareDevice* hardware_device, - _In_opt_ const OrtKeyValuePairs* ep_metadata, - _In_opt_ const OrtKeyValuePairs* ep_options, - _Out_ OrtEpDevice** ep_device); - - ORT_CLASS_RELEASE(EpDevice); - - /** \brief Specify nodes that are supported by an OrtEp and should be fused into one node. - * - * Because the nodes will be fused into one "fused node", there must not exist an unsupported node in - * a path between two of the provided nodes. Otherwise, the graph will become invalid. - * - * This function can be called multiple times. A subsequent call to this function will force the next set of - * nodes to be fused into a different node. - * - * \param[in] graph_support_info OrtEpGraphSupportInfo instance to which to add the supported nodes. - * \param[in] nodes Array of nodes supported by the EP that should be fused/compiled. - * \param[in] num_nodes The number of supported nodes. - * \param[in] node_fusion_options Optional node fusion options. Ignored if set to NULL. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(EpGraphSupportInfo_AddNodesToFuse, _In_ OrtEpGraphSupportInfo* graph_support_info, - _In_reads_(num_nodes) const OrtNode* const* nodes, _In_ size_t num_nodes, - _In_opt_ const OrtNodeFusionOptions* node_fusion_options); - - /** \brief Specify a node that is supported by an OrtEp and should be run with a registered EP kernel. - * - * \param[in] graph_support_info OrtEpGraphSupportInfo instance to which to add the supported node. - * \param[in] node The supported OrtNode instance. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(EpGraphSupportInfo_AddSingleNode, _In_ OrtEpGraphSupportInfo* graph_support_info, - _In_ const OrtNode* node); - - /** \brief Query a OrtNodeComputeContext for the name of the node that encapsulates the compiled/fused node. - * - * Used in OrtNodeComputeInfo::CreateComputeState(). - * - * \param[in] context The OrtNodeComputeContext instance to query. - * \return The node's name. - * - * \note Returned string is owned by ORT and valid only while OrtNodeComputeInfo::CreateComputeState() is called. - * - * \since Version 1.23. - */ - ORT_API_T(const char*, NodeComputeContext_NodeName, _In_ const OrtNodeComputeContext* context); - - /** \brief Register an allocator with the OrtEpDevice. - * - * This allows an EP to provide OrtMemoryInfo for DEFAULT and HOST_ACCESSIBLE memory type as needed. - * The registered values will be used in calls to OrtEpFactory::CreateAllocator to ensure the required allocator/s - * are available for EP usage. - * - * Multiple calls for the same entry type will replace a previous entry. - * - * Available entries: - * - OrtDeviceAllocator with type of OrtDeviceMemoryType_DEFAULT - * - OrtDeviceAllocator with type of OrtDeviceMemoryType_HOST_ACCESSIBLE - * - OrtReadOnlyAllocator with type of OrtDeviceMemoryType_DEFAULT - * - if provided this allocator will only be used to copy initializers to the device the EP uses. - * ORT will use the OrtDeviceAllocator if not provided. - * - * \param[in] ep_device The OrtEpDevice instance to register the OrtMemoryInfo with. - * \param[in] allocator_memory_info The OrtMemoryInfo information for the allocator. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(EpDevice_AddAllocatorInfo, _In_ OrtEpDevice* ep_device, - _In_ const OrtMemoryInfo* allocator_memory_info); - - /** \brief Get the OrtMemoryDevice from an OrtMemoryInfo instance. - * - * This is required for OrtDataTransferImpl (which implements onnxruntime::IDataTransfer) where the OrtMemoryDevice - * is used in the CanCopy and CopyTensors functions. - * - * \param[in] memory_info The OrtMemoryInfo instance to get the memory device from. - * \return The OrtMemoryDevice associated with the OrtMemoryInfo instance. - * - * \since Version 1.23. - */ - ORT_API_T(const OrtMemoryDevice*, MemoryInfo_GetMemoryDevice, _In_ const OrtMemoryInfo* memory_info); - - /** \brief Get the OrtMemoryDevice from an OrtValue instance if it contains a Tensor. - * - * \param[in] value The OrtValue instance to get the memory device from. - * \return Memory device if OrtValue contains a Tensor, nullptr otherwise. - * - * \since Version 1.23. - */ - ORT_API_T(const OrtMemoryDevice*, Value_GetMemoryDevice, _In_ const OrtValue* value); - - /** \brief Compare two OrtMemoryDevice instances for equality. - * - * This is used to check if two memory devices are the same. - * Used to implement DataTransferImpl::CanCopy. - * - * \param[in] a The first OrtMemoryDevice instance to compare. - * \param[in] b The second OrtMemoryDevice instance to compare. - * \return True if the two OrtMemoryDevice instances are equal, false otherwise. - * - * \since Version 1.23. - */ - ORT_API_T(bool, MemoryDevice_AreEqual, _In_ const OrtMemoryDevice* a, _In_ const OrtMemoryDevice* b); - - /** \brief Get the OrtMemoryInfoDeviceType value from an OrtMemoryDevice instance. - * - * \param[in] memory_device OrtMemoryDevice instance. - * \return The OrtMemoryInfoDeviceType value. - * - * \since Version 1.23. - */ - ORT_API_T(OrtMemoryInfoDeviceType, MemoryDevice_GetDeviceType, _In_ const OrtMemoryDevice* memory_device); - - /** \brief Get the OrtDeviceMemoryType value from an OrtMemoryDevice instance. - * - * \param[in] memory_device OrtMemoryDevice instance. - * \return The OrtDeviceMemoryType value. - * - * \since Version 1.23. - */ - ORT_API_T(OrtDeviceMemoryType, MemoryDevice_GetMemoryType, _In_ const OrtMemoryDevice* memory_device); - - /** \brief Get the vendor ID from an OrtMemoryDevice instance. - * - * The vendor ID is used to identify the vendor of the device, and is typically set to the PCI vendor ID. - * - * If the device is not vendor specific (e.g. CPU memory) the vendor ID is set to 0. - * - * \param[in] memory_device OrtMemoryDevice instance. - * \return The vendor ID value. - * - * \since Version 1.23. - */ - ORT_API_T(uint32_t, MemoryDevice_GetVendorId, _In_ const OrtMemoryDevice* memory_device); - - /** \brief Get the device ID from an OrtMemoryDevice instance. - * - * \param[in] memory_device OrtMemoryDevice instance. - * \return The device ID. - * - * \since Version 1.23. - */ - ORT_API_T(uint32_t, MemoryDevice_GetDeviceId, _In_ const OrtMemoryDevice* memory_device); - - /** \brief Get the OrtSyncStreamImpl associated with an OrtSyncStream instance. - * - * This allows an the plugin library to connect its OrtSyncStreamImpl instance with an OrtSyncStream if needed. - * - * \param[in] stream The OrtSyncStream instance to find an OrtSyncStreamImpl for. - * \return The associated OrtSyncStreamImpl if found. nullptr otherwise. - * - * \since Version 1.23. - * - * \remarks There should always be an OrtSyncStreamImpl associated with an OrtSyncStream instance that the EP gets. - */ - ORT_API_T(const OrtSyncStreamImpl*, SyncStream_GetImpl, _In_ const OrtSyncStream* stream); - - /** \brief Get the current sync ID for a stream. - * - * \param[in] stream The OrtSyncStream to get the sync ID for. - * \return Current sync ID. - * - * \since Version 1.23. - */ - ORT_API_T(uint64_t, SyncStream_GetSyncId, _In_ const OrtSyncStream* stream); - - /** \brief Get the sync ID for the last time the consumer_stream waited on the producer_stream. - * - * When two streams are synchronized, the sync id represents the event used in that synchronization. - * - * \param[in] producer_stream The OrtSyncStream that produced the data. - * \param[in] consumer_stream The OrtSyncStream that waited on the producer_stream. - * \return ID for last sync. 0 if no sync has occurred between the two streams. - * - * \since Version 1.23. - */ - ORT_API_T(uint64_t, GetSyncIdForLastWaitOnSyncStream, - _In_ const OrtSyncStream* producer_stream, _In_ const OrtSyncStream* consumer_stream); -}; - -/** - * \brief The data layout type. - * - * EPs may specify a preferred data layout type. ORT's default layout type is OrtEpDataLayout_NCHW, or - * OrtEpDataLayout_Default. - * - * \since Version 1.23. - */ -typedef enum OrtEpDataLayout { - OrtEpDataLayout_NCHW = 0, - OrtEpDataLayout_NHWC, - - OrtEpDataLayout_Default = OrtEpDataLayout_NCHW, -} OrtEpDataLayout; - -/** - * \brief The OrtEp struct provides functions to implement for an execution provider. - * \since Version 1.22. - */ -struct OrtEp { - /** \brief The ONNX Runtime version the execution provider was compiled with. - * - * Implementation should set to ORT_API_VERSION. - * ORT will use this to ensure it does not call functions that were not available when the library was compiled. - * - * \since Version 1.22. - */ - uint32_t ort_version_supported; - - /** \brief Get the execution provider name. - * - * The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it. - * - * \param[in] this_ptr The OrtEp instance. - * \return The execution provider name. - * - * \since Version 1.22. - */ - ORT_API_T(const char*, GetName, _In_ const OrtEp* this_ptr); - - /** \brief Get information about the nodes supported by the OrtEp instance. - * - * IMPORTANT: This is not the final version of this API function. This is currently experimental but will - * be stabilized by the ONNX Runtime 1.23 release. - * - * \param[in] this_ptr The OrtEp instance. - * \param[in] graph The OrtGraph instance for which to populate node support. The OrtGraph could be a nested subgraph - * contained by a node (e.g., an If or Loop node). ONNX Runtime calls this function separately - * for each nested subgraph. - * \param[inout] graph_support_info OrtEpGraphSupportInfo instance that the implementer must fill out in order to - * specify the supported nodes. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(GetCapability, _In_ OrtEp* this_ptr, _In_ const OrtGraph* graph, - _Inout_ OrtEpGraphSupportInfo* graph_support_info); - - /** \brief Compile OrtGraph instances assigned to the OrtEp. Implementer must set a OrtNodeComputeInfo instance - * for each OrtGraph in order to define its computation function. - * - * If the session is configured to generate a pre-compiled model, the execution provider must return EPContext nodes, - * as OrtNode instances, that ONNX Runtime uses to create a pre-compiled model, known as an "EPContext model". - * An EPContext model contains EPContext nodes. Each EPContext node encapsulates the pre-compiled binary data for a - * OrtGraph compiled for a specific execution provider. For more details about the EPContext design, refer to: - * \htmlonly - * EPContext design document. - * \endhtmlonly - * - * \param[in] this_ptr The OrtEp instance. - * \param[in] graphs Array of `count` OrtGraph instances to compile. Each graph contains only the nodes for - * which the execution provider indicated support. Nested subgraphs contained by a - * node, such as an If or Loop, have separate OrtGraph instances. - * \param[in] fused_nodes Array of `count` fused nodes that will replace the compiled graphs. - * Each fused node is an OrtNode initialized with the intended fused node name and - * input/output information. - * \param[in] count The number of OrtGraph instances to compile. - * \param[out] node_compute_infos Array of `count` OrtNodeComputeInfo instances that define each OrtGraph instance's - * computation function. The implementer allocates the OrtNodeComputeInfo instances. - * ORT calls ReleaseNodeComputeInfos() to release multiple instances in a batch. - * \param[out] ep_context_nodes Output array of `count` OrtNode instances, each representing an EPContext - * node for a compiled OrtGraph. The execution provider must use - * OrtModelEditorApi::CreateNode to create the OrtNode instances. ONNX Runtime takes - * ownership of the OrtNode instances, so the execution provider must NOT call - * OrtApi::ReleaseNode. Should be ignored if the session is not configured to generate an - * EPContext model. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \note Do NOT cache the provided OrtGraph instances in any of the OrtNodeComputeInfo functions because the - * graphs are only valid for the duration of the call to Compile. Any graph/node/input/output - * names that are needed by the OrtNodeComputeInfo functions must be copied and stored by the OrtEp. - * - * \since Version 1.23. - */ - ORT_API2_STATUS(Compile, _In_ OrtEp* this_ptr, _In_ const OrtGraph** graphs, - _In_ const OrtNode** fused_nodes, _In_ size_t count, - _Out_writes_all_(count) OrtNodeComputeInfo** node_compute_infos, - _Out_writes_(count) OrtNode** ep_context_nodes); - - /** \brief Release OrtNodeComputeInfo instances. - * - * \param[in] this_ptr The OrtEp instance. - * \param[inout] node_compute_infos The OrtNodeComputeInfo instances to release. - * \param[in] num_node_compute_infos The number of OrtNodeComputeInfo instances. - * - * \since Version 1.23. - */ - ORT_API_T(void, ReleaseNodeComputeInfos, _In_ OrtEp* this_ptr, - OrtNodeComputeInfo** node_compute_infos, - _In_ size_t num_node_compute_infos); - - /** \brief Get the EP's preferred data layout. - * - * \note Implementation of this function is optional. - * If not implemented, ORT will assume that this EP prefers the data layout `OrtEpDataLayout::NCHW`. - * - * \param[in] this_ptr The OrtEp instance. - * \param[out] preferred_data_layout The EP's preferred data layout. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(GetPreferredDataLayout, _In_ OrtEp* this_ptr, _Out_ OrtEpDataLayout* preferred_data_layout); - - /** \brief Given an op with domain `domain` and type `op_type`, determine whether an associated node's data layout - * should be converted to `target_data_layout`. - * If the EP prefers a non-default data layout (see `GetPreferredDataLayout()`), this function will be called - * during layout transformation with `target_data_layout` set to the EP's preferred data layout. - * - * \note Implementation of this function is optional. - * If an EP prefers a non-default data layout, it may implement this to customize the specific op data layout - * preferences at a finer granularity. - * - * \param[in] this_ptr The OrtEp instance. - * \param[in] domain The op domain. An empty string means the ONNX domain. - * \param[in] op_type The op type. - * \param[in] target_data_layout The target data layout. - * \param[out] should_convert Whether the associated node's data layout should be converted to `target_data_layout`. - * If greater than 0, convert. - * If 0, don't convert. - * Otherwise, if less than 0, leave the decision to ORT. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(ShouldConvertDataLayoutForOp, _In_ OrtEp* this_ptr, - _In_z_ const char* domain, _In_z_ const char* op_type, - _In_ OrtEpDataLayout target_data_layout, - _Outptr_ int* should_convert); - - /** \brief Set dynamic options on this EP. - * - * Dynamic options can be set by the user at any time after session creation with `OrtApi::SetEpDynamicOptions()`. - * - * \param[in] this_ptr The OrtEp instance. - * \param[in] option_keys The dynamic option keys. - * \param[in] option_values The dynamic option values. - * \param[in] num_options The number of dynamic options. - * - * \note Implementation of this function is optional. - * An EP should only implement this if it needs to handle any dynamic options. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(SetDynamicOptions, _In_ OrtEp* this_ptr, - _In_reads_(num_options) const char* const* option_keys, - _In_reads_(num_options) const char* const* option_values, - _In_ size_t num_options); - - /** \brief Called by ORT to notify the EP of the start of a run. - * - * \param[in] this_ptr The OrtEp instance. - * \param[in] run_options The run options for this run. - * - * \note Implementation of this function is optional. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(OnRunStart, _In_ OrtEp* this_ptr, _In_ const OrtRunOptions* run_options); - - /** \brief Called by ORT to notify the EP of the end of a run. - * - * \param[in] this_ptr The OrtEp instance. - * \param[in] run_options The run options for this run. - * \param[in] sync_stream Whether any associated stream should be synchronized during this call. - * Only applicable if there is such a stream. - * - * \note Implementation of this function is optional. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(OnRunEnd, _In_ OrtEp* this_ptr, _In_ const OrtRunOptions* run_options, _In_ bool sync_stream); - - /** \brief Create an OrtAllocator for the given OrtMemoryInfo for an OrtSession. - * - * The OrtMemoryInfo instance will match one of the values set in the OrtEpDevice using EpDevice_AddAllocatorInfo. - * Any allocator specific options should be read from the session options. - * - * If nullptr OrtEpFactory::CreateAllocator will be used. - * - * \param[in] this_ptr The OrtEpFactory instance. - * \param[in] memory_info The OrtMemoryInfo to create the allocator for. May be nullptr. - * \param[out] allocator The created OrtAllocator instance. Set to nullptr if the default CPU allocator is used. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(CreateAllocator, _In_ OrtEp* this_ptr, - _In_ const OrtMemoryInfo* memory_info, - _Outptr_result_maybenull_ OrtAllocator** allocator); - - /** \brief Create a synchronization stream for the given memory device for an OrtSession. - * - * This is used to create a synchronization stream for the execution provider and is used to synchronize - * operations on the device during model execution. - * Any stream specific options should be read from the session options. - * - * If nullptr OrtEpFactory::CreateSyncStreamForDevice will be used. - * - * \param[in] this_ptr The OrtEpFactory instance. - * \param[in] memory_device The OrtMemoryDevice to create the synchronization stream for. - * \param[out] stream The created OrtSyncStreamImpl instance. nullptr if the execution provider is not stream aware. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(CreateSyncStreamForDevice, _In_ OrtEp* this_ptr, - _In_ const OrtMemoryDevice* memory_device, - _Outptr_ OrtSyncStreamImpl** stream); - - /** \brief Get a string with details about the EP stack used to produce a compiled model. - * - * This function gets a compatibility information string that contains details about the execution provider - * used to compile a given model. This string can later be used with ValidateCompiledModelCompatibilityInfo - * to determine if a compiled model is compatible with the EP. - * - * The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it. - * - * \param[in] this_ptr The OrtEp instance. - * \param[in] graph The OrtGraph instance for which to generate compatibility information. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API_T(const char*, GetCompiledModelCompatibilityInfo, _In_ OrtEp* this_ptr, - _In_ const OrtGraph* graph); -}; - -/** \brief The function signature that ORT will call to create OrtEpFactory instances. - * - * This must be available in a function called 'CreateEpFactories' in the execution provider library. - * - * \param[in] registered_name The name the execution library is registered with by RegisterExecutionProviderLibrary - * \param[in] ort_api_base The OrtApiBase instance that is used by the factory to get the OrtApi instance for the - * version of ORT that the library was compiled against. - * \param[in] default_logger The default ORT logger that can be used for logging outside of an inference session. - * \param[in,out] factories The implementation should create and add OrtEpFactory instances to this - * pre-allocated array. - * i.e. usage is `factories[0] = new MyEpFactory();` - * \param[in] max_factories The maximum number of OrtEpFactory instances that can be added to `factories`. - * Current default is to allow 4 factories. This can be increased in the future if needed. - * \param[out] num_factories The number of OrtEpFactory instances created by the factory and added to `factories`. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ -typedef OrtStatus* (*CreateEpApiFactoriesFn)(_In_ const char* registered_name, _In_ const OrtApiBase* ort_api_base, - _In_ const OrtLogger* default_logger, - _Inout_ OrtEpFactory** factories, _In_ size_t max_factories, - _Out_ size_t* num_factories); - -/** \brief The function signature that ORT will call to release an OrtEpFactory instance. - * - * This must be available in a function called 'ReleaseEpFactory' in the execution provider library. - * - * \param[in] factory The OrtEpFactory instance to release. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ -typedef OrtStatus* (*ReleaseEpApiFactoryFn)(_In_ OrtEpFactory* factory); - -/** - * \brief The OrtEpFactory provides functions to create and manage execution providers. - * \since Version 1.22. - */ -struct OrtEpFactory { - /** \brief The ONNX Runtime version the execution provider was compiled with. - * - * Implementation should set to ORT_API_VERSION. - * ORT will use this to ensure it does not call functions that were not available when the library was compiled. - * - * \since Version 1.22. - */ - uint32_t ort_version_supported; - - /** \brief Get the name of the execution provider that the factory creates. - * - * The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it. - * - * \param[in] this_ptr The OrtEpFactory instance. - * \return The name of the execution provider the factory creates. - * - * \since Version 1.22. - */ - ORT_API_T(const char*, GetName, const OrtEpFactory* this_ptr); - - /** \brief Get the name of vendor who owns the execution provider that the factory creates. - * - * The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it. - * - * \param[in] this_ptr The OrtEpFactory instance. - * \return vendor The vendor name of the execution provider the factory creates. - * - * \since Version 1.22. - */ - ORT_API_T(const char*, GetVendor, const OrtEpFactory* this_ptr); // return EP vendor - - /** \brief Get information from the execution provider about OrtHardwareDevice support. - * - * \param[in] this_ptr The OrtEpFactory instance. - * Non-const as the factory is passed through to the CreateEp call via the OrtEpDevice. - * \param[in] devices The OrtHardwareDevice instances that are available. - * \param[in] num_devices The number of OrtHardwareDevice instances. - * \param[out] ep_devices OrtEpDevice instances for each OrtHardwareDevice that the EP can use. - * The implementation should call OrtEpApi::CreateEpDevice to create, and add the OrtEpDevice - * instances to this pre-allocated array. ORT will take ownership of the values returned. - * i.e. usage is `ep_devices[0] = ;` - * \param[in] max_ep_devices The maximum number of OrtEpDevices that can be added to ep_devices. - * Current default is 8. This can be increased if needed. - * \param[out] num_ep_devices The number of EP devices added to ep_devices. - * \return true if the factory can create an execution provider that uses `device`. - * - * \since Version 1.22. - */ - ORT_API2_STATUS(GetSupportedDevices, _In_ OrtEpFactory* this_ptr, - _In_reads_(num_devices) const OrtHardwareDevice* const* devices, - _In_ size_t num_devices, - _Inout_ OrtEpDevice** ep_devices, - _In_ size_t max_ep_devices, - _Out_ size_t* num_ep_devices); - - /** \brief Function to create an OrtEp instance for use in a Session. - * - * ORT will call ReleaseEp to release the instance when it is no longer needed. - * - * \param[in] this_ptr The OrtEpFactory instance. - * \param[in] devices The OrtHardwareDevice instances that the execution provider was selected to use. - * May be a subset of the OrtHardwareDevice instances that the execution provider's factory - * set as supported in the call to OrtEpFactory::GetSupportedDevices. - * \param[in] ep_metadata_pairs Execution provider metadata that was provided to OrtEpApi::CreateEpDevice, for each - * device. - * \param[in] num_devices The number of devices the execution provider was selected for. - * \param[in] session_options The OrtSessionOptions instance that contains the configuration options for the - * session. This will include ep_options from GetSupportedDevices as well as any - * user provided overrides. - * Execution provider options will have been added with a prefix of 'ep.[ep name].'. - * The OrtSessionOptions instance will NOT be valid after this call and should not be - * stored for later use. - * \param[in] logger The OrtLogger instance for the session that the execution provider should use for logging. - * \param[out] ep The OrtEp instance created by the factory. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.22. - */ - ORT_API2_STATUS(CreateEp, _In_ OrtEpFactory* this_ptr, - _In_reads_(num_devices) const OrtHardwareDevice* const* devices, - _In_reads_(num_devices) const OrtKeyValuePairs* const* ep_metadata_pairs, - _In_ size_t num_devices, - _In_ const OrtSessionOptions* session_options, - _In_ const OrtLogger* logger, _Outptr_ OrtEp** ep); - - /** \brief Release the OrtEp instance. - * - * \param[in] this_ptr The OrtEpFactory instance. - * \param[in] ep The OrtEp instance to release. - * - * \since Version 1.22. - */ - ORT_API_T(void, ReleaseEp, OrtEpFactory* this_ptr, struct OrtEp* ep); - - /** \brief Get the vendor id who owns the execution provider that the factory creates. - * - * This is typically the PCI vendor ID. See https://pcisig.com/membership/member-companies - * - * \param[in] this_ptr The OrtEpFactory instance. - * \return vendor_id The vendor ID of the execution provider the factory creates. - * - * \since Version 1.23. - */ - ORT_API_T(uint32_t, GetVendorId, const OrtEpFactory* this_ptr); - - /** \brief Get the version of the execution provider that the factory creates. - * - * The version string should adhere to the Semantic Versioning 2.0 specification - * (https://github.com/semver/semver/blob/v2.0.0/semver.md). - * - * The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it. - * - * \param[in] this_ptr The OrtEpFactory instance. - * \return The execution provider version string. - * - * \since Version 1.23. - */ - ORT_API_T(const char*, GetVersion, _In_ const OrtEpFactory* this_ptr); - - /** \brief Validate the compatibility of a compiled model with the execution provider factory for one or more devices. - * - * Given a compatibility info string produced during model compilation, the EP factory should determine whether the - * compiled model is compatible with the EP factory when targeting the provided hardware devices. All devices provided - * must belong to the same execution provider instance that this factory creates. - * - * The EP factory implementation should consider the set of devices (e.g., multi-adapter or multi-GPU scenarios) when - * evaluating compatibility and set `model_compatibility` accordingly. - * - * \param[in] this_ptr The OrtEpFactory instance. - * \param[in] devices Array of OrtHardwareDevice pointers that the EP would run on. All must map to this EP. - * \param[in] num_devices Number of entries in `devices`. - * \param[in] compatibility_info The compatibility information string produced when the model was compiled. - * \param[out] model_compatibility OrtCompiledModelCompatibility value describing the compatibility of the model with the EP. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(ValidateCompiledModelCompatibilityInfo, _In_ OrtEpFactory* this_ptr, - _In_reads_(num_devices) const OrtHardwareDevice* const* devices, - _In_ size_t num_devices, - _In_ const char* compatibility_info, - _Out_ OrtCompiledModelCompatibility* model_compatibility); - - /** \brief Create an OrtAllocator that can be shared across sessions for the given OrtMemoryInfo. - * - * The factory that creates the EP is responsible for providing the allocators required by the EP. - * The OrtMemoryInfo instance will match one of the values set in the OrtEpDevice using EpDevice_AddAllocatorInfo. - * - * \param[in] this_ptr The OrtEpFactory instance. - * \param[in] memory_info The OrtMemoryInfo to create the allocator for. May be nullptr. - * \param[in] allocator_options Optional key-value pairs for allocator options, can be nullptr. - * \param[out] allocator The created OrtAllocator instance. Set to nullptr if the default CPU allocator is used. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(CreateAllocator, _In_ OrtEpFactory* this_ptr, - _In_ const OrtMemoryInfo* memory_info, - _In_opt_ const OrtKeyValuePairs* allocator_options, - _Outptr_result_maybenull_ OrtAllocator** allocator); - - /** \brief Release an OrtAllocator created by the factory. - * - * \since Version 1.23. - */ - ORT_API_T(void, ReleaseAllocator, _In_ OrtEpFactory* this_ptr, _In_ OrtAllocator* allocator); - - /** \brief Create an OrtDataTransferImpl instance for the factory. - * - * This is used to create an IDataTransfer implementation that can be used to copy data between devices - * that the execution provider supports. - * - * \param[in] this_ptr The OrtEpFactory instance. - * \param[out] data_transfer The created OrtDataTransferImpl instance. Set to nullptr if not required. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(CreateDataTransfer, _In_ OrtEpFactory* this_ptr, - _Outptr_result_maybenull_ OrtDataTransferImpl** data_transfer); - - /** \brief Check if execution providers created by the factory are stream aware. - * - * \param[in] this_ptr The OrtEpFactory instance. - * \return True if the factory creates execution providers that are stream aware and it implements CreateSyncStreamForDevice. - * - * \since Version 1.23. - */ - ORT_API_T(bool, IsStreamAware, _In_ const OrtEpFactory* this_ptr); - - /** \brief Create a synchronization stream for the given memory device. - * - * This is used to create a synchronization stream for the memory device that can be used for operations outside of - * a session. - * - * \param[in] this_ptr The OrtEpFactory instance. - * \param[in] memory_device The OrtMemoryDevice to create the synchronization stream for. - * \param[in] stream_options Options for stream creation. May be nullptr. - * \param[out] stream The created OrtSyncStreamImpl instance. nullptr if the execution provider is not stream aware. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.23. - */ - ORT_API2_STATUS(CreateSyncStreamForDevice, _In_ OrtEpFactory* this_ptr, - _In_ const OrtMemoryDevice* memory_device, - _In_opt_ const OrtKeyValuePairs* stream_options, - _Outptr_ OrtSyncStreamImpl** stream); -}; - -#ifdef __cplusplus -} -#endif diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_ep_device_ep_metadata_keys.h b/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_ep_device_ep_metadata_keys.h deleted file mode 100644 index bbd6a43..0000000 --- a/prebuilt_libs/onnxruntime-win-x64-1.23.2/include/onnxruntime_ep_device_ep_metadata_keys.h +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -// This file contains well-known keys for OrtEpDevice EP metadata entries. -// It does NOT specify all available metadata keys. - -// Key for the execution provider version string. This should be available for all plugin EPs. -static const char* const kOrtEpDevice_EpMetadataKey_Version = "version"; - -// Prefix for execution provider compatibility information stored in model metadata. -// Used when generating EP context models to store compatibility strings for each EP. -// Full key format: "ep_compatibility_info." -static const char* const kOrtModelMetadata_EpCompatibilityInfoPrefix = "ep_compatibility_info."; - -// Key for the execution provider library path (for dynamically loaded EPs) -static const char* const kOrtEpDevice_EpMetadataKey_LibraryPath = "library_path"; diff --git a/prebuilt_libs/onnxruntime-win-x64-1.23.2/lib/onnxruntime_providers_shared.pdb b/prebuilt_libs/onnxruntime-win-x64-1.23.2/lib/onnxruntime_providers_shared.pdb deleted file mode 100644 index 012d6b0..0000000 Binary files a/prebuilt_libs/onnxruntime-win-x64-1.23.2/lib/onnxruntime_providers_shared.pdb and /dev/null differ diff --git a/prebuilt_libs/win-x64/my_yolo_sdk.dll b/prebuilt_libs/win-x64/my_yolo_sdk.dll index 6a900e7..3dfcfe8 100644 Binary files a/prebuilt_libs/win-x64/my_yolo_sdk.dll and b/prebuilt_libs/win-x64/my_yolo_sdk.dll differ diff --git a/prebuilt_libs/win-x64/opencv.txt b/prebuilt_libs/win-x64/prebuild_libs.txt similarity index 94% rename from prebuilt_libs/win-x64/opencv.txt rename to prebuilt_libs/win-x64/prebuild_libs.txt index 673a389..0501500 100644 Binary files a/prebuilt_libs/win-x64/opencv.txt and b/prebuilt_libs/win-x64/prebuild_libs.txt differ